From d9ca572849a1dd11a9d2de043c47d23362fa7eec Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Thu, 18 Jul 2024 16:05:01 +0200 Subject: [PATCH] Make flush write count threshold/time configurable --- .../java/mozilla/telemetry/glean/Glean.kt | 2 + .../telemetry/glean/config/Configuration.kt | 4 + .../ios/Glean/Config/Configuration.swift | 10 +- glean-core/ios/Glean/Glean.swift | 4 +- glean-core/python/glean/glean.py | 2 + .../python/glean/net/ping_upload_worker.py | 2 + .../rlb/examples/ping-lifetime-flush.rs | 2 + glean-core/rlb/src/configuration.rs | 25 +++++ glean-core/rlb/src/lib.rs | 2 + glean-core/src/core/mod.rs | 14 ++- glean-core/src/database/mod.rs | 93 +++++++++++-------- glean-core/src/glean.udl | 2 + glean-core/src/lib.rs | 5 + glean-core/src/lib_unit_tests.rs | 2 + glean-core/tests/common/mod.rs | 2 + glean-core/tests/event.rs | 2 + glean-core/tests/ping_maker.rs | 4 + 17 files changed, 133 insertions(+), 44 deletions(-) diff --git a/glean-core/android/src/main/java/mozilla/telemetry/glean/Glean.kt b/glean-core/android/src/main/java/mozilla/telemetry/glean/Glean.kt index c45d6e71d8..364503f6fa 100644 --- a/glean-core/android/src/main/java/mozilla/telemetry/glean/Glean.kt +++ b/glean-core/android/src/main/java/mozilla/telemetry/glean/Glean.kt @@ -260,6 +260,8 @@ open class GleanInternalAPI internal constructor() { experimentationId = configuration.experimentationId, enableInternalPings = configuration.enableInternalPings, pingSchedule = emptyMap(), + pingLifetimeThreshold = configuration.pingLifetimeThreshold.toULong(), + pingLifetimeMaxTime = configuration.pingLifetimeMaxTime.toULong(), ) val clientInfo = getClientInfo(configuration, buildInfo) val callbacks = OnGleanEventsImpl(this@GleanInternalAPI) diff --git a/glean-core/android/src/main/java/mozilla/telemetry/glean/config/Configuration.kt b/glean-core/android/src/main/java/mozilla/telemetry/glean/config/Configuration.kt index 4be7c5e2a3..4d690e56bd 100644 --- a/glean-core/android/src/main/java/mozilla/telemetry/glean/config/Configuration.kt +++ b/glean-core/android/src/main/java/mozilla/telemetry/glean/config/Configuration.kt @@ -25,6 +25,8 @@ import mozilla.telemetry.glean.net.PingUploader * to be sent with all pings. * @property enableInternalPings Whether to enable internal pings. * @property delayPingLifetimeIo Whether Glean should delay persistence of data from metrics with ping lifetime. + * @property pingLifetimeThreshold Write count threshold when to auto-flush. `0` disables it. + * @property pingLifetimeMaxTime After what time to auto-flush. 0 disables it. */ data class Configuration @JvmOverloads constructor( val serverEndpoint: String = DEFAULT_TELEMETRY_ENDPOINT, @@ -40,6 +42,8 @@ data class Configuration @JvmOverloads constructor( val experimentationId: String? = null, val enableInternalPings: Boolean = true, val delayPingLifetimeIo: Boolean = true, + val pingLifetimeThreshold: Int = 1000, + val pingLifetimeMaxTime: Int = 0, ) { companion object { /** diff --git a/glean-core/ios/Glean/Config/Configuration.swift b/glean-core/ios/Glean/Config/Configuration.swift index 93f6828dbd..705854e396 100644 --- a/glean-core/ios/Glean/Config/Configuration.swift +++ b/glean-core/ios/Glean/Config/Configuration.swift @@ -13,6 +13,8 @@ public struct Configuration { let enableEventTimestamps: Bool let experimentationId: String? let enableInternalPings: Bool + let pingLifetimeThreshold: Int + let pingLifetimeMaxTime: Int struct Constants { static let defaultTelemetryEndpoint = "https://incoming.telemetry.mozilla.org" @@ -32,6 +34,8 @@ public struct Configuration { /// * experimentationId An experimentation identifier derived by the application /// to be sent with all pings. /// * enableInternalPings Whether to enable internal pings. + /// * pingLifetimeThreshold Write count threshold when to auto-flush. `0` disables it. + /// * pingLifetimeMaxTime After what time to auto-flush. 0 disables it. public init( maxEvents: Int32? = nil, channel: String? = nil, @@ -40,7 +44,9 @@ public struct Configuration { logLevel: LevelFilter? = nil, enableEventTimestamps: Bool = true, experimentationId: String? = nil, - enableInternalPings: Bool = true + enableInternalPings: Bool = true, + pingLifetimeThreshold: Int = 0, + pingLifetimeMaxTime: Int = 0 ) { self.serverEndpoint = serverEndpoint ?? Constants.defaultTelemetryEndpoint self.maxEvents = maxEvents @@ -50,5 +56,7 @@ public struct Configuration { self.enableEventTimestamps = enableEventTimestamps self.experimentationId = experimentationId self.enableInternalPings = enableInternalPings + self.pingLifetimeThreshold = pingLifetimeThreshold + self.pingLifetimeMaxTime = pingLifetimeMaxTime } } diff --git a/glean-core/ios/Glean/Glean.swift b/glean-core/ios/Glean/Glean.swift index 43424a7928..ea3829c82c 100644 --- a/glean-core/ios/Glean/Glean.swift +++ b/glean-core/ios/Glean/Glean.swift @@ -196,7 +196,9 @@ public class Glean { enableEventTimestamps: configuration.enableEventTimestamps, experimentationId: configuration.experimentationId, enableInternalPings: configuration.enableInternalPings, - pingSchedule: [:] + pingSchedule: [:], + pingLifetimeThreshold: UInt64(configuration.pingLifetimeThreshold), + pingLifetimeMaxTime: UInt64(configuration.pingLifetimeMaxTime) ) let clientInfo = getClientInfo(configuration, buildInfo: buildInfo) let callbacks = OnGleanEventsImpl(glean: self) diff --git a/glean-core/python/glean/glean.py b/glean-core/python/glean/glean.py index b6ed757a2e..088306427f 100644 --- a/glean-core/python/glean/glean.py +++ b/glean-core/python/glean/glean.py @@ -235,6 +235,8 @@ def initialize( experimentation_id=configuration.experimentation_id, enable_internal_pings=configuration.enable_internal_pings, ping_schedule={}, + ping_lifetime_threshold=0, + ping_lifetime_max_time=0, ) _uniffi.glean_initialize(cfg, client_info, callbacks) diff --git a/glean-core/python/glean/net/ping_upload_worker.py b/glean-core/python/glean/net/ping_upload_worker.py index 008b07f719..6c00cbda15 100644 --- a/glean-core/python/glean/net/ping_upload_worker.py +++ b/glean-core/python/glean/net/ping_upload_worker.py @@ -120,6 +120,8 @@ def _process(data_dir: Path, application_id: str, configuration) -> bool: experimentation_id=None, enable_internal_pings=False, ping_schedule={}, + ping_lifetime_threshold=0, + ping_lifetime_max_time=0, ) if not glean_initialize_for_subprocess(cfg): log.error("Couldn't initialize Glean in subprocess") diff --git a/glean-core/rlb/examples/ping-lifetime-flush.rs b/glean-core/rlb/examples/ping-lifetime-flush.rs index 08c35ac816..2aedbe20bd 100644 --- a/glean-core/rlb/examples/ping-lifetime-flush.rs +++ b/glean-core/rlb/examples/ping-lifetime-flush.rs @@ -92,6 +92,8 @@ fn main() { .with_use_core_mps(false) .with_uploader(uploader) .with_delay_ping_lifetime_io(true) + .with_ping_lifetime_threshold(1000) + .with_ping_lifetime_max_time(Duration::from_millis(2000)) .build(); let client_info = ClientInfoMetrics { diff --git a/glean-core/rlb/src/configuration.rs b/glean-core/rlb/src/configuration.rs index 1f786254e1..12220009d3 100644 --- a/glean-core/rlb/src/configuration.rs +++ b/glean-core/rlb/src/configuration.rs @@ -8,6 +8,7 @@ use crate::net::PingUploader; use std::collections::HashMap; use std::path::PathBuf; +use std::time::Duration; /// The default server pings are sent to. pub(crate) const DEFAULT_GLEAN_ENDPOINT: &str = "https://incoming.telemetry.mozilla.org"; @@ -53,6 +54,10 @@ pub struct Configuration { /// Maps a ping name to a list of pings to schedule along with it. /// Only used if the ping's own ping schedule list is empty. pub ping_schedule: HashMap>, + /// Write count threshold when to auto-flush. `0` disables it. + pub ping_lifetime_threshold: usize, + /// After what time to auto-flush. 0 disables it. + pub ping_lifetime_max_time: Duration, } /// Configuration builder. @@ -105,6 +110,10 @@ pub struct Builder { /// Maps a ping name to a list of pings to schedule along with it. /// Only used if the ping's own ping schedule list is empty. pub ping_schedule: HashMap>, + /// Write count threshold when to auto-flush. `0` disables it. + pub ping_lifetime_threshold: usize, + /// After what time to auto-flush. 0 disables it. + pub ping_lifetime_max_time: Duration, } impl Builder { @@ -130,6 +139,8 @@ impl Builder { experimentation_id: None, enable_internal_pings: true, ping_schedule: HashMap::new(), + ping_lifetime_threshold: 0, + ping_lifetime_max_time: Duration::ZERO, } } @@ -151,6 +162,8 @@ impl Builder { experimentation_id: self.experimentation_id, enable_internal_pings: self.enable_internal_pings, ping_schedule: self.ping_schedule, + ping_lifetime_threshold: self.ping_lifetime_threshold, + ping_lifetime_max_time: self.ping_lifetime_max_time, } } @@ -213,4 +226,16 @@ impl Builder { self.ping_schedule = value; self } + + /// Write count threshold when to auto-flush. `0` disables it. + pub fn with_ping_lifetime_threshold(mut self, value: usize) -> Self { + self.ping_lifetime_threshold = value; + self + } + + /// After what time to auto-flush. 0 disables it. + pub fn with_ping_lifetime_max_time(mut self, value: Duration) -> Self { + self.ping_lifetime_max_time = value; + self + } } diff --git a/glean-core/rlb/src/lib.rs b/glean-core/rlb/src/lib.rs index 34b3670526..2aa2ba900b 100644 --- a/glean-core/rlb/src/lib.rs +++ b/glean-core/rlb/src/lib.rs @@ -124,6 +124,8 @@ fn initialize_internal(cfg: Configuration, client_info: ClientInfoMetrics) -> Op experimentation_id: cfg.experimentation_id, enable_internal_pings: cfg.enable_internal_pings, ping_schedule: cfg.ping_schedule, + ping_lifetime_threshold: cfg.ping_lifetime_threshold as u64, + ping_lifetime_max_time: cfg.ping_lifetime_max_time.as_millis() as u64, }; glean_core::glean_initialize(core_cfg, client_info.into(), callbacks); diff --git a/glean-core/src/core/mod.rs b/glean-core/src/core/mod.rs index a9a37ba390..985acdcee9 100644 --- a/glean-core/src/core/mod.rs +++ b/glean-core/src/core/mod.rs @@ -6,6 +6,7 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU8, Ordering}; use std::sync::{Arc, Mutex}; +use std::time::Duration; use chrono::{DateTime, FixedOffset}; use once_cell::sync::OnceCell; @@ -122,6 +123,8 @@ where /// experimentation_id: None, /// enable_internal_pings: true, /// ping_schedule: Default::default(), +/// ping_lifetime_threshold: 1000, +/// ping_lifetime_max_time: 2000, /// }; /// let mut glean = Glean::new(cfg).unwrap(); /// let ping = PingType::new("sample", true, false, true, true, true, vec![], vec![]); @@ -250,7 +253,14 @@ impl Glean { // Creating the data store creates the necessary path as well. // If that fails we bail out and don't initialize further. let data_path = Path::new(&cfg.data_path); - glean.data_store = Some(Database::new(data_path, cfg.delay_ping_lifetime_io)?); + let ping_lifetime_threshold = cfg.ping_lifetime_threshold as usize; + let ping_lifetime_max_time = Duration::from_millis(cfg.ping_lifetime_max_time); + glean.data_store = Some(Database::new( + data_path, + cfg.delay_ping_lifetime_io, + ping_lifetime_threshold, + ping_lifetime_max_time, + )?); // Set experimentation identifier (if any) if let Some(experimentation_id) = &cfg.experimentation_id { @@ -329,6 +339,8 @@ impl Glean { experimentation_id: None, enable_internal_pings, ping_schedule: Default::default(), + ping_lifetime_threshold: 0, + ping_lifetime_max_time: 0, }; let mut glean = Self::new(cfg).unwrap(); diff --git a/glean-core/src/database/mod.rs b/glean-core/src/database/mod.rs index cee2e05139..da712ff753 100644 --- a/glean-core/src/database/mod.rs +++ b/glean-core/src/database/mod.rs @@ -183,15 +183,6 @@ use crate::Glean; use crate::Lifetime; use crate::Result; -/// The number of writes we accept writes to the ping-lifetime in-memory map -/// before data is flushed to disk. -/// -/// Only considered if `delay_ping_lifetime_io` is set to `true`. -const PING_LIFETIME_THRESHOLD: usize = 1000; - -// Save atleast every 2 seconds. -const PING_LIFETIME_MAX_TIME: Duration = Duration::from_millis(2000); - pub struct Database { /// Handle to the database environment. rkv: Rkv, @@ -211,19 +202,25 @@ pub struct Database { /// A count of how many database writes have been done since the last ping-lifetime flush. /// - /// A ping-lifetime flush is automatically done after `PING_LIFETIME_THRESHOLD` writes. + /// A ping-lifetime flush is automatically done after `ping_lifetime_threshold` writes. /// /// Only relevant if `delay_ping_lifetime_io` is set to `true`, ping_lifetime_count: AtomicUsize, + /// Write-count threshold when to auto-flush. `0` disables it. + ping_lifetime_threshold: usize, + /// The last time the `lifetime=ping` data was flushed to disk. /// /// Data is flushed to disk automatically when the last flush was more than - /// `PING_LIFETIME_MAX_TIME` ago. + /// `ping_lifetime_max_time` ago. /// /// Only relevant if `delay_ping_lifetime_io` is set to `true`, ping_lifetime_store_ts: Cell, + /// After what time to auto-flush. 0 disables it. + ping_lifetime_max_time: Duration, + /// Initial file size when opening the database. file_size: Option, @@ -285,7 +282,12 @@ impl Database { /// /// It also loads any Lifetime::Ping data that might be /// persisted, in case `delay_ping_lifetime_io` is set. - pub fn new(data_path: &Path, delay_ping_lifetime_io: bool) -> Result { + pub fn new( + data_path: &Path, + delay_ping_lifetime_io: bool, + ping_lifetime_threshold: usize, + ping_lifetime_max_time: Duration, + ) -> Result { let path = data_path.join("db"); log::debug!("Database path: {:?}", path.display()); let file_size = database_size(&path); @@ -314,7 +316,9 @@ impl Database { application_store, ping_lifetime_data, ping_lifetime_count: AtomicUsize::new(0), + ping_lifetime_threshold, ping_lifetime_store_ts: Cell::new(now), + ping_lifetime_max_time, file_size, rkv_load_state, write_timings, @@ -885,24 +889,31 @@ impl Database { &self, data: &BTreeMap, ) -> Result<()> { - self.ping_lifetime_count.fetch_add(1, Ordering::Release); + if self.ping_lifetime_threshold == 0 && self.ping_lifetime_max_time.is_zero() { + log::trace!("Auto-flush disabled."); + return Ok(()); + } - let write_count = self.ping_lifetime_count.load(Ordering::Relaxed); + let write_count = self.ping_lifetime_count.fetch_add(1, Ordering::Release) + 1; let last_write = self.ping_lifetime_store_ts.get(); let elapsed = last_write.elapsed(); - if write_count < PING_LIFETIME_THRESHOLD && elapsed < PING_LIFETIME_MAX_TIME { + if (self.ping_lifetime_threshold == 0 || write_count < self.ping_lifetime_threshold) + && (self.ping_lifetime_max_time.is_zero() || elapsed < self.ping_lifetime_max_time) + { log::trace!("Not flushing. write_count={write_count}, elapsed={elapsed:?}"); return Ok(()); } - if write_count >= PING_LIFETIME_THRESHOLD { + if write_count >= self.ping_lifetime_threshold { log::debug!( - "Flushing database due to threshold of {PING_LIFETIME_THRESHOLD} reached." + "Flushing database due to threshold of {} reached.", + self.ping_lifetime_threshold ) - } else if elapsed >= PING_LIFETIME_MAX_TIME { + } else if elapsed >= self.ping_lifetime_max_time { log::debug!( - "Flushing database due to last write more than {PING_LIFETIME_MAX_TIME:?} ago" + "Flushing database due to last write more than {:?} ago", + self.ping_lifetime_max_time ); } @@ -933,7 +944,7 @@ mod test { #[test] fn test_panicks_if_fails_dir_creation() { let path = Path::new("/!#\"'@#°ç"); - assert!(Database::new(path, false).is_err()); + assert!(Database::new(path, false, 0, Duration::ZERO).is_err()); } #[test] @@ -951,7 +962,7 @@ mod test { let dir = tempdir().unwrap(); let path = dir.path().join(os_str); - let res = Database::new(&path, false); + let res = Database::new(&path, false, 0, Duration::ZERO); assert!( res.is_ok(), @@ -975,7 +986,7 @@ mod test { let dir = tempdir().unwrap(); let path = dir.path().join(os_str); - let res = Database::new(&path, false); + let res = Database::new(&path, false, 0, Duration::ZERO); assert!( res.is_ok(), "Database should not fail at {}: {:?}", @@ -998,7 +1009,7 @@ mod test { let dir = tempdir().unwrap(); let path = dir.path().join(os_str); - let res = Database::new(&path, false); + let res = Database::new(&path, false, 0, Duration::ZERO); assert!( res.is_err(), "Database should not fail at {}: {:?}", @@ -1010,7 +1021,7 @@ mod test { #[test] fn test_data_dir_rkv_inits() { let dir = tempdir().unwrap(); - Database::new(dir.path(), false).unwrap(); + Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); assert!(dir.path().exists()); } @@ -1019,7 +1030,7 @@ mod test { fn test_ping_lifetime_metric_recorded() { // Init the database in a temporary directory. let dir = tempdir().unwrap(); - let db = Database::new(dir.path(), false).unwrap(); + let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); assert!(db.ping_lifetime_data.is_none()); @@ -1055,7 +1066,7 @@ mod test { fn test_application_lifetime_metric_recorded() { // Init the database in a temporary directory. let dir = tempdir().unwrap(); - let db = Database::new(dir.path(), false).unwrap(); + let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); // Attempt to record a known value. let test_value = "test-value"; @@ -1092,7 +1103,7 @@ mod test { fn test_user_lifetime_metric_recorded() { // Init the database in a temporary directory. let dir = tempdir().unwrap(); - let db = Database::new(dir.path(), false).unwrap(); + let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); // Attempt to record a known value. let test_value = "test-value"; @@ -1126,7 +1137,7 @@ mod test { fn test_clear_ping_storage() { // Init the database in a temporary directory. let dir = tempdir().unwrap(); - let db = Database::new(dir.path(), false).unwrap(); + let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); // Attempt to record a known value for every single lifetime. let test_storage = "test-storage"; @@ -1201,7 +1212,7 @@ mod test { fn test_remove_single_metric() { // Init the database in a temporary directory. let dir = tempdir().unwrap(); - let db = Database::new(dir.path(), false).unwrap(); + let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); let test_storage = "test-storage-single-lifetime"; let metric_id_pattern = "telemetry_test.single_metric"; @@ -1257,7 +1268,7 @@ mod test { fn test_delayed_ping_lifetime_persistence() { // Init the database in a temporary directory. let dir = tempdir().unwrap(); - let db = Database::new(dir.path(), true).unwrap(); + let db = Database::new(dir.path(), true, 0, Duration::ZERO).unwrap(); let test_storage = "test-storage"; assert!(db.ping_lifetime_data.is_some()); @@ -1373,7 +1384,7 @@ mod test { let test_metric_id = "telemetry_test.test_name"; { - let db = Database::new(dir.path(), true).unwrap(); + let db = Database::new(dir.path(), true, 0, Duration::ZERO).unwrap(); // Attempt to record a known value. db.record_per_lifetime( @@ -1412,7 +1423,7 @@ mod test { // Now create a new instace of the db and check if data was // correctly loaded from rkv to memory. { - let db = Database::new(dir.path(), true).unwrap(); + let db = Database::new(dir.path(), true, 0, Duration::ZERO).unwrap(); // Verify that test_value is in memory. let data = match &db.ping_lifetime_data { @@ -1441,7 +1452,7 @@ mod test { fn test_delayed_ping_lifetime_clear() { // Init the database in a temporary directory. let dir = tempdir().unwrap(); - let db = Database::new(dir.path(), true).unwrap(); + let db = Database::new(dir.path(), true, 0, Duration::ZERO).unwrap(); let test_storage = "test-storage"; assert!(db.ping_lifetime_data.is_some()); @@ -1514,7 +1525,7 @@ mod test { // Attempt to record metric with the record and record_with functions, // this should work since upload is enabled. - let db = Database::new(dir.path(), true).unwrap(); + let db = Database::new(dir.path(), true, 0, Duration::ZERO).unwrap(); db.record(&glean, &test_data, &Metric::String("record".to_owned())); db.iter_store_from( Lifetime::Ping, @@ -1611,7 +1622,7 @@ mod test { let f = File::create(safebin).expect("create database file"); drop(f); - let db = Database::new(dir.path(), false).unwrap(); + let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); assert!(dir.path().exists()); assert!( @@ -1632,7 +1643,7 @@ mod test { let safebin = database_dir.join("data.safe.bin"); fs::write(safebin, "").expect("write to database file"); - let db = Database::new(dir.path(), false).unwrap(); + let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); assert!(dir.path().exists()); assert!( @@ -1681,7 +1692,7 @@ mod test { // First open should migrate the data. { - let db = Database::new(dir.path(), false).unwrap(); + let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); let safebin = database_dir.join("data.safe.bin"); assert!(safebin.exists(), "safe-mode file should exist"); assert!(!datamdb.exists(), "LMDB data should be deleted"); @@ -1701,7 +1712,7 @@ mod test { // Next open should not re-create the LMDB files. { - let db = Database::new(dir.path(), false).unwrap(); + let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); let safebin = database_dir.join("data.safe.bin"); assert!(safebin.exists(), "safe-mode file exists"); assert!(!datamdb.exists(), "LMDB data should not be recreated"); @@ -1780,7 +1791,7 @@ mod test { // First open should try migration and ignore it, because destination is not empty. // It also deletes the leftover LMDB database. { - let db = Database::new(dir.path(), false).unwrap(); + let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); let safebin = database_dir.join("data.safe.bin"); assert!(safebin.exists(), "safe-mode file should exist"); assert!(!datamdb.exists(), "LMDB data should be deleted"); @@ -1845,7 +1856,7 @@ mod test { // First open should try migration and ignore it, because destination is not empty. // It also deletes the leftover LMDB database. { - let db = Database::new(dir.path(), false).unwrap(); + let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); let safebin = database_dir.join("data.safe.bin"); assert!(safebin.exists(), "safe-mode file should exist"); assert!(!datamdb.exists(), "LMDB data should be deleted"); @@ -1890,7 +1901,7 @@ mod test { // safe-mode does not write an empty database to disk. // It also deletes the leftover LMDB database. { - let _db = Database::new(dir.path(), false).unwrap(); + let _db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); let safebin = database_dir.join("data.safe.bin"); assert!(!safebin.exists(), "safe-mode file should exist"); assert!(!datamdb.exists(), "LMDB data should be deleted"); diff --git a/glean-core/src/glean.udl b/glean-core/src/glean.udl index a94e0d54dc..bc8ca9ec36 100644 --- a/glean-core/src/glean.udl +++ b/glean-core/src/glean.udl @@ -94,6 +94,8 @@ dictionary InternalConfiguration { string? experimentation_id; boolean enable_internal_pings; record> ping_schedule; + u64 ping_lifetime_threshold; + u64 ping_lifetime_max_time; // in millis }; // How to specify the rate pings may be uploaded before they are throttled. diff --git a/glean-core/src/lib.rs b/glean-core/src/lib.rs index ad38e83ddc..5558f490b7 100644 --- a/glean-core/src/lib.rs +++ b/glean-core/src/lib.rs @@ -143,6 +143,11 @@ pub struct InternalConfiguration { /// Maps a ping name to a list of pings to schedule along with it. /// Only used if the ping's own ping schedule list is empty. pub ping_schedule: HashMap>, + + /// Write count threshold when to auto-flush. `0` disables it. + pub ping_lifetime_threshold: u64, + /// After what time to auto-flush. 0 disables it. + pub ping_lifetime_max_time: u64, } /// How to specify the rate at which pings may be uploaded before they are throttled. diff --git a/glean-core/src/lib_unit_tests.rs b/glean-core/src/lib_unit_tests.rs index 635ba39736..7ea320858a 100644 --- a/glean-core/src/lib_unit_tests.rs +++ b/glean-core/src/lib_unit_tests.rs @@ -199,6 +199,8 @@ fn experimentation_id_is_set_correctly() { experimentation_id: Some(experimentation_id.to_string()), enable_internal_pings: true, ping_schedule: Default::default(), + ping_lifetime_threshold: 0, + ping_lifetime_max_time: 0, }) .unwrap(); diff --git a/glean-core/tests/common/mod.rs b/glean-core/tests/common/mod.rs index 5891169374..7bc08436df 100644 --- a/glean-core/tests/common/mod.rs +++ b/glean-core/tests/common/mod.rs @@ -65,6 +65,8 @@ pub fn new_glean(tempdir: Option) -> (Glean, tempfile::TempDi experimentation_id: None, enable_internal_pings: true, ping_schedule: Default::default(), + ping_lifetime_threshold: 0, + ping_lifetime_max_time: 0, }; let glean = Glean::new(cfg).unwrap(); diff --git a/glean-core/tests/event.rs b/glean-core/tests/event.rs index 5495087871..9a27904d31 100644 --- a/glean-core/tests/event.rs +++ b/glean-core/tests/event.rs @@ -487,6 +487,8 @@ fn with_event_timestamps() { experimentation_id: None, // Enabling event timestamps enable_internal_pings: true, ping_schedule: Default::default(), + ping_lifetime_threshold: 0, + ping_lifetime_max_time: 0, }; let glean = Glean::new(cfg).unwrap(); diff --git a/glean-core/tests/ping_maker.rs b/glean-core/tests/ping_maker.rs index 242029fec3..32d5fc777a 100644 --- a/glean-core/tests/ping_maker.rs +++ b/glean-core/tests/ping_maker.rs @@ -93,6 +93,8 @@ fn test_metrics_must_report_experimentation_id() { experimentation_id: Some("test-experimentation-id".to_string()), enable_internal_pings: true, ping_schedule: Default::default(), + ping_lifetime_threshold: 0, + ping_lifetime_max_time: 0, }) .unwrap(); let ping_maker = PingMaker::new(); @@ -147,6 +149,8 @@ fn experimentation_id_is_removed_if_send_if_empty_is_false() { experimentation_id: Some("test-experimentation-id".to_string()), enable_internal_pings: true, ping_schedule: Default::default(), + ping_lifetime_threshold: 0, + ping_lifetime_max_time: 0, }) .unwrap(); let ping_maker = PingMaker::new();