Skip to content

Commit

Permalink
Remove auto_flush feature, it will be runtime-configurable
Browse files Browse the repository at this point in the history
  • Loading branch information
badboy committed Jul 31, 2024
1 parent 6b4210e commit 2ad6222
Show file tree
Hide file tree
Showing 4 changed files with 33 additions and 60 deletions.
2 changes: 0 additions & 2 deletions glean-core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -64,5 +64,3 @@ uniffi = { version = "0.27.0", default-features = false, features = ["build"] }
preinit_million_queue = []
# Enable `env_logger`. Only works on non-Android non-iOS targets.
enable_env_logger = ["env_logger"]
# Automatic count/time-based flushing when delayPingLifetimeIo=true
auto_flush = []
2 changes: 0 additions & 2 deletions glean-core/rlb/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,6 @@ tempfile = "3.1.0"

[features]
preinit_million_queue = ["glean-core/preinit_million_queue"]
auto_flush = ["glean-core/auto_flush"]

[[example]]
name = "ping-lifetime-flush"
required-features = ["auto_flush"]
2 changes: 1 addition & 1 deletion glean-core/rlb/tests/test-ping-lifetime-flush.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ trap cleanup INT ABRT TERM EXIT
tmp="${TMPDIR:-/tmp}"
datapath=$(mktemp -d "${tmp}/glean_ping_lifetime_flush.XXXX")

cmd="cargo run -p glean --example ping-lifetime-flush --features auto_flush -- $datapath"
cmd="cargo run -p glean --example ping-lifetime-flush -- $datapath"

# First run "crashes" -> no increment stored
$cmd accumulate_one_and_pretend_crash
Expand Down
87 changes: 32 additions & 55 deletions glean-core/src/database/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,17 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.

use std::cell::RefCell;
#[cfg(any(target_os = "android", feature = "auto_flush"))]
use std::cell::Cell;
use std::cell::{Cell, RefCell};
use std::collections::btree_map::Entry;
use std::collections::BTreeMap;
use std::fs;
use std::io;
use std::num::NonZeroU64;
use std::path::Path;
use std::str;
#[cfg(any(target_os = "android", feature = "auto_flush"))]
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::RwLock;
#[cfg(any(target_os = "android", feature = "auto_flush"))]
use std::time::Duration;
#[cfg(any(target_os = "android", feature = "auto_flush"))]
use std::time::Instant;
use std::time::{Duration, Instant};

use crate::ErrorKind;

Expand Down Expand Up @@ -193,11 +187,9 @@ use crate::Result;
/// before data is flushed to disk.
///
/// Only considered if `delay_ping_lifetime_io` is set to `true`.
#[cfg(any(target_os = "android", feature = "auto_flush"))]
const PING_LIFETIME_THRESHOLD: usize = 1000;

// Save atleast every 2 seconds.
#[cfg(any(target_os = "android", feature = "auto_flush"))]
const PING_LIFETIME_MAX_TIME: Duration = Duration::from_millis(2000);

pub struct Database {
Expand All @@ -222,7 +214,6 @@ pub struct Database {
/// A ping-lifetime flush is automatically done after `PING_LIFETIME_THRESHOLD` writes.
///
/// Only relevant if `delay_ping_lifetime_io` is set to `true`,
#[cfg(any(target_os = "android", feature = "auto_flush"))]
ping_lifetime_count: AtomicUsize,

/// The last time the `lifetime=ping` data was flushed to disk.
Expand All @@ -231,7 +222,6 @@ pub struct Database {
/// `PING_LIFETIME_MAX_TIME` ago.
///
/// Only relevant if `delay_ping_lifetime_io` is set to `true`,
#[cfg(any(target_os = "android", feature = "auto_flush"))]
ping_lifetime_store_ts: Cell<Instant>,

/// Initial file size when opening the database.
Expand Down Expand Up @@ -315,7 +305,6 @@ impl Database {
// The value was chosen at random.
let write_timings = RefCell::new(Vec::with_capacity(64));

#[cfg(any(target_os = "android", feature = "auto_flush"))]
let now = Instant::now();

let db = Self {
Expand All @@ -324,9 +313,7 @@ impl Database {
ping_store,
application_store,
ping_lifetime_data,
#[cfg(any(target_os = "android", feature = "auto_flush"))]
ping_lifetime_count: AtomicUsize::new(0),
#[cfg(any(target_os = "android", feature = "auto_flush"))]
ping_lifetime_store_ts: Cell::new(now),
file_size,
rkv_load_state,
Expand Down Expand Up @@ -875,9 +862,7 @@ impl Database {
.expect("Can't read ping lifetime data");

// We can reset the write-counter. Current data has been persisted.
#[cfg(any(target_os = "android", feature = "auto_flush"))]
self.ping_lifetime_count.store(0, Ordering::Release);
#[cfg(any(target_os = "android", feature = "auto_flush"))]
self.ping_lifetime_store_ts.replace(Instant::now());

self.write_with_store(Lifetime::Ping, |mut writer, store| {
Expand All @@ -900,49 +885,41 @@ impl Database {
&self,
data: &BTreeMap<String, Metric>,
) -> Result<()> {
#[cfg(any(target_os = "android", feature = "auto_flush"))]
{
self.ping_lifetime_count.fetch_add(1, Ordering::Release);

let write_count = self.ping_lifetime_count.load(Ordering::Relaxed);
let last_write = self.ping_lifetime_store_ts.get();
let elapsed = last_write.elapsed();
self.ping_lifetime_count.fetch_add(1, Ordering::Release);

if write_count < PING_LIFETIME_THRESHOLD && elapsed < PING_LIFETIME_MAX_TIME {
log::trace!("Not flushing. write_count={write_count}, elapsed={elapsed:?}");
return Ok(());
}
let write_count = self.ping_lifetime_count.load(Ordering::Relaxed);
let last_write = self.ping_lifetime_store_ts.get();
let elapsed = last_write.elapsed();

if write_count >= PING_LIFETIME_THRESHOLD {
log::debug!(
"Flushing database due to threshold of {PING_LIFETIME_THRESHOLD} reached."
)
} else if elapsed >= PING_LIFETIME_MAX_TIME {
log::debug!(
"Flushing database due to last write more than {PING_LIFETIME_MAX_TIME:?} ago"
);
}
if write_count < PING_LIFETIME_THRESHOLD && elapsed < PING_LIFETIME_MAX_TIME {
log::trace!("Not flushing. write_count={write_count}, elapsed={elapsed:?}");
return Ok(());
}

self.ping_lifetime_count.store(0, Ordering::Release);
self.ping_lifetime_store_ts.replace(Instant::now());
self.write_with_store(Lifetime::Ping, |mut writer, store| {
for (key, value) in data.iter() {
let encoded =
bincode::serialize(&value).expect("IMPOSSIBLE: Serializing metric failed");
// There is no need for `get_storage_key` here because
// the key is already formatted from when it was saved
// to ping_lifetime_data.
store.put(&mut writer, key, &rkv::Value::Blob(&encoded))?;
}
writer.commit()?;
Ok(())
})
if write_count >= PING_LIFETIME_THRESHOLD {
log::debug!(
"Flushing database due to threshold of {PING_LIFETIME_THRESHOLD} reached."
)
} else if elapsed >= PING_LIFETIME_MAX_TIME {
log::debug!(
"Flushing database due to last write more than {PING_LIFETIME_MAX_TIME:?} ago"
);
}
#[cfg(not(any(target_os = "android", feature = "auto_flush")))]
{
_ = data; // suppress unused_variables warning.

self.ping_lifetime_count.store(0, Ordering::Release);
self.ping_lifetime_store_ts.replace(Instant::now());
self.write_with_store(Lifetime::Ping, |mut writer, store| {
for (key, value) in data.iter() {
let encoded =
bincode::serialize(&value).expect("IMPOSSIBLE: Serializing metric failed");
// There is no need for `get_storage_key` here because
// the key is already formatted from when it was saved
// to ping_lifetime_data.
store.put(&mut writer, key, &rkv::Value::Blob(&encoded))?;
}
writer.commit()?;
Ok(())
}
})
}
}

Expand Down

0 comments on commit 2ad6222

Please sign in to comment.