From 63505e7673c2ca9c6b1074002a602b45c8679df3 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Thu, 7 Mar 2024 14:17:09 -0500 Subject: [PATCH 01/53] WIP --- Cargo.lock | 34 ++++++------ mobile_verifier/src/boosting_oracles/mod.rs | 55 ++++++------------- .../src/boosting_oracles/urbanization.rs | 41 ++++++++++++++ 3 files changed, 74 insertions(+), 56 deletions(-) create mode 100644 mobile_verifier/src/boosting_oracles/urbanization.rs diff --git a/Cargo.lock b/Cargo.lock index 0b17475e2..f62ca4d9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -442,7 +442,7 @@ dependencies = [ "arrayref", "base64 0.13.1", "bincode", - "borsh 0.9.3", + "borsh 0.10.3", "bytemuck", "getrandom 0.2.10", "solana-program", @@ -2020,7 +2020,7 @@ version = "0.1.0" source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang 0.26.0", + "anchor-lang 0.28.0", ] [[package]] @@ -2610,7 +2610,7 @@ version = "0.2.1" source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang 0.26.0", + "anchor-lang 0.28.0", ] [[package]] @@ -2993,7 +2993,7 @@ version = "0.1.0" source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang 0.26.0", + "anchor-lang 0.28.0", ] [[package]] @@ -3503,7 +3503,7 @@ version = "0.1.0" source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang 0.26.0", + "anchor-lang 0.28.0", "circuit-breaker", "data-credits", "fanout", @@ -3529,7 +3529,7 @@ dependencies = [ "bs58 0.5.0", "byteorder", "ed25519-compact", - "getrandom 0.1.16", + "getrandom 0.2.10", "k256", "lazy_static", "multihash", @@ -3550,7 +3550,7 @@ version = "0.2.4" source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang 0.26.0", + "anchor-lang 0.28.0", ] [[package]] @@ -3573,7 +3573,7 @@ version = "0.1.4" source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang 0.26.0", + "anchor-lang 0.28.0", ] [[package]] @@ -3612,7 +3612,7 @@ version = "0.0.2" source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang 0.26.0", + "anchor-lang 0.28.0", ] [[package]] @@ -4321,7 +4321,7 @@ version = "0.1.0" source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang 0.26.0", + "anchor-lang 0.28.0", ] [[package]] @@ -4330,7 +4330,7 @@ version = "0.2.0" source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang 0.26.0", + "anchor-lang 0.28.0", ] [[package]] @@ -4709,7 +4709,7 @@ version = "0.1.1" source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang 0.26.0", + "anchor-lang 0.28.0", ] [[package]] @@ -5409,7 +5409,7 @@ version = "0.2.1" source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang 0.26.0", + "anchor-lang 0.28.0", ] [[package]] @@ -5958,7 +5958,7 @@ version = "0.2.0" source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang 0.26.0", + "anchor-lang 0.28.0", ] [[package]] @@ -8139,7 +8139,7 @@ version = "0.2.0" source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang 0.26.0", + "anchor-lang 0.28.0", ] [[package]] @@ -8183,7 +8183,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", - "rand 0.7.3", + "rand 0.8.5", "static_assertions", ] @@ -8375,7 +8375,7 @@ version = "0.3.0" source = "git+https://github.com/helium/helium-anchor-gen.git#2c9e0741f19146c625ea15e9890b6f920f4c93d8" dependencies = [ "anchor-gen", - "anchor-lang 0.26.0", + "anchor-lang 0.28.0", ] [[package]] diff --git a/mobile_verifier/src/boosting_oracles/mod.rs b/mobile_verifier/src/boosting_oracles/mod.rs index 95f4f5d82..01cbac831 100644 --- a/mobile_verifier/src/boosting_oracles/mod.rs +++ b/mobile_verifier/src/boosting_oracles/mod.rs @@ -1,11 +1,27 @@ pub mod assignment; +pub mod urbanization; use std::collections::HashMap; +use std::path::Path; +use std::sync::Arc; use crate::geofence::GeofenceValidator; pub use assignment::Assignment; +pub use urbanization::Urbanization; use hextree::disktree::DiskTreeMap; +pub trait DataSet { + const PREFIX: &'static str; + + fn update(&mut self, path: &Path) -> hextree::Result<()>; + + fn assign(&self, hex: u64) -> Assignment; +} + +pub struct DataSetDownloaderDaemon { + data_set: T, +} + pub trait DiskTreeLike: Send + Sync + 'static { fn get(&self, cell: hextree::Cell) -> hextree::Result>; } @@ -29,42 +45,3 @@ impl DiskTreeLike for MockDiskTree { Ok(Some((cell, &[]))) } } - -pub struct Urbanization { - urbanized: DT, - usa_geofence: GF, -} - -impl Urbanization { - pub fn new(urbanized: DT, usa_geofence: GF) -> Self { - Self { - urbanized, - usa_geofence, - } - } -} - -impl Urbanization -where - DT: DiskTreeLike, - GF: GeofenceValidator, -{ - fn is_urbanized(&self, location: u64) -> anyhow::Result { - let cell = hextree::Cell::from_raw(location)?; - let result = self.urbanized.get(cell)?; - Ok(result.is_some()) - } - - pub fn hex_assignment(&self, hex: u64) -> anyhow::Result { - let assignment = if self.usa_geofence.in_valid_region(&hex) { - if self.is_urbanized(hex)? { - Assignment::A - } else { - Assignment::B - } - } else { - Assignment::C - }; - Ok(assignment) - } -} diff --git a/mobile_verifier/src/boosting_oracles/urbanization.rs b/mobile_verifier/src/boosting_oracles/urbanization.rs new file mode 100644 index 000000000..f92a0459d --- /dev/null +++ b/mobile_verifier/src/boosting_oracles/urbanization.rs @@ -0,0 +1,41 @@ +use super::Assignment; +use crate::geofence::GeofenceValidator; + +pub struct Urbanization { + urbanized: DT, + usa_geofence: GF, +} + +impl Urbanization { + pub fn new(urbanized: DT, usa_geofence: GF) -> Self { + Self { + urbanized, + usa_geofence, + } + } +} + +impl Urbanization +where + DT: DiskTreeLike, + GF: GeofenceValidator, +{ + fn is_urbanized(&self, location: u64) -> anyhow::Result { + let cell = hextree::Cell::from_raw(location)?; + let result = self.urbanized.get(cell)?; + Ok(result.is_some()) + } + + pub fn hex_assignment(&self, hex: u64) -> anyhow::Result { + let assignment = if self.usa_geofence.in_valid_region(&hex) { + if self.is_urbanized(hex)? { + Assignment::A + } else { + Assignment::B + } + } else { + Assignment::C + }; + Ok(assignment) + } +} From 7186673c17a14cff525997612013001f04ba194a Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Thu, 7 Mar 2024 15:21:34 -0500 Subject: [PATCH 02/53] WIP --- mobile_verifier/src/boosting_oracles/mod.rs | 23 +++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/mod.rs b/mobile_verifier/src/boosting_oracles/mod.rs index 01cbac831..565982a22 100644 --- a/mobile_verifier/src/boosting_oracles/mod.rs +++ b/mobile_verifier/src/boosting_oracles/mod.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use crate::geofence::GeofenceValidator; pub use assignment::Assignment; +use sqlx::PgPool; pub use urbanization::Urbanization; use hextree::disktree::DiskTreeMap; @@ -18,8 +19,26 @@ pub trait DataSet { fn assign(&self, hex: u64) -> Assignment; } -pub struct DataSetDownloaderDaemon { - data_set: T, +pub struct DataSetDownloaderDaemon { + pool: PgPool, + urbanization: A, + /* + footfall: B, + landtype: C, + */ +} + +impl DataSetDownloaderDaemon { + pub fn run(self) -> anyhow::Result<()> { + /* + find the next timestamp for each data set + insert that into the pending table, + wait until the given time and then download that file + update the data sets + */ + + Ok(()) + } } pub trait DiskTreeLike: Send + Sync + 'static { From 8337ed2bc580cca2e4ee944c9556d85ae34e19a6 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Tue, 12 Mar 2024 15:21:54 -0400 Subject: [PATCH 03/53] Implement automatic downloading of data set files --- file_store/src/cli/dump.rs | 9 + file_store/src/file_info.rs | 15 + mobile_verifier/migrations/28_data_sets.sql | 18 + mobile_verifier/src/boosting_oracles/mod.rs | 390 +++++++++++++++++- .../src/boosting_oracles/urbanization.rs | 53 ++- mobile_verifier/src/cli/server.rs | 42 +- mobile_verifier/src/coverage.rs | 51 +-- mobile_verifier/src/rewarder.rs | 6 + mobile_verifier/src/settings.rs | 4 +- mobile_verifier/tests/boosting_oracles.rs | 11 +- mobile_verifier/tests/hex_boosting.rs | 10 +- mobile_verifier/tests/modeled_coverage.rs | 11 +- mobile_verifier/tests/rewarder_poc_dc.rs | 10 +- 13 files changed, 536 insertions(+), 94 deletions(-) create mode 100644 mobile_verifier/migrations/28_data_sets.sql diff --git a/file_store/src/cli/dump.rs b/file_store/src/cli/dump.rs index 2ebc05755..7d42a1e0c 100644 --- a/file_store/src/cli/dump.rs +++ b/file_store/src/cli/dump.rs @@ -3,6 +3,7 @@ use crate::{ file_source, heartbeat::{CbrsHeartbeat, CbrsHeartbeatIngestReport}, iot_packet::IotValidPacket, + mobile_radio_threshold::VerifiedRadioThresholdIngestReport, mobile_session::{DataTransferSessionIngestReport, InvalidDataTransferIngestReport}, mobile_subscriber::{SubscriberLocationIngestReport, VerifiedSubscriberLocationIngestReport}, speedtest::{CellSpeedtest, CellSpeedtestIngestReport}, @@ -318,6 +319,14 @@ impl Cmd { "status": report.status, "recv_timestamp": report.report.received_timestamp}))?; } + FileType::VerifiedRadioThresholdIngestReport => { + let report = VerifiedRadioThresholdIngestReport::decode(msg)?; + print_json(&json!({ + "hotspot_pubkey": report.report.report.hotspot_pubkey, + "cbsd_id": report.report.report.cbsd_id, + "status": report.status, + }))?; + } _ => (), } } diff --git a/file_store/src/file_info.rs b/file_store/src/file_info.rs index b9c5ada2a..aa69a6375 100644 --- a/file_store/src/file_info.rs +++ b/file_store/src/file_info.rs @@ -149,6 +149,9 @@ pub const COVERAGE_OBJECT_INGEST_REPORT: &str = "coverage_object_ingest_report"; pub const SENIORITY_UPDATE: &str = "seniority_update"; pub const BOOSTED_HEX_UPDATE: &str = "boosted_hex_update"; pub const ORACLE_BOOSTING_REPORT: &str = "oracle_boosting_report"; +pub const URBANIZATION_DATA_SET: &str = "urbanization"; +pub const FOOTFALL_DATA_SET: &str = "footfall"; +pub const LANDTYPE_DATA_SET: &str = "landtype"; #[derive(Debug, PartialEq, Eq, Clone, Serialize, Copy, strum::EnumCount)] #[serde(rename_all = "snake_case")] @@ -195,6 +198,9 @@ pub enum FileType { RadioThresholdReq, RadioThresholdIngestReport, VerifiedRadioThresholdIngestReport, + UrbanizationDataSet, + FootfallDataSet, + LandtypeDataSet, } impl fmt::Display for FileType { @@ -246,6 +252,9 @@ impl fmt::Display for FileType { Self::SeniorityUpdate => SENIORITY_UPDATE, Self::BoostedHexUpdate => BOOSTED_HEX_UPDATE, Self::OracleBoostingReport => ORACLE_BOOSTING_REPORT, + Self::UrbanizationDataSet => URBANIZATION_DATA_SET, + Self::FootfallDataSet => FOOTFALL_DATA_SET, + Self::LandtypeDataSet => LANDTYPE_DATA_SET, }; f.write_str(s) } @@ -300,6 +309,9 @@ impl FileType { Self::SeniorityUpdate => SENIORITY_UPDATE, Self::BoostedHexUpdate => BOOSTED_HEX_UPDATE, Self::OracleBoostingReport => ORACLE_BOOSTING_REPORT, + Self::UrbanizationDataSet => URBANIZATION_DATA_SET, + Self::FootfallDataSet => FOOTFALL_DATA_SET, + Self::LandtypeDataSet => LANDTYPE_DATA_SET, } } } @@ -354,6 +366,9 @@ impl FromStr for FileType { SENIORITY_UPDATE => Self::SeniorityUpdate, BOOSTED_HEX_UPDATE => Self::BoostedHexUpdate, ORACLE_BOOSTING_REPORT => Self::OracleBoostingReport, + URBANIZATION_DATA_SET => Self::UrbanizationDataSet, + FOOTFALL_DATA_SET => Self::FootfallDataSet, + LANDTYPE_DATA_SET => Self::LandtypeDataSet, _ => return Err(Error::from(io::Error::from(io::ErrorKind::InvalidInput))), }; Ok(result) diff --git a/mobile_verifier/migrations/28_data_sets.sql b/mobile_verifier/migrations/28_data_sets.sql new file mode 100644 index 000000000..01f44d668 --- /dev/null +++ b/mobile_verifier/migrations/28_data_sets.sql @@ -0,0 +1,18 @@ +CREATE TYPE data_set_status AS enum ( + 'pending', + 'downloaded', + 'processed' +); + +CREATE TYPE data_set_type AS enum ( + 'urbanization', + 'footfall', + 'landtype' +); + +CREATE TABLE data_sets ( + filename TEXT PRIMARY KEY, + data_set data_set_type NOT NULL, + time_to_use TIMESTAMPTZ NOT NULL, + status data_set_status NOT NULL +); diff --git a/mobile_verifier/src/boosting_oracles/mod.rs b/mobile_verifier/src/boosting_oracles/mod.rs index 565982a22..d9cae7f73 100644 --- a/mobile_verifier/src/boosting_oracles/mod.rs +++ b/mobile_verifier/src/boosting_oracles/mod.rs @@ -1,46 +1,306 @@ +//! Module for handling the downloading of new data sets from S3. +//! +//! We've split this task into two parts, the CheckForNewDataSetsDaemon and the DataSetDownloaderDaemon. +//! As their names imply, the CheckForNewDataSetsDaemon is responsible for checking S3 for new files for +//! a given data set and inserting those new data sets into the `data_sets` table as [DataSetStatus::Pending]. +//! The DataSetDownloaderDaemon is responsible for checking the `data_sets` table and downloading any new +//! data sets and processing them. +//! +//! It seems unnecessary to split this task into two separate daemons, why not have one daemon that handles +//! both? Well, firstly, it's not two daemons, it's actually four (when all data sets are implemented), since +//! CheckForNewDataSetsDaemon only handles one type of data set. And yes, it is unnecessary, as it would be +//! possible in theory to put everything into one daemon that continuously polls S3 and checks if needs to +//! download new files and if some files need to be processed. +//! +//! But it would be extremely complicated. For one thing, the DataSetDownloaderDaemon needs to update the +//! data sets at a particular time, specified by the data set's timestamp. Keeping tracking of everything +//! in one place would make things more prone to bugs, and the implementation becomes a lot simpler. + pub mod assignment; pub mod urbanization; use std::collections::HashMap; use std::path::Path; +use std::pin::pin; use std::sync::Arc; -use crate::geofence::GeofenceValidator; +use crate::{ + boosting_oracles::assignment::urbanization_multiplier, + coverage::SignalLevel, + geofence::{Geofence, GeofenceValidator}, +}; pub use assignment::Assignment; -use sqlx::PgPool; -pub use urbanization::Urbanization; +use chrono::{DateTime, Duration, Utc}; +use file_store::{file_sink::FileSinkClient, traits::TimestampEncode, FileStore}; +use futures_util::{Stream, StreamExt, TryFutureExt, TryStreamExt}; +use helium_proto::services::poc_mobile as proto; use hextree::disktree::DiskTreeMap; +use rust_decimal::prelude::ToPrimitive; +use rust_decimal_macros::dec; +use sqlx::{FromRow, PgPool, QueryBuilder}; +use task_manager::ManagedTask; +use tokio::{fs::File, io::AsyncWriteExt, sync::Mutex}; +pub use urbanization::Urbanization; +use uuid::Uuid; pub trait DataSet { - const PREFIX: &'static str; + const TYPE: DataSetType; + + fn timestamp(&self) -> Option>; + + fn update(&mut self, path: &Path, time_to_use: DateTime) -> anyhow::Result<()>; + + fn assign(&self, hex: u64) -> anyhow::Result; +} + +pub struct DataSetDownloaderDaemon { + pool: PgPool, + // Later: footfall and landtype + urbanization: Arc>>, + store: FileStore, + oracle_boosting_sink: FileSinkClient, +} + +#[derive(FromRow)] +pub struct NewDataSet { + file_name: String, + time_to_use: DateTime, + status: DataSetStatus, +} + +#[derive(Copy, Clone, sqlx::Type)] +#[sqlx(type_name = "data_set_status")] +#[sqlx(rename_all = "lowercase")] +pub enum DataSetStatus { + Pending, + Downloaded, + Processed, +} + +impl DataSetStatus { + pub fn is_downloaded(&self) -> bool { + matches!(self, Self::Downloaded) + } +} + +impl ManagedTask for DataSetDownloaderDaemon { + fn start_task( + self: Box, + shutdown: triggered::Listener, + ) -> futures::prelude::future::LocalBoxFuture<'static, anyhow::Result<()>> { + let handle = tokio::spawn(self.run(shutdown)); + Box::pin( + handle + .map_err(anyhow::Error::from) + .and_then(|result| async move { result.map_err(anyhow::Error::from) }), + ) + } +} + +impl DataSetDownloaderDaemon { + pub fn new( + pool: PgPool, + urbanization: Arc>>, + store: FileStore, + oracle_boosting_sink: FileSinkClient, + ) -> Self { + Self { + pool, + urbanization, + store, + oracle_boosting_sink, + } + } + + pub async fn run(self, shutdown: triggered::Listener) -> anyhow::Result<()> { + // Another option I considered instead of polling was to use ENOTIFY, but that seemed + // not as good, as it would hog a pool connection. + // + // We set the poll duration to 30 minutes since that seems fine. + let poll_duration = Duration::minutes(30); + + loop { + // Find the latest urbanization file + tracing::info!("Checking for new data sets"); + let mut urbanization = self.urbanization.lock().await; + let curr_urbanization_data_set = urbanization.timestamp(); + let latest_data_set: Option = sqlx::query_as( + "SELECT file_name, time_to_use, status FROM data_sets WHERE status != 'processed' AND data_set = 'urbanization' AND COALESCE(time_to_use > $1, TRUE) AND time_to_use <= $2 ORDER BY time_to_use DESC LIMIT 1" + ) + .bind(curr_urbanization_data_set) + .bind(Utc::now()) + .fetch_optional(&self.pool) + .await?; + + if let Some(latest_data_set) = latest_data_set { + let path = format!( + "urbanization.{}.res10.h3tree", + latest_data_set.time_to_use.timestamp() + ); + + // Download the file if it hasn't been downloaded already: + if !latest_data_set.status.is_downloaded() { + tracing::info!("Downloading new data set: {path}"); + // TODO: abstract this out to a function + let mut bytes = self.store.get(latest_data_set.file_name.clone()).await?; + let mut file = File::open(Path::new(&path)).await?; + while let Some(bytes) = bytes.next().await.transpose()? { + file.write_all(&bytes).await?; + } + // Set the status to be downloaded + sqlx::query("UPDATE data_sets SET status = 'downloaded' WHERE file_name = $1") + .bind(&latest_data_set.file_name) + .execute(&self.pool) + .await?; + } + + // Now that we've downloaded the file, load it into the urbanization data set + urbanization.update(Path::new(&path), latest_data_set.time_to_use)?; + + // Update the hexes + let boosting_reports = set_oracle_boosting_assignments( + UnassignedHex::fetch_all(&self.pool), + &*urbanization, + &self.pool, + ) + .await?; + + sqlx::query("UPDATE data_sets SET status = 'processed' WHERE file_name = $1") + .bind(latest_data_set.file_name) + .execute(&self.pool) + .await?; + + self.oracle_boosting_sink + .write_all(boosting_reports) + .await?; + self.oracle_boosting_sink.commit().await?; + tracing::info!("Data set download complete"); + } + + // We don't want to shut down in the middle of downloading a data set, so we hold off until + // we are sleeping + #[rustfmt::skip] + tokio::select! { + biased; + _ = shutdown.clone() => { + tracing::info!("DataSetDownloaderDaemon shutting down"); + break; + } + _ = tokio::time::sleep(poll_duration.to_std()?) => { + continue; + } + } + } + + Ok(()) + } +} - fn update(&mut self, path: &Path) -> hextree::Result<()>; +#[derive(Copy, Clone, sqlx::Type)] +#[sqlx(type_name = "data_set_type")] +#[sqlx(rename_all = "lowercase")] +pub enum DataSetType { + Urbanization, + Footfall, + Landtype, +} - fn assign(&self, hex: u64) -> Assignment; +impl DataSetType { + pub fn to_prefix(self) -> &'static str { + match self { + Self::Urbanization => "urbanization", + Self::Footfall => "footfall", + Self::Landtype => "landtype", + } + } } -pub struct DataSetDownloaderDaemon { +// Better name welcome +pub struct CheckForNewDataSetDaemon { pool: PgPool, - urbanization: A, - /* - footfall: B, - landtype: C, - */ -} - -impl DataSetDownloaderDaemon { - pub fn run(self) -> anyhow::Result<()> { - /* - find the next timestamp for each data set - insert that into the pending table, - wait until the given time and then download that file - update the data sets - */ + store: FileStore, + data_set: DataSetType, +} + +impl ManagedTask for CheckForNewDataSetDaemon { + fn start_task( + self: Box, + shutdown: triggered::Listener, + ) -> futures::prelude::future::LocalBoxFuture<'static, anyhow::Result<()>> { + let handle = tokio::spawn(self.run(shutdown)); + Box::pin( + handle + .map_err(anyhow::Error::from) + .and_then(|result| async move { result.map_err(anyhow::Error::from) }), + ) + } +} + +impl CheckForNewDataSetDaemon { + pub fn new(pool: PgPool, store: FileStore, data_set: DataSetType) -> Self { + Self { + pool, + data_set, + store, + } + } + + pub async fn run(self, shutdown: triggered::Listener) -> anyhow::Result<()> { + // We should check for new data sets more often than we download them. 15 minutes seems fine. + let poll_duration = Duration::minutes(15); + + let prefix = self.data_set.to_prefix(); + let mut latest_file_date: Option> = + sqlx::query_scalar("SELECT time_to_use FROM data_sets ORDER BY time_to_use WHERE data_set = $1 DESC LIMIT 1") + .bind(self.data_set) + .fetch_optional(&self.pool) + .await?; + + loop { + #[rustfmt::skip] + tokio::select! { + biased; + _ = shutdown.clone() => { + tracing::info!("CheckForNewDataSetDaemon shutting down"); + break; + } + _ = tokio::time::sleep(poll_duration.to_std()?) => { + let mut new_data_sets = self.store.list(prefix, latest_file_date, None); + while let Some(new_data_set) = new_data_sets.next().await.transpose()? { + sqlx::query( + r#" + INSERT INTO data_sets (filename, data_set, time_to_use, status) + VALUES ($1, $2, $3, 'pending') + "#, + ) + .bind(new_data_set.key) + .bind(self.data_set) + .bind(new_data_set.timestamp) + .execute(&self.pool) + .await?; + latest_file_date = Some(new_data_set.timestamp); + } + } + } + } Ok(()) } } +/// Check if there are any pending or downloaded files prior to the given reward period +pub async fn check_for_unprocessed_data_sets( + pool: &PgPool, + period_end: DateTime, +) -> sqlx::Result { + sqlx::query_scalar( + "SELECT COUNT(*) > 0 FROM data_sets WHERE time_to_use <= $1 AND status != 'processed'", + ) + .bind(period_end) + .fetch_one(pool) + .await +} + pub trait DiskTreeLike: Send + Sync + 'static { fn get(&self, cell: hextree::Cell) -> hextree::Result>; } @@ -64,3 +324,89 @@ impl DiskTreeLike for MockDiskTree { Ok(Some((cell, &[]))) } } + +#[derive(FromRow)] +pub struct UnassignedHex { + uuid: uuid::Uuid, + #[sqlx(try_from = "i64")] + hex: u64, + signal_level: SignalLevel, + signal_power: i32, +} + +impl UnassignedHex { + pub fn fetch_all(pool: &PgPool) -> impl Stream> + '_ { + sqlx::query_as("SELECT uuid, hex, signal_level, signal_power FROM hexes").fetch(pool) + } + + pub fn fetch_unassigned(pool: &PgPool) -> impl Stream> + '_ { + sqlx::query_as( + "SELECT uuid, hex, signal_level, signal_power FROM hexes WHERE urbanized IS NULL", + ) + .fetch(pool) + } +} + +pub async fn set_oracle_boosting_assignments<'a>( + unassigned_hexes: impl Stream>, + urbanization: &Urbanization>, + pool: &'a PgPool, +) -> anyhow::Result> { + const NUMBER_OF_FIELDS_IN_QUERY: u16 = 5; + const ASSIGNMENTS_MAX_BATCH_ENTRIES: usize = (u16::MAX / NUMBER_OF_FIELDS_IN_QUERY) as usize; + + let now = Utc::now(); + let mut boost_results = HashMap::>::new(); + let mut unassigned_hexes = pin!(unassigned_hexes.try_chunks(ASSIGNMENTS_MAX_BATCH_ENTRIES)); + + while let Some(hexes) = unassigned_hexes.try_next().await? { + let hexes: anyhow::Result> = hexes + .into_iter() + .map(|hex| { + let urbanized = urbanization.hex_assignment(hex.hex)?; + let location = format!("{:x}", hex.hex); + let assignment_multiplier = (urbanization_multiplier(urbanized) * dec!(1000)) + .to_u32() + .unwrap_or(0); + + boost_results.entry(hex.uuid).or_default().push( + proto::OracleBoostingHexAssignment { + location, + urbanized: urbanized.into(), + assignment_multiplier, + }, + ); + + Ok((hex, urbanized)) + }) + .collect(); + + QueryBuilder::new("INSERT INTO hexes (uuid, hex, signal_level, signal_power, urbanized)") + .push_values(hexes?, |mut b, (hex, urbanized)| { + b.push_bind(hex.uuid) + .push_bind(hex.hex as i64) + .push_bind(hex.signal_level) + .push_bind(hex.signal_power) + .push_bind(urbanized); + }) + .push( + r#" + ON CONFLICT (uuid, hex) DO UPDATE SET + urbanized = EXCLUDED.urbanized + "#, + ) + .build() + .execute(pool) + .await?; + } + + Ok(boost_results + .into_iter() + .map( + move |(coverage_object, assignments)| proto::OracleBoostingReportV1 { + coverage_object: Vec::from(coverage_object.into_bytes()), + assignments, + timestamp: now.encode_timestamp(), + }, + )) +} diff --git a/mobile_verifier/src/boosting_oracles/urbanization.rs b/mobile_verifier/src/boosting_oracles/urbanization.rs index f92a0459d..155204044 100644 --- a/mobile_verifier/src/boosting_oracles/urbanization.rs +++ b/mobile_verifier/src/boosting_oracles/urbanization.rs @@ -1,18 +1,33 @@ -use super::Assignment; +use std::path::Path; + +use chrono::{DateTime, Utc}; +use hextree::disktree::DiskTreeMap; + +use super::{Assignment, DataSet, DataSetType, DiskTreeLike}; use crate::geofence::GeofenceValidator; pub struct Urbanization { - urbanized: DT, + urbanized: Option
, + timestamp: Option>, usa_geofence: GF, } impl Urbanization { - pub fn new(urbanized: DT, usa_geofence: GF) -> Self { + pub fn new(usa_geofence: GF) -> Self { Self { - urbanized, + urbanized: None, + timestamp: None, usa_geofence, } } + + pub fn new_mock(urbanized: DT, usa_geofence: GF) -> Self { + Self { + urbanized: Some(urbanized), + usa_geofence, + timestamp: None, + } + } } impl Urbanization @@ -20,9 +35,16 @@ where DT: DiskTreeLike, GF: GeofenceValidator, { + pub fn is_ready(&self) -> bool { + self.urbanized.is_some() + } + fn is_urbanized(&self, location: u64) -> anyhow::Result { + let Some(ref urbanized) = self.urbanized else { + anyhow::bail!("No urbanization data set file has been loaded"); + }; let cell = hextree::Cell::from_raw(location)?; - let result = self.urbanized.get(cell)?; + let result = urbanized.get(cell)?; Ok(result.is_some()) } @@ -39,3 +61,24 @@ where Ok(assignment) } } + +impl DataSet for Urbanization +where + GF: GeofenceValidator, +{ + const TYPE: DataSetType = DataSetType::Urbanization; + + fn timestamp(&self) -> Option> { + self.timestamp + } + + fn update(&mut self, path: &Path, time_to_use: DateTime) -> anyhow::Result<()> { + self.urbanized = Some(DiskTreeMap::open(path)?); + self.timestamp = Some(time_to_use); + Ok(()) + } + + fn assign(&self, hex: u64) -> anyhow::Result { + self.hex_assignment(hex) + } +} diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index c998af571..38501b4e2 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -1,9 +1,16 @@ +use std::sync::Arc; + use crate::{ - boosting_oracles::Urbanization, coverage::CoverageDaemon, data_session::DataSessionIngestor, - geofence::Geofence, heartbeats::cbrs::HeartbeatDaemon as CellHeartbeatDaemon, - heartbeats::wifi::HeartbeatDaemon as WifiHeartbeatDaemon, rewarder::Rewarder, - speedtests::SpeedtestDaemon, subscriber_location::SubscriberLocationIngestor, telemetry, - Settings, + boosting_oracles::{CheckForNewDataSetDaemon, DataSetDownloaderDaemon, Urbanization}, + coverage::CoverageDaemon, + data_session::DataSessionIngestor, + geofence::Geofence, + heartbeats::cbrs::HeartbeatDaemon as CellHeartbeatDaemon, + heartbeats::wifi::HeartbeatDaemon as WifiHeartbeatDaemon, + rewarder::Rewarder, + speedtests::SpeedtestDaemon, + subscriber_location::SubscriberLocationIngestor, + telemetry, Settings, }; use anyhow::Result; use chrono::Duration; @@ -14,13 +21,13 @@ use file_store::{ speedtest::CellSpeedtestIngestReport, wifi_heartbeat::WifiHeartbeatIngestReport, FileStore, FileType, }; -use hextree::disktree::DiskTreeMap; use mobile_config::client::{ entity_client::EntityClient, hex_boosting_client::HexBoostingClient, AuthorizationClient, CarrierServiceClient, GatewayClient, }; use price::PriceTracker; use task_manager::TaskManager; +use tokio::sync::Mutex; #[derive(Debug, clap::Args)] pub struct Cmd {} @@ -210,19 +217,32 @@ impl Cmd { .create() .await?; - let disktree = DiskTreeMap::open(&settings.urbanization_data_set)?; - let urbanization = Urbanization::new(disktree, usa_geofence); + let urbanization = Arc::new(Mutex::new(Urbanization::new(usa_geofence))); let coverage_daemon = CoverageDaemon::new( pool.clone(), auth_client.clone(), - urbanization, + urbanization.clone(), coverage_objs, valid_coverage_objs, - oracle_boosting_reports, + oracle_boosting_reports.clone(), ) .await?; + // Data sets and downloaders + let data_sets_file_store = FileStore::from_settings(&settings.data_sets).await?; + let data_set_downloader = DataSetDownloaderDaemon::new( + pool.clone(), + urbanization.clone(), + data_sets_file_store.clone(), + oracle_boosting_reports, + ); + let check_for_urbanization_data_sets = CheckForNewDataSetDaemon::new( + pool.clone(), + data_sets_file_store, + crate::boosting_oracles::DataSetType::Urbanization, + ); + // Mobile rewards let reward_period_hours = settings.rewards; let (mobile_rewards, mobile_rewards_server) = file_sink::FileSinkBuilder::new( @@ -323,6 +343,8 @@ impl Cmd { .add_task(rewarder) .add_task(subscriber_location_ingest_server) .add_task(data_session_ingestor) + .add_task(data_set_downloader) + .add_task(check_for_urbanization_data_sets) .start() .await } diff --git a/mobile_verifier/src/coverage.rs b/mobile_verifier/src/coverage.rs index b2aea9dae..2a4a7028c 100644 --- a/mobile_verifier/src/coverage.rs +++ b/mobile_verifier/src/coverage.rs @@ -1,6 +1,7 @@ use crate::{ boosting_oracles::{ - assignment::urbanization_multiplier, Assignment, DiskTreeLike, Urbanization, + assignment::urbanization_multiplier, set_oracle_boosting_assignments, Assignment, + DiskTreeLike, UnassignedHex, Urbanization, }, geofence::GeofenceValidator, heartbeats::{HbType, KeyType, OwnedKeyType}, @@ -21,17 +22,14 @@ use h3o::{CellIndex, LatLng}; use helium_crypto::PublicKeyBinary; use helium_proto::services::{ mobile_config::NetworkKeyRole, - poc_mobile::{ - self as proto, CoverageObjectValidity, OracleBoostingReportV1, - SignalLevel as SignalLevelProto, - }, + poc_mobile::{self as proto, CoverageObjectValidity, SignalLevel as SignalLevelProto}, }; use mobile_config::{ boosted_hex_info::{BoostedHex, BoostedHexes}, client::AuthorizationClient, }; use retainer::{entry::CacheReadGuard, Cache}; -use rust_decimal::{prelude::ToPrimitive, Decimal}; +use rust_decimal::Decimal; use rust_decimal_macros::dec; use sqlx::{FromRow, PgPool, Pool, Postgres, QueryBuilder, Transaction, Type}; use std::{ @@ -42,7 +40,7 @@ use std::{ time::Instant, }; use task_manager::ManagedTask; -use tokio::sync::mpsc::Receiver; +use tokio::sync::{mpsc::Receiver, Mutex}; use uuid::Uuid; #[derive(Copy, Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Type)] @@ -69,9 +67,8 @@ impl From for SignalLevel { pub struct CoverageDaemon { pool: Pool, auth_client: AuthorizationClient, - urbanization: Urbanization, + urbanization: Arc>>, coverage_objs: Receiver>, - initial_boosting_reports: Option>, coverage_obj_sink: FileSinkClient, oracle_boosting_sink: FileSinkClient, } @@ -84,20 +81,11 @@ where pub async fn new( pool: PgPool, auth_client: AuthorizationClient, - urbanization: Urbanization, + urbanization: Arc>>, coverage_objs: Receiver>, coverage_obj_sink: FileSinkClient, oracle_boosting_sink: FileSinkClient, ) -> anyhow::Result { - tracing::info!("Setting initial values for the urbanization column"); - - let unassigned_hexes = UnassignedHex::fetch(&pool); - let initial_boosting_reports = Some( - set_oracle_boosting_assignments(unassigned_hexes, &urbanization, &pool) - .await? - .collect(), - ); - Ok(Self { pool, auth_client, @@ -105,20 +93,10 @@ where coverage_objs, coverage_obj_sink, oracle_boosting_sink, - initial_boosting_reports, }) } pub async fn run(mut self, shutdown: triggered::Listener) -> anyhow::Result<()> { - let Some(initial_boosting_reports) = std::mem::take(&mut self.initial_boosting_reports) - else { - anyhow::bail!("Initial boosting reports is None"); - }; - self.oracle_boosting_sink - .write_all(initial_boosting_reports) - .await?; - self.oracle_boosting_sink.commit().await?; - loop { #[rustfmt::skip] tokio::select! { @@ -161,12 +139,15 @@ where self.coverage_obj_sink.commit().await?; transaction.commit().await?; - // After writing all of the coverage objects, we set their oracle boosting assignments. This is - // done in two steps to improve the testability of the assignments. - let unassigned_hexes = UnassignedHex::fetch(&self.pool); + // After writing all of the coverage objects, we set their oracle boosting assignments. + // If the urbanized data set is not ready, we leave everything as NULL. + let urbanization = self.urbanization.lock().await; + if !urbanization.is_ready() { + return Ok(()); + } + let unassigned_hexes = UnassignedHex::fetch_unassigned(&self.pool); let boosting_reports = - set_oracle_boosting_assignments(unassigned_hexes, &self.urbanization, &self.pool) - .await?; + set_oracle_boosting_assignments(unassigned_hexes, &*urbanization, &self.pool).await?; self.oracle_boosting_sink .write_all(boosting_reports) .await?; @@ -176,6 +157,7 @@ where } } +/* #[derive(FromRow)] pub struct UnassignedHex { uuid: Uuid, @@ -257,6 +239,7 @@ pub async fn set_oracle_boosting_assignments<'a>( }, )) } +*/ impl ManagedTask for CoverageDaemon where diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index f46e471aa..a698dc321 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -1,4 +1,5 @@ use crate::{ + boosting_oracles::check_for_unprocessed_data_sets, coverage, data_session, heartbeats::{self, HeartbeatReward}, reward_shares::{self, CoveragePoints, MapperShares, ServiceProviderShares, TransferRewards}, @@ -161,6 +162,11 @@ where tracing::info!("No speedtests found past reward period"); return Ok(false); } + + if check_for_unprocessed_data_sets(&self.pool, reward_period.end).await? { + tracing::info!("Data sets still need to be processed"); + return Ok(false); + } } else { tracing::info!("Complete data checks are disabled for this reward period"); } diff --git a/mobile_verifier/src/settings.rs b/mobile_verifier/src/settings.rs index b645836b2..7c486f8fb 100644 --- a/mobile_verifier/src/settings.rs +++ b/mobile_verifier/src/settings.rs @@ -1,7 +1,7 @@ use chrono::{DateTime, TimeZone, Utc}; use config::{Config, ConfigError, Environment, File}; use serde::Deserialize; -use std::path::{Path, PathBuf}; +use std::path::Path; #[derive(Debug, Deserialize)] pub struct Settings { @@ -20,6 +20,7 @@ pub struct Settings { pub ingest: file_store::Settings, pub data_transfer_ingest: file_store::Settings, pub output: file_store::Settings, + pub data_sets: file_store::Settings, pub metrics: poc_metrics::Settings, pub price_tracker: price::price_tracker::Settings, pub config_client: mobile_config::ClientSettings, @@ -42,7 +43,6 @@ pub struct Settings { pub usa_geofence_regions: String, #[serde(default = "default_fencing_resolution")] pub usa_fencing_resolution: u8, - pub urbanization_data_set: PathBuf, } fn default_fencing_resolution() -> u8 { diff --git a/mobile_verifier/tests/boosting_oracles.rs b/mobile_verifier/tests/boosting_oracles.rs index 359ec321b..e180d64c9 100644 --- a/mobile_verifier/tests/boosting_oracles.rs +++ b/mobile_verifier/tests/boosting_oracles.rs @@ -9,11 +9,8 @@ use helium_crypto::PublicKeyBinary; use helium_proto::services::poc_mobile::{CoverageObjectValidity, SignalLevel}; use mobile_config::boosted_hex_info::BoostedHexes; use mobile_verifier::{ - boosting_oracles::Urbanization, - coverage::{ - set_oracle_boosting_assignments, CoverageClaimTimeCache, CoverageObject, - CoverageObjectCache, Seniority, UnassignedHex, - }, + boosting_oracles::{set_oracle_boosting_assignments, UnassignedHex, Urbanization}, + coverage::{CoverageClaimTimeCache, CoverageObject, CoverageObjectCache, Seniority}, geofence::GeofenceValidator, heartbeats::{Heartbeat, HeartbeatReward, SeniorityUpdate, ValidatedHeartbeat}, reward_shares::CoveragePoints, @@ -162,8 +159,8 @@ async fn test_urbanization(pool: PgPool) -> anyhow::Result<()> { .await?; transaction.commit().await?; - let urbanization = Urbanization::new(urbanized, geofence); - let unassigned_hexes = UnassignedHex::fetch(&pool); + let urbanization = Urbanization::new_mock(urbanized, geofence); + let unassigned_hexes = UnassignedHex::fetch_unassigned(&pool); let _ = set_oracle_boosting_assignments(unassigned_hexes, &urbanization, &pool).await?; let heartbeats = heartbeats(12, start, &owner, &cbsd_id, 0.0, 0.0, uuid); diff --git a/mobile_verifier/tests/hex_boosting.rs b/mobile_verifier/tests/hex_boosting.rs index ba9646311..7b01384ca 100644 --- a/mobile_verifier/tests/hex_boosting.rs +++ b/mobile_verifier/tests/hex_boosting.rs @@ -17,9 +17,11 @@ use mobile_config::{ client::{hex_boosting_client::HexBoostingInfoResolver, ClientError}, }; use mobile_verifier::{ - boosting_oracles::{MockDiskTree, Urbanization}, + boosting_oracles::{ + set_oracle_boosting_assignments, MockDiskTree, UnassignedHex, Urbanization, + }, cell_type::CellType, - coverage::{set_oracle_boosting_assignments, CoverageObject, UnassignedHex}, + coverage::CoverageObject, geofence::GeofenceValidator, heartbeats::{HbType, Heartbeat, ValidatedHeartbeat}, reward_shares, rewarder, speedtests, @@ -69,8 +71,8 @@ impl GeofenceValidator for MockGeofence { } async fn update_assignments(pool: &PgPool) -> anyhow::Result<()> { - let urbanization = Urbanization::new(MockDiskTree, MockGeofence); - let unassigned_hexes = UnassignedHex::fetch(pool); + let urbanization = Urbanization::new_mock(MockDiskTree, MockGeofence); + let unassigned_hexes = UnassignedHex::fetch_unassigned(pool); let _ = set_oracle_boosting_assignments(unassigned_hexes, &urbanization, pool).await?; Ok(()) } diff --git a/mobile_verifier/tests/modeled_coverage.rs b/mobile_verifier/tests/modeled_coverage.rs index f759dbd86..b5a425e6d 100644 --- a/mobile_verifier/tests/modeled_coverage.rs +++ b/mobile_verifier/tests/modeled_coverage.rs @@ -13,11 +13,10 @@ use helium_proto::services::{ }; use mobile_config::boosted_hex_info::{BoostedHexInfo, BoostedHexes}; use mobile_verifier::{ - boosting_oracles::{MockDiskTree, Urbanization}, - coverage::{ - set_oracle_boosting_assignments, CoverageClaimTimeCache, CoverageObject, - CoverageObjectCache, Seniority, UnassignedHex, + boosting_oracles::{ + set_oracle_boosting_assignments, MockDiskTree, UnassignedHex, Urbanization, }, + coverage::{CoverageClaimTimeCache, CoverageObject, CoverageObjectCache, Seniority}, geofence::GeofenceValidator, heartbeats::{Heartbeat, HeartbeatReward, KeyType, SeniorityUpdate, ValidatedHeartbeat}, reward_shares::CoveragePoints, @@ -407,8 +406,8 @@ async fn process_input( } transaction.commit().await?; - let urbanization = Urbanization::new(MockDiskTree, MockGeofence); - let unassigned_hexes = UnassignedHex::fetch(pool); + let urbanization = Urbanization::new_mock(MockDiskTree, MockGeofence); + let unassigned_hexes = UnassignedHex::fetch_unassigned(pool); let _ = set_oracle_boosting_assignments(unassigned_hexes, &urbanization, pool).await?; let mut transaction = pool.begin().await?; diff --git a/mobile_verifier/tests/rewarder_poc_dc.rs b/mobile_verifier/tests/rewarder_poc_dc.rs index c6b15ac32..e0693f489 100644 --- a/mobile_verifier/tests/rewarder_poc_dc.rs +++ b/mobile_verifier/tests/rewarder_poc_dc.rs @@ -17,9 +17,11 @@ use mobile_config::{ client::{hex_boosting_client::HexBoostingInfoResolver, ClientError}, }; use mobile_verifier::{ - boosting_oracles::{MockDiskTree, Urbanization}, + boosting_oracles::{ + set_oracle_boosting_assignments, MockDiskTree, UnassignedHex, Urbanization, + }, cell_type::CellType, - coverage::{set_oracle_boosting_assignments, CoverageObject, UnassignedHex}, + coverage::CoverageObject, data_session, geofence::GeofenceValidator, heartbeats::{HbType, Heartbeat, ValidatedHeartbeat}, @@ -305,8 +307,8 @@ async fn seed_heartbeats( } async fn update_assignments(pool: &PgPool) -> anyhow::Result<()> { - let urbanization = Urbanization::new(MockDiskTree, MockGeofence); - let unassigned_hexes = UnassignedHex::fetch(pool); + let urbanization = Urbanization::new_mock(MockDiskTree, MockGeofence); + let unassigned_hexes = UnassignedHex::fetch_unassigned(pool); let _ = set_oracle_boosting_assignments(unassigned_hexes, &urbanization, pool).await?; Ok(()) } From d6cc42c314712c951042475445261ead7e5d8dda Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Tue, 12 Mar 2024 15:26:19 -0400 Subject: [PATCH 04/53] Revert dump.rs --- file_store/src/cli/dump.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/file_store/src/cli/dump.rs b/file_store/src/cli/dump.rs index 7d42a1e0c..2ebc05755 100644 --- a/file_store/src/cli/dump.rs +++ b/file_store/src/cli/dump.rs @@ -3,7 +3,6 @@ use crate::{ file_source, heartbeat::{CbrsHeartbeat, CbrsHeartbeatIngestReport}, iot_packet::IotValidPacket, - mobile_radio_threshold::VerifiedRadioThresholdIngestReport, mobile_session::{DataTransferSessionIngestReport, InvalidDataTransferIngestReport}, mobile_subscriber::{SubscriberLocationIngestReport, VerifiedSubscriberLocationIngestReport}, speedtest::{CellSpeedtest, CellSpeedtestIngestReport}, @@ -319,14 +318,6 @@ impl Cmd { "status": report.status, "recv_timestamp": report.report.received_timestamp}))?; } - FileType::VerifiedRadioThresholdIngestReport => { - let report = VerifiedRadioThresholdIngestReport::decode(msg)?; - print_json(&json!({ - "hotspot_pubkey": report.report.report.hotspot_pubkey, - "cbsd_id": report.report.report.cbsd_id, - "status": report.status, - }))?; - } _ => (), } } From 0e06a9a35ba9d8a90938f4a1beab1922d11d4205 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Wed, 3 Apr 2024 17:27:21 -0400 Subject: [PATCH 05/53] Fixes from merge --- .../{28_data_sets.sql => 32_data_sets.sql} | 0 .../src/boosting_oracles/footfall.rs | 62 +++++++ mobile_verifier/src/boosting_oracles/mod.rs | 161 +++++++----------- .../src/boosting_oracles/urbanization.rs | 60 +++---- mobile_verifier/src/cli/server.rs | 39 +++-- mobile_verifier/src/coverage.rs | 93 ++-------- mobile_verifier/src/settings.rs | 2 - mobile_verifier/tests/boosting_oracles.rs | 11 +- mobile_verifier/tests/common/mod.rs | 11 +- mobile_verifier/tests/hex_boosting.rs | 3 +- mobile_verifier/tests/modeled_coverage.rs | 5 +- mobile_verifier/tests/rewarder_poc_dc.rs | 1 + 12 files changed, 200 insertions(+), 248 deletions(-) rename mobile_verifier/migrations/{28_data_sets.sql => 32_data_sets.sql} (100%) create mode 100644 mobile_verifier/src/boosting_oracles/footfall.rs diff --git a/mobile_verifier/migrations/28_data_sets.sql b/mobile_verifier/migrations/32_data_sets.sql similarity index 100% rename from mobile_verifier/migrations/28_data_sets.sql rename to mobile_verifier/migrations/32_data_sets.sql diff --git a/mobile_verifier/src/boosting_oracles/footfall.rs b/mobile_verifier/src/boosting_oracles/footfall.rs new file mode 100644 index 000000000..b8b96b697 --- /dev/null +++ b/mobile_verifier/src/boosting_oracles/footfall.rs @@ -0,0 +1,62 @@ +use std::path::Path; + +use chrono::{DateTime, Utc}; +use hextree::disktree::DiskTreeMap; + +use super::{Assignment, DataSet, DataSetType, DiskTreeLike, HexAssignment}; + +pub struct Footfall { + footfall: Option, + timestamp: Option>, +} + +impl Footfall { + pub fn new() -> Self { + Self { + footfall: None, + timestamp: None, + } + } +} + +impl Default for Footfall { + fn default() -> Self { + Self::new() + } +} + +impl DataSet for Footfall { + const TYPE: DataSetType = DataSetType::Footfall; + + fn timestamp(&self) -> Option> { + self.timestamp + } + + fn update(&mut self, path: &Path, time_to_use: DateTime) -> anyhow::Result<()> { + self.footfall = Some(DiskTreeMap::open(path)?); + self.timestamp = Some(time_to_use); + Ok(()) + } + + fn is_ready(&self) -> bool { + self.footfall.is_some() + } +} + +impl HexAssignment for Footfall +where + Foot: DiskTreeLike + Send + Sync + 'static, +{ + fn assignment(&self, cell: hextree::Cell) -> anyhow::Result { + let Some(ref footfall) = self.footfall else { + anyhow::bail!("No footfall data set has been loaded"); + }; + + match footfall.get(cell)? { + Some((_, &[x])) if x >= 1 => Ok(Assignment::A), + Some((_, &[0])) => Ok(Assignment::B), + None => Ok(Assignment::C), + Some((_, other)) => anyhow::bail!("Unexpected disktree data: {cell:?} {other:?}"), + } + } +} diff --git a/mobile_verifier/src/boosting_oracles/mod.rs b/mobile_verifier/src/boosting_oracles/mod.rs index 94a76c31e..126ac6dd1 100644 --- a/mobile_verifier/src/boosting_oracles/mod.rs +++ b/mobile_verifier/src/boosting_oracles/mod.rs @@ -17,6 +17,7 @@ //! in one place would make things more prone to bugs, and the implementation becomes a lot simpler. pub mod assignment; +pub mod footfall; pub mod urbanization; use std::collections::HashMap; @@ -25,10 +26,7 @@ use std::pin::pin; use std::sync::Arc; use crate::{ - boosting_oracles::assignment::urbanization_multiplier, - coverage::SignalLevel, - geofence::{Geofence, GeofenceValidator}, - Settings, + boosting_oracles::assignment::footfall_and_urbanization_multiplier, coverage::SignalLevel, }; pub use assignment::Assignment; use chrono::{DateTime, Duration, Utc}; @@ -44,14 +42,14 @@ use tokio::{fs::File, io::AsyncWriteExt, sync::Mutex}; pub use urbanization::Urbanization; use uuid::Uuid; -pub trait DataSet { +pub trait DataSet: HexAssignment + Send + Sync + 'static { const TYPE: DataSetType; fn timestamp(&self) -> Option>; fn update(&mut self, path: &Path, time_to_use: DateTime) -> anyhow::Result<()>; - fn assign(&self, hex: u64) -> anyhow::Result; + fn is_ready(&self) -> bool; } pub struct DataSetDownloaderDaemon { @@ -85,9 +83,11 @@ impl DataSetStatus { } } -impl ManagedTask for DataSetDownloaderDaemon +impl ManagedTask for DataSetDownloaderDaemon where - T: DataSet + T: DataSet, + A: HexAssignment, + B: HexAssignment, { fn start_task( self: Box, @@ -102,7 +102,7 @@ where } } -impl DataSetDownloaderDaemon +impl DataSetDownloaderDaemon where T: DataSet, A: HexAssignment, @@ -111,7 +111,7 @@ where pub fn new( pool: PgPool, data_set: Arc>, - data_sets: Arc>>, + data_sets: HexBoostData, store: FileStore, oracle_boosting_sink: FileSinkClient, ) -> Self { @@ -134,7 +134,7 @@ where loop { // Find the latest urbanization file tracing::info!("Checking for new data sets"); - let mut data_set = self.urbanization.lock().await; + let mut data_set = self.data_set.lock().await; let curr_data_set = data_set.timestamp(); let latest_data_set: Option = sqlx::query_as( "SELECT file_name, time_to_use, status FROM data_sets WHERE status != 'processed' AND data_set = 'urbanization' AND COALESCE(time_to_use > $1, TRUE) AND time_to_use <= $2 ORDER BY time_to_use DESC LIMIT 1" @@ -316,21 +316,7 @@ pub async fn check_for_unprocessed_data_sets( .await } -pub fn make_hex_boost_data( - settings: &Settings, - usa_geofence: Geofence, -) -> anyhow::Result> { - let urban_disktree = DiskTreeMap::open(&settings.urbanization_data_set)?; - let footfall_disktree = DiskTreeMap::open(&settings.footfall_data_set)?; - - let urbanization = UrbanizationData::new(urban_disktree, usa_geofence); - let footfall_data = FootfallData::new(footfall_disktree); - let hex_boost_data = HexBoostData::new(urbanization, footfall_data); - - Ok(hex_boost_data) -} - -pub trait HexAssignment: Send + Sync { +pub trait HexAssignment: Send + Sync + 'static { fn assignment(&self, cell: hextree::Cell) -> anyhow::Result; } @@ -339,36 +325,31 @@ pub struct HexBoostData { pub footfall: Arc>, } -pub struct UrbanizationData { - urbanized: Urban, - usa_geofence: Geo, -} - -pub struct FootfallData { - footfall: Foot, -} - -impl HexBoostData { - pub fn new(urbanization: Urban, footfall: Foot) -> Self { +impl Clone for HexBoostData { + fn clone(&self) -> Self { Self { - urbanization, - footfall, + urbanization: self.urbanization.clone(), + footfall: self.footfall.clone(), } } } -impl UrbanizationData { - pub fn new(urbanized: Urban, usa_geofence: Geo) -> Self { +impl HexBoostData { + pub fn new(urbanization: Urban, footfall: Foot) -> Self { Self { - urbanized, - usa_geofence, + urbanization: Arc::new(Mutex::new(urbanization)), + footfall: Arc::new(Mutex::new(footfall)), } } } -impl FootfallData { - pub fn new(footfall: Foot) -> Self { - Self { footfall } +impl HexBoostData +where + Urban: DataSet, + Foot: DataSet, +{ + pub async fn is_ready(&self) -> bool { + self.urbanization.lock().await.is_ready() && self.footfall.lock().await.is_ready() } } @@ -384,7 +365,7 @@ impl DiskTreeLike for DiskTreeMap { impl DiskTreeLike for std::collections::HashSet { fn get(&self, cell: hextree::Cell) -> hextree::Result> { - Ok(self.contains(&cell).then(|| (cell, &[]))) + Ok(self.contains(&cell).then_some((cell, &[]))) } } @@ -420,7 +401,7 @@ impl UnassignedHex { pub async fn set_oracle_boosting_assignments<'a>( unassigned_hexes: impl Stream>, - data_sets: &HexAssignment, + data_sets: &HexBoostData, pool: &'a PgPool, ) -> anyhow::Result> { const NUMBER_OF_FIELDS_IN_QUERY: u16 = 5; @@ -430,17 +411,21 @@ pub async fn set_oracle_boosting_assignments<'a>( let mut boost_results = HashMap::>::new(); let mut unassigned_hexes = pin!(unassigned_hexes.try_chunks(ASSIGNMENTS_MAX_BATCH_ENTRIES)); + let urbanization = data_sets.urbanization.lock().await; + let footfall = data_sets.footfall.lock().await; + while let Some(hexes) = unassigned_hexes.try_next().await? { let hexes: anyhow::Result> = hexes .into_iter() .map(|hex| { let cell = hextree::Cell::try_from(hex.hex)?; - let urbanized = data_set.urbanization.assignment(cell)?; - let footfall = data_set.footfall.assignment(cell)?;// urbanization.hex_assignment(hex.hex)?; + let urbanized = urbanization.assignment(cell)?; + let footfall = footfall.assignment(cell)?; let location = format!("{:x}", hex.hex); - let assignment_multiplier = (urbanization_multiplier(urbanized) * dec!(1000)) - .to_u32() - .unwrap_or(0); + let assignment_multiplier = + (footfall_and_urbanization_multiplier(footfall, urbanized) * dec!(1000)) + .to_u32() + .unwrap_or(0); boost_results.entry(hex.uuid).or_default().push( proto::OracleBoostingHexAssignment { @@ -451,27 +436,31 @@ pub async fn set_oracle_boosting_assignments<'a>( }, ); - Ok((hex, urbanized)) + Ok((hex, urbanized, footfall)) }) .collect(); - QueryBuilder::new("INSERT INTO hexes (uuid, hex, signal_level, signal_power, urbanized)") - .push_values(hexes?, |mut b, (hex, urbanized)| { - b.push_bind(hex.uuid) - .push_bind(hex.hex as i64) - .push_bind(hex.signal_level) - .push_bind(hex.signal_power) - .push_bind(urbanized); - }) - .push( - r#" + QueryBuilder::new( + "INSERT INTO hexes (uuid, hex, signal_level, signal_power, urbanized, footfall)", + ) + .push_values(hexes?, |mut b, (hex, urbanized, footfall)| { + b.push_bind(hex.uuid) + .push_bind(hex.hex as i64) + .push_bind(hex.signal_level) + .push_bind(hex.signal_power) + .push_bind(urbanized) + .push_bind(footfall); + }) + .push( + r#" ON CONFLICT (uuid, hex) DO UPDATE SET - urbanized = EXCLUDED.urbanized + urbanized = EXCLUDED.urbanized, + footfall = EXCLUDED.footfall "#, - ) - .build() - .execute(pool) - .await?; + ) + .build() + .execute(pool) + .await?; } Ok(boost_results @@ -485,40 +474,6 @@ pub async fn set_oracle_boosting_assignments<'a>( )) } -impl HexAssignment for UrbanizationData -where - Urban: DiskTreeLike, - Geo: GeofenceValidator, -{ - fn assignment(&self, cell: hextree::Cell) -> anyhow::Result { - if !self.usa_geofence.in_valid_region(&cell) { - return Ok(Assignment::C); - } - - match self.urbanized.get(cell)?.is_some() { - true => Ok(Assignment::A), - false => Ok(Assignment::B), - } - } -} - -impl HexAssignment for FootfallData -where - Foot: DiskTreeLike, -{ - fn assignment(&self, cell: hextree::Cell) -> anyhow::Result { - let Some((_, vals)) = self.footfall.get(cell)? else { - return Ok(Assignment::C); - }; - - match vals { - &[x] if x >= 1 => Ok(Assignment::A), - &[0] => Ok(Assignment::B), - other => anyhow::bail!("unexpected disktree data: {cell:?} {other:?}"), - } - } -} - impl HexAssignment for Assignment { fn assignment(&self, _cell: hextree::Cell) -> anyhow::Result { Ok(*self) diff --git a/mobile_verifier/src/boosting_oracles/urbanization.rs b/mobile_verifier/src/boosting_oracles/urbanization.rs index 155204044..348a41500 100644 --- a/mobile_verifier/src/boosting_oracles/urbanization.rs +++ b/mobile_verifier/src/boosting_oracles/urbanization.rs @@ -3,7 +3,7 @@ use std::path::Path; use chrono::{DateTime, Utc}; use hextree::disktree::DiskTreeMap; -use super::{Assignment, DataSet, DataSetType, DiskTreeLike}; +use super::{Assignment, DataSet, DataSetType, DiskTreeLike, HexAssignment}; use crate::geofence::GeofenceValidator; pub struct Urbanization { @@ -30,41 +30,9 @@ impl Urbanization { } } -impl Urbanization -where - DT: DiskTreeLike, - GF: GeofenceValidator, -{ - pub fn is_ready(&self) -> bool { - self.urbanized.is_some() - } - - fn is_urbanized(&self, location: u64) -> anyhow::Result { - let Some(ref urbanized) = self.urbanized else { - anyhow::bail!("No urbanization data set file has been loaded"); - }; - let cell = hextree::Cell::from_raw(location)?; - let result = urbanized.get(cell)?; - Ok(result.is_some()) - } - - pub fn hex_assignment(&self, hex: u64) -> anyhow::Result { - let assignment = if self.usa_geofence.in_valid_region(&hex) { - if self.is_urbanized(hex)? { - Assignment::A - } else { - Assignment::B - } - } else { - Assignment::C - }; - Ok(assignment) - } -} - impl DataSet for Urbanization where - GF: GeofenceValidator, + GF: GeofenceValidator + Send + Sync + 'static, { const TYPE: DataSetType = DataSetType::Urbanization; @@ -78,7 +46,27 @@ where Ok(()) } - fn assign(&self, hex: u64) -> anyhow::Result { - self.hex_assignment(hex) + fn is_ready(&self) -> bool { + self.urbanized.is_some() + } +} + +impl HexAssignment for Urbanization +where + Urban: DiskTreeLike + Send + Sync + 'static, + Geo: GeofenceValidator + Send + Sync + 'static, +{ + fn assignment(&self, cell: hextree::Cell) -> anyhow::Result { + let Some(ref urbanized) = self.urbanized else { + anyhow::bail!("No urbanization data set has been loaded"); + }; + + if !self.usa_geofence.in_valid_region(&cell) { + Ok(Assignment::C) + } else if urbanized.get(cell)?.is_some() { + Ok(Assignment::A) + } else { + Ok(Assignment::B) + } } } diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index 830bf5e07..60b9b29fb 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -1,7 +1,8 @@ -use std::sync::Arc; - use crate::{ - boosting_oracles::{self, CheckForNewDataSetDaemon, DataSetDownloaderDaemon, Urbanization}, + boosting_oracles::{ + footfall::Footfall, CheckForNewDataSetDaemon, DataSetDownloaderDaemon, HexBoostData, + Urbanization, + }, coverage::CoverageDaemon, data_session::DataSessionIngestor, geofence::Geofence, @@ -26,13 +27,13 @@ use file_store::{ speedtest::CellSpeedtestIngestReport, wifi_heartbeat::WifiHeartbeatIngestReport, FileStore, FileType, }; +use hextree::disktree::DiskTreeMap; use mobile_config::client::{ entity_client::EntityClient, hex_boosting_client::HexBoostingClient, AuthorizationClient, CarrierServiceClient, GatewayClient, }; use price::PriceTracker; use task_manager::TaskManager; -use tokio::sync::Mutex; #[derive(Debug, clap::Args)] pub struct Cmd {} @@ -222,11 +223,14 @@ impl Cmd { .create() .await?; - let hex_boost_data = boosting_oracles::make_hex_boost_data(settings, usa_geofence)?; + let urbanization: Urbanization = Urbanization::new(usa_geofence); + let footfall: Footfall = Footfall::new(); + let hex_boost_data = HexBoostData::new(urbanization, footfall); + let coverage_daemon = CoverageDaemon::new( pool.clone(), auth_client.clone(), - hex_boost_data, + hex_boost_data.clone(), coverage_objs, valid_coverage_objs, oracle_boosting_reports.clone(), @@ -235,17 +239,30 @@ impl Cmd { // Data sets and downloaders let data_sets_file_store = FileStore::from_settings(&settings.data_sets).await?; - let data_set_downloader = DataSetDownloaderDaemon::new( + let urbanization_data_set_downloader = DataSetDownloaderDaemon::new( + pool.clone(), + hex_boost_data.urbanization.clone(), + hex_boost_data.clone(), + data_sets_file_store.clone(), + oracle_boosting_reports.clone(), + ); + let footfall_data_set_downloader = DataSetDownloaderDaemon::new( pool.clone(), - urbanization.clone(), + hex_boost_data.footfall.clone(), + hex_boost_data.clone(), data_sets_file_store.clone(), oracle_boosting_reports, ); let check_for_urbanization_data_sets = CheckForNewDataSetDaemon::new( pool.clone(), - data_sets_file_store, + data_sets_file_store.clone(), crate::boosting_oracles::DataSetType::Urbanization, ); + let check_for_footfall_data_sets = CheckForNewDataSetDaemon::new( + pool.clone(), + data_sets_file_store, + crate::boosting_oracles::DataSetType::Footfall, + ); // Mobile rewards let reward_period_hours = settings.rewards; @@ -412,8 +429,10 @@ impl Cmd { .add_task(radio_threshold_ingest_server) .add_task(invalidated_radio_threshold_ingest_server) .add_task(data_session_ingestor) - .add_task(data_set_downloader) + .add_task(urbanization_data_set_downloader) + .add_task(footfall_data_set_downloader) .add_task(check_for_urbanization_data_sets) + .add_task(check_for_footfall_data_sets) .start() .await } diff --git a/mobile_verifier/src/coverage.rs b/mobile_verifier/src/coverage.rs index 317133d31..a5d348b15 100644 --- a/mobile_verifier/src/coverage.rs +++ b/mobile_verifier/src/coverage.rs @@ -1,7 +1,7 @@ use crate::{ boosting_oracles::{ - set_oracle_boosting_assignments - assignment::footfall_and_urbanization_multiplier, Assignment, HexAssignment, HexBoostData, + assignment::footfall_and_urbanization_multiplier, set_oracle_boosting_assignments, + Assignment, DataSet, HexBoostData, UnassignedHex, }, heartbeats::{HbType, KeyType, OwnedKeyType}, IsAuthorized, @@ -40,7 +40,7 @@ use std::{ time::Instant, }; use task_manager::ManagedTask; -use tokio::sync::{mpsc::Receiver, Mutex}; +use tokio::sync::mpsc::Receiver; use uuid::Uuid; #[derive(Copy, Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Type)] @@ -64,14 +64,10 @@ impl From for SignalLevel { } } -pub struct CoverageDaemon -where - Urban: HexAssignment, - Foot: HexAssignment, -{ +pub struct CoverageDaemon { pool: Pool, auth_client: AuthorizationClient, - hex_boost_data: Arc>>, + hex_boost_data: HexBoostData, coverage_objs: Receiver>, coverage_obj_sink: FileSinkClient, oracle_boosting_sink: FileSinkClient, @@ -79,8 +75,8 @@ where impl CoverageDaemon where - Urban: HexAssignment, - Foot: HexAssignment, + Urban: DataSet, + Foot: DataSet, { pub async fn new( pool: PgPool, @@ -144,9 +140,8 @@ where transaction.commit().await?; // After writing all of the coverage objects, we set their oracle boosting assignments. - // If the urbanized data set is not ready, we leave everything as NULL. - let urbanization = self.urbanization.lock().await; - if !urbanization.is_ready() { + // If the data sets are not ready, we leave everything as NULL. + if !self.hex_boost_data.is_ready().await { return Ok(()); } let unassigned_hexes = UnassignedHex::fetch_unassigned(&self.pool); @@ -207,76 +202,10 @@ pub async fn set_oracle_boosting_assignments( } */ -async fn initialize_unassigned_hexes( - unassigned_urbinization_hexes: impl Stream>, - hex_boost_data: &HexBoostData, - pool: &Pool, -) -> Result>, anyhow::Error> { - const NUMBER_OF_FIELDS_IN_QUERY: u16 = 6; - const ASSIGNMENTS_MAX_BATCH_ENTRIES: usize = (u16::MAX / NUMBER_OF_FIELDS_IN_QUERY) as usize; - - let mut boost_results = HashMap::>::new(); - - let mut unassigned_hexes = - pin!(unassigned_urbinization_hexes.try_chunks(ASSIGNMENTS_MAX_BATCH_ENTRIES)); - - while let Some(hexes) = unassigned_hexes.try_next().await? { - let hexes: anyhow::Result> = hexes - .into_iter() - .map(|hex| { - let cell = hextree::Cell::from_raw(hex.hex)?; - let urbanized = hex_boost_data.urbanization.assignment(cell)?; - let footfall = hex_boost_data.footfall.assignment(cell)?; - - let location = hex.to_location_string(); - let assignment_multiplier = - (footfall_and_urbanization_multiplier(footfall, urbanized) * dec!(1000)) - .to_u32() - .unwrap_or(0); - - boost_results.entry(hex.uuid).or_default().push( - proto::OracleBoostingHexAssignment { - location, - urbanized: urbanized.into(), - footfall: footfall.into(), - assignment_multiplier, - }, - ); - - Ok((hex, urbanized, footfall)) - }) - .collect(); - - QueryBuilder::new( - "INSERT INTO hexes (uuid, hex, signal_level, signal_power, urbanized, footfall)", - ) - .push_values(hexes?, |mut b, (hex, urbanized, footfall)| { - b.push_bind(hex.uuid) - .push_bind(hex.hex as i64) - .push_bind(hex.signal_level) - .push_bind(hex.signal_power) - .push_bind(urbanized) - .push_bind(footfall); - }) - .push( - r#" - ON CONFLICT (uuid, hex) DO UPDATE SET - urbanized = EXCLUDED.urbanized, - footfall = EXCLUDED.footfall - "#, - ) - .build() - .execute(pool) - .await?; - } - - Ok(boost_results) -} - impl ManagedTask for CoverageDaemon where - Urban: HexAssignment + 'static, - Foot: HexAssignment + 'static, + Urban: DataSet, + Foot: DataSet, { fn start_task( self: Box, diff --git a/mobile_verifier/src/settings.rs b/mobile_verifier/src/settings.rs index 5c4d08c58..7c486f8fb 100644 --- a/mobile_verifier/src/settings.rs +++ b/mobile_verifier/src/settings.rs @@ -43,8 +43,6 @@ pub struct Settings { pub usa_geofence_regions: String, #[serde(default = "default_fencing_resolution")] pub usa_fencing_resolution: u8, - pub urbanization_data_set: PathBuf, - pub footfall_data_set: PathBuf, } fn default_fencing_resolution() -> u8 { diff --git a/mobile_verifier/tests/boosting_oracles.rs b/mobile_verifier/tests/boosting_oracles.rs index 1ae06b916..bbd6191a2 100644 --- a/mobile_verifier/tests/boosting_oracles.rs +++ b/mobile_verifier/tests/boosting_oracles.rs @@ -11,11 +11,10 @@ use helium_proto::services::poc_mobile::{ }; use mobile_config::boosted_hex_info::BoostedHexes; use mobile_verifier::{ - boosting_oracles::{Assignment, HexBoostData, UrbanizationData}, - coverage::{ - set_oracle_boosting_assignments, CoverageClaimTimeCache, CoverageObject, - CoverageObjectCache, Seniority, UnassignedHex, + boosting_oracles::{ + set_oracle_boosting_assignments, Assignment, HexBoostData, UnassignedHex, Urbanization, }, + coverage::{CoverageClaimTimeCache, CoverageObject, CoverageObjectCache, Seniority}, geofence::GeofenceValidator, heartbeats::{Heartbeat, HeartbeatReward, LocationCache, SeniorityUpdate, ValidatedHeartbeat}, radio_threshold::VerifiedRadioThresholds, @@ -233,7 +232,7 @@ async fn test_footfall_and_urbanization_report(pool: PgPool) -> anyhow::Result<( transaction.commit().await?; let unassigned_hexes = UnassignedHex::fetch_unassigned(&pool); - let urbanization = UrbanizationData::new(urbanized, geofence); + let urbanization = Urbanization::new_mock(urbanized, geofence); let hex_boost_data = HexBoostData::new(urbanization, footfall); let oba = set_oracle_boosting_assignments(unassigned_hexes, &hex_boost_data, &pool) .await? @@ -363,7 +362,7 @@ async fn test_footfall_and_urbanization(pool: PgPool) -> anyhow::Result<()> { transaction.commit().await?; let unassigned_hexes = UnassignedHex::fetch_unassigned(&pool); - let urbanization = UrbanizationData::new(urbanized, geofence); + let urbanization = Urbanization::new_mock(urbanized, geofence); let hex_boost_data = HexBoostData::new(urbanization, footfall); let _ = set_oracle_boosting_assignments(unassigned_hexes, &hex_boost_data, &pool).await?; diff --git a/mobile_verifier/tests/common/mod.rs b/mobile_verifier/tests/common/mod.rs index c3ae9b8f6..6555186d9 100644 --- a/mobile_verifier/tests/common/mod.rs +++ b/mobile_verifier/tests/common/mod.rs @@ -8,8 +8,11 @@ use helium_proto::{ }; use mobile_config::boosted_hex_info::BoostedHexInfo; use mobile_verifier::boosting_oracles::{Assignment, HexAssignment, HexBoostData}; -use std::collections::HashMap; -use tokio::{sync::mpsc::error::TryRecvError, time::timeout}; +use std::{collections::HashMap, sync::Arc}; +use tokio::{ + sync::{mpsc::error::TryRecvError, Mutex}, + time::timeout, +}; pub type ValidSpMap = HashMap; @@ -176,8 +179,8 @@ pub struct MockHexAssignments; impl MockHexAssignments { pub fn best() -> HexBoostData { HexBoostData { - urbanization: Assignment::A, - footfall: Assignment::A, + urbanization: Arc::new(Mutex::new(Assignment::A)), + footfall: Arc::new(Mutex::new(Assignment::A)), } } } diff --git a/mobile_verifier/tests/hex_boosting.rs b/mobile_verifier/tests/hex_boosting.rs index 0a52e7915..e97b3f87d 100644 --- a/mobile_verifier/tests/hex_boosting.rs +++ b/mobile_verifier/tests/hex_boosting.rs @@ -18,8 +18,9 @@ use mobile_config::{ client::{hex_boosting_client::HexBoostingInfoResolver, ClientError}, }; use mobile_verifier::{ + boosting_oracles::{set_oracle_boosting_assignments, UnassignedHex}, cell_type::CellType, - coverage::{set_oracle_boosting_assignments, CoverageObject, UnassignedHex}, + coverage::CoverageObject, heartbeats::{HbType, Heartbeat, ValidatedHeartbeat}, radio_threshold, reward_shares, rewarder, speedtests, }; diff --git a/mobile_verifier/tests/modeled_coverage.rs b/mobile_verifier/tests/modeled_coverage.rs index 5abe2a6d9..8b0db72ad 100644 --- a/mobile_verifier/tests/modeled_coverage.rs +++ b/mobile_verifier/tests/modeled_coverage.rs @@ -15,10 +15,7 @@ use helium_proto::services::{ use mobile_config::boosted_hex_info::{BoostedHexInfo, BoostedHexes}; use mobile_verifier::{ - coverage::{ - set_oracle_boosting_assignments, CoverageClaimTimeCache, CoverageObject, - CoverageObjectCache, Seniority, UnassignedHex, - }, + boosting_oracles::{set_oracle_boosting_assignments, UnassignedHex}, coverage::{CoverageClaimTimeCache, CoverageObject, CoverageObjectCache, Seniority}, geofence::GeofenceValidator, heartbeats::{ diff --git a/mobile_verifier/tests/rewarder_poc_dc.rs b/mobile_verifier/tests/rewarder_poc_dc.rs index 4b49c8789..c4cb654d3 100644 --- a/mobile_verifier/tests/rewarder_poc_dc.rs +++ b/mobile_verifier/tests/rewarder_poc_dc.rs @@ -17,6 +17,7 @@ use mobile_config::{ client::{hex_boosting_client::HexBoostingInfoResolver, ClientError}, }; use mobile_verifier::{ + boosting_oracles::{set_oracle_boosting_assignments, UnassignedHex}, cell_type::CellType, coverage::CoverageObject, data_session, From 1b7f80cadbb62aa0e0fb75f7e7f464db0f674e47 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Fri, 5 Apr 2024 15:37:44 -0400 Subject: [PATCH 06/53] Fixes --- Cargo.lock | 2 + mobile_verifier/Cargo.toml | 4 +- mobile_verifier/src/boosting_oracles/mod.rs | 67 +++++++++++++++------ mobile_verifier/src/cli/server.rs | 2 + mobile_verifier/src/settings.rs | 3 +- 5 files changed, 59 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4465b9c6a..010941a99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4751,6 +4751,7 @@ name = "mobile-verifier" version = "0.1.0" dependencies = [ "anyhow", + "async-compression", "async-trait", "backon", "base64 0.21.7", @@ -4790,6 +4791,7 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", + "tokio-util", "tonic", "tracing", "tracing-subscriber", diff --git a/mobile_verifier/Cargo.toml b/mobile_verifier/Cargo.toml index 00889525f..9c0c6c3c0 100644 --- a/mobile_verifier/Cargo.toml +++ b/mobile_verifier/Cargo.toml @@ -8,6 +8,7 @@ authors.workspace = true [dependencies] anyhow = {workspace = true} +async-compression = {version = "0", features = ["tokio", "gzip"]} config = {workspace = true} thiserror = {workspace = true} serde = {workspace = true} @@ -36,7 +37,8 @@ humantime = {workspace = true} rust_decimal = {workspace = true} rust_decimal_macros = {workspace = true} tonic = {workspace = true} -tokio-stream = { workspace = true } +tokio-stream = {workspace = true} +tokio-util = {workspace = true} metrics = {workspace = true} metrics-exporter-prometheus = {workspace = true} mobile-config = {path = "../mobile_config"} diff --git a/mobile_verifier/src/boosting_oracles/mod.rs b/mobile_verifier/src/boosting_oracles/mod.rs index 126ac6dd1..390f81519 100644 --- a/mobile_verifier/src/boosting_oracles/mod.rs +++ b/mobile_verifier/src/boosting_oracles/mod.rs @@ -20,7 +20,7 @@ pub mod assignment; pub mod footfall; pub mod urbanization; -use std::collections::HashMap; +use std::{collections::HashMap, path::PathBuf}; use std::path::Path; use std::pin::pin; use std::sync::Arc; @@ -59,11 +59,12 @@ pub struct DataSetDownloaderDaemon { data_sets: HexBoostData, store: FileStore, oracle_boosting_sink: FileSinkClient, + data_set_directory: PathBuf, } #[derive(FromRow)] pub struct NewDataSet { - file_name: String, + filename: String, time_to_use: DateTime, status: DataSetStatus, } @@ -114,6 +115,7 @@ where data_sets: HexBoostData, store: FileStore, oracle_boosting_sink: FileSinkClient, + data_set_directory: PathBuf, ) -> Self { Self { pool, @@ -121,10 +123,36 @@ where data_sets, store, oracle_boosting_sink, + data_set_directory, } } + fn get_data_set_path(&self, time_to_use: DateTime) -> PathBuf { + let path = PathBuf::from(format!( + "{}.{}.res10.h3tree", + T::TYPE.to_prefix(), + time_to_use.timestamp() + )); + let mut dir = self.data_set_directory.clone(); + dir.push(path); + dir + } + pub async fn run(self, shutdown: triggered::Listener) -> anyhow::Result<()> { + // Get the first data set: + if let Some(time_to_use) = + sqlx::query_scalar( + "SELECT time_to_use FROM data_sets WHERE status = 'processed' AND data_set = $1 ORDER BY time_to_use DESC LIMIT 1" + ) + .bind(T::TYPE) + .fetch_optional(&self.pool) + .await? + { + let data_set_path = self.get_data_set_path(time_to_use); + tracing::info!("Found initial {} data set: {}", T::TYPE.to_prefix(), data_set_path.to_string_lossy()); + self.data_set.lock().await.update(&data_set_path, time_to_use)?; + } + // Another option I considered instead of polling was to use ENOTIFY, but that seemed // not as good, as it would hog a pool connection. // @@ -137,32 +165,35 @@ where let mut data_set = self.data_set.lock().await; let curr_data_set = data_set.timestamp(); let latest_data_set: Option = sqlx::query_as( - "SELECT file_name, time_to_use, status FROM data_sets WHERE status != 'processed' AND data_set = 'urbanization' AND COALESCE(time_to_use > $1, TRUE) AND time_to_use <= $2 ORDER BY time_to_use DESC LIMIT 1" + "SELECT filename, time_to_use, status FROM data_sets WHERE status != 'processed' AND data_set = $1 AND COALESCE(time_to_use > $2, TRUE) AND time_to_use <= $3 ORDER BY time_to_use DESC LIMIT 1" ) + .bind(T::TYPE) .bind(curr_data_set) .bind(Utc::now()) .fetch_optional(&self.pool) .await?; if let Some(latest_data_set) = latest_data_set { - let path = format!( - "{}.{}.res10.h3tree", - T::TYPE.to_prefix(), - latest_data_set.time_to_use.timestamp() - ); + let path = self.get_data_set_path(latest_data_set.time_to_use); // Download the file if it hasn't been downloaded already: if !latest_data_set.status.is_downloaded() { - tracing::info!("Downloading new data set: {path}"); + tracing::info!("Downloading new data set: {}", path.to_string_lossy()); // TODO: abstract this out to a function - let mut bytes = self.store.get(latest_data_set.file_name.clone()).await?; - let mut file = File::open(Path::new(&path)).await?; + let stream = self.store.get_raw(latest_data_set.filename.clone()).await?; + let mut bytes = tokio_util::codec::FramedRead::new( + async_compression::tokio::bufread::GzipDecoder::new( + tokio_util::io::StreamReader::new(stream) + ), + tokio_util::codec::BytesCodec::new(), + ); + let mut file = File::create(&path).await?; while let Some(bytes) = bytes.next().await.transpose()? { file.write_all(&bytes).await?; } // Set the status to be downloaded - sqlx::query("UPDATE data_sets SET status = 'downloaded' WHERE file_name = $1") - .bind(&latest_data_set.file_name) + sqlx::query("UPDATE data_sets SET status = 'downloaded' WHERE filename = $1") + .bind(&latest_data_set.filename) .execute(&self.pool) .await?; } @@ -180,8 +211,8 @@ where ) .await?; - sqlx::query("UPDATE data_sets SET status = 'processed' WHERE file_name = $1") - .bind(latest_data_set.file_name) + sqlx::query("UPDATE data_sets SET status = 'processed' WHERE filename = $1") + .bind(latest_data_set.filename) .execute(&self.pool) .await?; @@ -266,7 +297,7 @@ impl CheckForNewDataSetDaemon { let prefix = self.data_set.to_prefix(); let mut latest_file_date: Option> = - sqlx::query_scalar("SELECT time_to_use FROM data_sets ORDER BY time_to_use WHERE data_set = $1 DESC LIMIT 1") + sqlx::query_scalar("SELECT time_to_use FROM data_sets WHERE data_set = $1 ORDER BY time_to_use DESC LIMIT 1") .bind(self.data_set) .fetch_optional(&self.pool) .await?; @@ -280,9 +311,11 @@ impl CheckForNewDataSetDaemon { break; } _ = tokio::time::sleep(poll_duration.to_std()?) => { + // tracing::info!("Checking file store for new data sets"); let mut new_data_sets = self.store.list(prefix, latest_file_date, None); while let Some(new_data_set) = new_data_sets.next().await.transpose()? { - sqlx::query( + tracing::info!("Found new data set: {}, {:#?}", new_data_set.key, new_data_set); + sqlx::query( r#" INSERT INTO data_sets (filename, data_set, time_to_use, status) VALUES ($1, $2, $3, 'pending') diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index 60b9b29fb..711f0395e 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -245,6 +245,7 @@ impl Cmd { hex_boost_data.clone(), data_sets_file_store.clone(), oracle_boosting_reports.clone(), + settings.data_sets_directory.clone(), ); let footfall_data_set_downloader = DataSetDownloaderDaemon::new( pool.clone(), @@ -252,6 +253,7 @@ impl Cmd { hex_boost_data.clone(), data_sets_file_store.clone(), oracle_boosting_reports, + settings.data_sets_directory.clone(), ); let check_for_urbanization_data_sets = CheckForNewDataSetDaemon::new( pool.clone(), diff --git a/mobile_verifier/src/settings.rs b/mobile_verifier/src/settings.rs index 7c486f8fb..f309d306b 100644 --- a/mobile_verifier/src/settings.rs +++ b/mobile_verifier/src/settings.rs @@ -1,7 +1,7 @@ use chrono::{DateTime, TimeZone, Utc}; use config::{Config, ConfigError, Environment, File}; use serde::Deserialize; -use std::path::Path; +use std::path::{Path, PathBuf}; #[derive(Debug, Deserialize)] pub struct Settings { @@ -36,6 +36,7 @@ pub struct Settings { /// beyond which its location weight will be reduced #[serde(default = "default_max_asserted_distance_deviation")] pub max_asserted_distance_deviation: u32, + pub data_sets_directory: PathBuf, // Geofencing settings pub usa_and_mexico_geofence_regions: String, #[serde(default = "default_fencing_resolution")] From f7e650a8146d84a83581a7c43200fab223e941b2 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Fri, 5 Apr 2024 15:39:51 -0400 Subject: [PATCH 07/53] Fmt --- mobile_verifier/src/boosting_oracles/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/mod.rs b/mobile_verifier/src/boosting_oracles/mod.rs index 390f81519..baf69a2fa 100644 --- a/mobile_verifier/src/boosting_oracles/mod.rs +++ b/mobile_verifier/src/boosting_oracles/mod.rs @@ -20,10 +20,10 @@ pub mod assignment; pub mod footfall; pub mod urbanization; -use std::{collections::HashMap, path::PathBuf}; use std::path::Path; use std::pin::pin; use std::sync::Arc; +use std::{collections::HashMap, path::PathBuf}; use crate::{ boosting_oracles::assignment::footfall_and_urbanization_multiplier, coverage::SignalLevel, @@ -183,7 +183,7 @@ where let stream = self.store.get_raw(latest_data_set.filename.clone()).await?; let mut bytes = tokio_util::codec::FramedRead::new( async_compression::tokio::bufread::GzipDecoder::new( - tokio_util::io::StreamReader::new(stream) + tokio_util::io::StreamReader::new(stream), ), tokio_util::codec::BytesCodec::new(), ); @@ -315,7 +315,7 @@ impl CheckForNewDataSetDaemon { let mut new_data_sets = self.store.list(prefix, latest_file_date, None); while let Some(new_data_set) = new_data_sets.next().await.transpose()? { tracing::info!("Found new data set: {}, {:#?}", new_data_set.key, new_data_set); - sqlx::query( + sqlx::query( r#" INSERT INTO data_sets (filename, data_set, time_to_use, status) VALUES ($1, $2, $3, 'pending') From fdb3dc88ef2f4a678f45343f644fe02eb2588bcb Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Wed, 10 Apr 2024 15:03:15 -0400 Subject: [PATCH 08/53] Lock the hexes table in order to prevent deadlock --- mobile_verifier/src/boosting_oracles/mod.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mobile_verifier/src/boosting_oracles/mod.rs b/mobile_verifier/src/boosting_oracles/mod.rs index baf69a2fa..50fd90e8c 100644 --- a/mobile_verifier/src/boosting_oracles/mod.rs +++ b/mobile_verifier/src/boosting_oracles/mod.rs @@ -473,6 +473,10 @@ pub async fn set_oracle_boosting_assignments<'a>( }) .collect(); + let mut transaction = pool.begin().await?; + sqlx::query("LOCK TABLE hexes") + .execute(&mut transaction) + .await?; QueryBuilder::new( "INSERT INTO hexes (uuid, hex, signal_level, signal_power, urbanized, footfall)", ) @@ -492,8 +496,9 @@ pub async fn set_oracle_boosting_assignments<'a>( "#, ) .build() - .execute(pool) + .execute(&mut transaction) .await?; + transaction.commit().await?; } Ok(boost_results From 7e7c3a2f8c46bb04d6f0cd381137348bf5203d98 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Mon, 22 Apr 2024 15:33:51 -0400 Subject: [PATCH 09/53] Remove check for new data sets daemon, slight refactor --- mobile_verifier/src/boosting_oracles/mod.rs | 276 ++++++++------------ mobile_verifier/src/cli/server.rs | 22 +- 2 files changed, 117 insertions(+), 181 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/mod.rs b/mobile_verifier/src/boosting_oracles/mod.rs index 50fd90e8c..dc06e7f23 100644 --- a/mobile_verifier/src/boosting_oracles/mod.rs +++ b/mobile_verifier/src/boosting_oracles/mod.rs @@ -1,21 +1,3 @@ -//! Module for handling the downloading of new data sets from S3. -//! -//! We've split this task into two parts, the CheckForNewDataSetsDaemon and the DataSetDownloaderDaemon. -//! As their names imply, the CheckForNewDataSetsDaemon is responsible for checking S3 for new files for -//! a given data set and inserting those new data sets into the `data_sets` table as [DataSetStatus::Pending]. -//! The DataSetDownloaderDaemon is responsible for checking the `data_sets` table and downloading any new -//! data sets and processing them. -//! -//! It seems unnecessary to split this task into two separate daemons, why not have one daemon that handles -//! both? Well, firstly, it's not two daemons, it's actually four (when all data sets are implemented), since -//! CheckForNewDataSetsDaemon only handles one type of data set. And yes, it is unnecessary, as it would be -//! possible in theory to put everything into one daemon that continuously polls S3 and checks if needs to -//! download new files and if some files need to be processed. -//! -//! But it would be extremely complicated. For one thing, the DataSetDownloaderDaemon needs to update the -//! data sets at a particular time, specified by the data set's timestamp. Keeping tracking of everything -//! in one place would make things more prone to bugs, and the implementation becomes a lot simpler. - pub mod assignment; pub mod footfall; pub mod urbanization; @@ -54,12 +36,12 @@ pub trait DataSet: HexAssignment + Send + Sync + 'static { pub struct DataSetDownloaderDaemon { pool: PgPool, - // Later: footfall and landtype data_set: Arc>, data_sets: HexBoostData, store: FileStore, oracle_boosting_sink: FileSinkClient, data_set_directory: PathBuf, + latest_file_date: Option>, } #[derive(FromRow)] @@ -109,22 +91,28 @@ where A: HexAssignment, B: HexAssignment, { - pub fn new( + pub async fn new( pool: PgPool, data_set: Arc>, data_sets: HexBoostData, store: FileStore, oracle_boosting_sink: FileSinkClient, data_set_directory: PathBuf, - ) -> Self { - Self { + ) -> anyhow::Result { + let latest_file_date: Option> = + sqlx::query_scalar("SELECT time_to_use FROM data_sets WHERE data_set = $1 ORDER BY time_to_use DESC LIMIT 1") + .bind(T::TYPE) + .fetch_optional(&pool) + .await?; + Ok(Self { pool, data_set, data_sets, store, oracle_boosting_sink, data_set_directory, - } + latest_file_date, + }) } fn get_data_set_path(&self, time_to_use: DateTime) -> PathBuf { @@ -138,33 +126,37 @@ where dir } - pub async fn run(self, shutdown: triggered::Listener) -> anyhow::Result<()> { - // Get the first data set: - if let Some(time_to_use) = - sqlx::query_scalar( - "SELECT time_to_use FROM data_sets WHERE status = 'processed' AND data_set = $1 ORDER BY time_to_use DESC LIMIT 1" + pub async fn check_for_available_data_sets(&mut self) -> anyhow::Result<()> { + let mut new_data_sets = self + .store + .list(T::TYPE.to_prefix(), self.latest_file_date, None); + while let Some(new_data_set) = new_data_sets.next().await.transpose()? { + tracing::info!( + "Found new data set: {}, {:#?}", + new_data_set.key, + new_data_set + ); + sqlx::query( + r#" + INSERT INTO data_sets (filename, data_set, time_to_use, status) + VALUES ($1, $2, $3, 'pending') + "#, ) + .bind(new_data_set.key) .bind(T::TYPE) - .fetch_optional(&self.pool) - .await? - { - let data_set_path = self.get_data_set_path(time_to_use); - tracing::info!("Found initial {} data set: {}", T::TYPE.to_prefix(), data_set_path.to_string_lossy()); - self.data_set.lock().await.update(&data_set_path, time_to_use)?; + .bind(new_data_set.timestamp) + .execute(&self.pool) + .await?; + self.latest_file_date = Some(new_data_set.timestamp); } + Ok(()) + } - // Another option I considered instead of polling was to use ENOTIFY, but that seemed - // not as good, as it would hog a pool connection. - // - // We set the poll duration to 30 minutes since that seems fine. - let poll_duration = Duration::minutes(30); - - loop { - // Find the latest urbanization file - tracing::info!("Checking for new data sets"); - let mut data_set = self.data_set.lock().await; - let curr_data_set = data_set.timestamp(); - let latest_data_set: Option = sqlx::query_as( + pub async fn process_data_sets(&self) -> anyhow::Result<()> { + tracing::info!("Checking for new data sets"); + let mut data_set = self.data_set.lock().await; + let curr_data_set = data_set.timestamp(); + let latest_data_set: Option = sqlx::query_as( "SELECT filename, time_to_use, status FROM data_sets WHERE status != 'processed' AND data_set = $1 AND COALESCE(time_to_use > $2, TRUE) AND time_to_use <= $3 ORDER BY time_to_use DESC LIMIT 1" ) .bind(T::TYPE) @@ -173,56 +165,64 @@ where .fetch_optional(&self.pool) .await?; - if let Some(latest_data_set) = latest_data_set { - let path = self.get_data_set_path(latest_data_set.time_to_use); - - // Download the file if it hasn't been downloaded already: - if !latest_data_set.status.is_downloaded() { - tracing::info!("Downloading new data set: {}", path.to_string_lossy()); - // TODO: abstract this out to a function - let stream = self.store.get_raw(latest_data_set.filename.clone()).await?; - let mut bytes = tokio_util::codec::FramedRead::new( - async_compression::tokio::bufread::GzipDecoder::new( - tokio_util::io::StreamReader::new(stream), - ), - tokio_util::codec::BytesCodec::new(), - ); - let mut file = File::create(&path).await?; - while let Some(bytes) = bytes.next().await.transpose()? { - file.write_all(&bytes).await?; - } - // Set the status to be downloaded - sqlx::query("UPDATE data_sets SET status = 'downloaded' WHERE filename = $1") - .bind(&latest_data_set.filename) - .execute(&self.pool) - .await?; - } + if let Some(latest_data_set) = latest_data_set { + let path = self.get_data_set_path(latest_data_set.time_to_use); - // Now that we've downloaded the file, load it into the data set - data_set.update(Path::new(&path), latest_data_set.time_to_use)?; + // Download the file if it hasn't been downloaded already: + if !latest_data_set.status.is_downloaded() { + download_data_set(&self.store, &latest_data_set.filename, &path).await?; + // Set the status to be downloaded + sqlx::query("UPDATE data_sets SET status = 'downloaded' WHERE filename = $1") + .bind(&latest_data_set.filename) + .execute(&self.pool) + .await?; + } + + // Now that we've downloaded the file, load it into the data set + data_set.update(Path::new(&path), latest_data_set.time_to_use)?; - drop(data_set); + drop(data_set); - // Update the hexes - let boosting_reports = set_oracle_boosting_assignments( - UnassignedHex::fetch_all(&self.pool), - &self.data_sets, - &self.pool, - ) + // Update the hexes + let boosting_reports = set_oracle_boosting_assignments( + UnassignedHex::fetch_all(&self.pool), + &self.data_sets, + &self.pool, + ) + .await?; + + sqlx::query("UPDATE data_sets SET status = 'processed' WHERE filename = $1") + .bind(latest_data_set.filename) + .execute(&self.pool) .await?; - sqlx::query("UPDATE data_sets SET status = 'processed' WHERE filename = $1") - .bind(latest_data_set.filename) - .execute(&self.pool) - .await?; + self.oracle_boosting_sink + .write_all(boosting_reports) + .await?; + self.oracle_boosting_sink.commit().await?; + tracing::info!("Data set download complete"); + } + Ok(()) + } - self.oracle_boosting_sink - .write_all(boosting_reports) - .await?; - self.oracle_boosting_sink.commit().await?; - tracing::info!("Data set download complete"); - } + pub async fn run(mut self, shutdown: triggered::Listener) -> anyhow::Result<()> { + // Get the first data set: + if let Some(time_to_use) = + sqlx::query_scalar( + "SELECT time_to_use FROM data_sets WHERE status = 'processed' AND data_set = $1 ORDER BY time_to_use DESC LIMIT 1" + ) + .bind(T::TYPE) + .fetch_optional(&self.pool) + .await? + { + let data_set_path = self.get_data_set_path(time_to_use); + tracing::info!("Found initial {} data set: {}", T::TYPE.to_prefix(), data_set_path.to_string_lossy()); + self.data_set.lock().await.update(&data_set_path, time_to_use)?; + } + + let poll_duration = Duration::minutes(5); + loop { // We don't want to shut down in the middle of downloading a data set, so we hold off until // we are sleeping #[rustfmt::skip] @@ -233,6 +233,8 @@ where break; } _ = tokio::time::sleep(poll_duration.to_std()?) => { + self.check_for_available_data_sets().await?; + self.process_data_sets().await?; continue; } } @@ -242,6 +244,27 @@ where } } +async fn download_data_set( + store: &FileStore, + in_file_name: &str, + out_path: &Path, +) -> anyhow::Result<()> { + tracing::info!("Downloading new data set: {}", out_path.to_string_lossy()); + // TODO: abstract this out to a function + let stream = store.get_raw(in_file_name).await?; + let mut bytes = tokio_util::codec::FramedRead::new( + async_compression::tokio::bufread::GzipDecoder::new(tokio_util::io::StreamReader::new( + stream, + )), + tokio_util::codec::BytesCodec::new(), + ); + let mut file = File::create(&out_path).await?; + while let Some(bytes) = bytes.next().await.transpose()? { + file.write_all(&bytes).await?; + } + Ok(()) +} + #[derive(Copy, Clone, sqlx::Type)] #[sqlx(type_name = "data_set_type")] #[sqlx(rename_all = "lowercase")] @@ -261,81 +284,6 @@ impl DataSetType { } } -// Better name welcome -pub struct CheckForNewDataSetDaemon { - pool: PgPool, - store: FileStore, - data_set: DataSetType, -} - -impl ManagedTask for CheckForNewDataSetDaemon { - fn start_task( - self: Box, - shutdown: triggered::Listener, - ) -> futures::prelude::future::LocalBoxFuture<'static, anyhow::Result<()>> { - let handle = tokio::spawn(self.run(shutdown)); - Box::pin( - handle - .map_err(anyhow::Error::from) - .and_then(|result| async move { result.map_err(anyhow::Error::from) }), - ) - } -} - -impl CheckForNewDataSetDaemon { - pub fn new(pool: PgPool, store: FileStore, data_set: DataSetType) -> Self { - Self { - pool, - data_set, - store, - } - } - - pub async fn run(self, shutdown: triggered::Listener) -> anyhow::Result<()> { - // We should check for new data sets more often than we download them. 15 minutes seems fine. - let poll_duration = Duration::minutes(15); - - let prefix = self.data_set.to_prefix(); - let mut latest_file_date: Option> = - sqlx::query_scalar("SELECT time_to_use FROM data_sets WHERE data_set = $1 ORDER BY time_to_use DESC LIMIT 1") - .bind(self.data_set) - .fetch_optional(&self.pool) - .await?; - - loop { - #[rustfmt::skip] - tokio::select! { - biased; - _ = shutdown.clone() => { - tracing::info!("CheckForNewDataSetDaemon shutting down"); - break; - } - _ = tokio::time::sleep(poll_duration.to_std()?) => { - // tracing::info!("Checking file store for new data sets"); - let mut new_data_sets = self.store.list(prefix, latest_file_date, None); - while let Some(new_data_set) = new_data_sets.next().await.transpose()? { - tracing::info!("Found new data set: {}, {:#?}", new_data_set.key, new_data_set); - sqlx::query( - r#" - INSERT INTO data_sets (filename, data_set, time_to_use, status) - VALUES ($1, $2, $3, 'pending') - "#, - ) - .bind(new_data_set.key) - .bind(self.data_set) - .bind(new_data_set.timestamp) - .execute(&self.pool) - .await?; - latest_file_date = Some(new_data_set.timestamp); - } - } - } - } - - Ok(()) - } -} - /// Check if there are any pending or downloaded files prior to the given reward period pub async fn check_for_unprocessed_data_sets( pool: &PgPool, diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index 711f0395e..4edc4f35a 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -1,7 +1,6 @@ use crate::{ boosting_oracles::{ - footfall::Footfall, CheckForNewDataSetDaemon, DataSetDownloaderDaemon, HexBoostData, - Urbanization, + footfall::Footfall, urbanization::Urbanization, DataSetDownloaderDaemon, HexBoostData, }, coverage::CoverageDaemon, data_session::DataSessionIngestor, @@ -246,7 +245,8 @@ impl Cmd { data_sets_file_store.clone(), oracle_boosting_reports.clone(), settings.data_sets_directory.clone(), - ); + ) + .await?; let footfall_data_set_downloader = DataSetDownloaderDaemon::new( pool.clone(), hex_boost_data.footfall.clone(), @@ -254,18 +254,8 @@ impl Cmd { data_sets_file_store.clone(), oracle_boosting_reports, settings.data_sets_directory.clone(), - ); - let check_for_urbanization_data_sets = CheckForNewDataSetDaemon::new( - pool.clone(), - data_sets_file_store.clone(), - crate::boosting_oracles::DataSetType::Urbanization, - ); - let check_for_footfall_data_sets = CheckForNewDataSetDaemon::new( - pool.clone(), - data_sets_file_store, - crate::boosting_oracles::DataSetType::Footfall, - ); - + ) + .await?; // Mobile rewards let reward_period_hours = settings.rewards; let (mobile_rewards, mobile_rewards_server) = file_sink::FileSinkBuilder::new( @@ -433,8 +423,6 @@ impl Cmd { .add_task(data_session_ingestor) .add_task(urbanization_data_set_downloader) .add_task(footfall_data_set_downloader) - .add_task(check_for_urbanization_data_sets) - .add_task(check_for_footfall_data_sets) .start() .await } From 3426c0df7090ec7540002d40cd56e2149a8b2663 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Tue, 23 Apr 2024 15:51:35 -0400 Subject: [PATCH 10/53] Shutdown downloaders instantly --- mobile_verifier/src/boosting_oracles/mod.rs | 32 ++++++--------- mobile_verifier/src/coverage.rs | 45 --------------------- 2 files changed, 13 insertions(+), 64 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/mod.rs b/mobile_verifier/src/boosting_oracles/mod.rs index dc06e7f23..11e1dbe31 100644 --- a/mobile_verifier/src/boosting_oracles/mod.rs +++ b/mobile_verifier/src/boosting_oracles/mod.rs @@ -76,7 +76,14 @@ where self: Box, shutdown: triggered::Listener, ) -> futures::prelude::future::LocalBoxFuture<'static, anyhow::Result<()>> { - let handle = tokio::spawn(self.run(shutdown)); + let handle = tokio::spawn(async move { + #[rustfmt::skip] + tokio::select! { + biased; + _ = shutdown.clone() => Ok(()), + result = self.run() => result, + } + }); Box::pin( handle .map_err(anyhow::Error::from) @@ -181,6 +188,7 @@ where // Now that we've downloaded the file, load it into the data set data_set.update(Path::new(&path), latest_data_set.time_to_use)?; + // Release the lock as it is no longer needed drop(data_set); // Update the hexes @@ -205,7 +213,7 @@ where Ok(()) } - pub async fn run(mut self, shutdown: triggered::Listener) -> anyhow::Result<()> { + pub async fn run(mut self) -> anyhow::Result<()> { // Get the first data set: if let Some(time_to_use) = sqlx::query_scalar( @@ -223,24 +231,10 @@ where let poll_duration = Duration::minutes(5); loop { - // We don't want to shut down in the middle of downloading a data set, so we hold off until - // we are sleeping - #[rustfmt::skip] - tokio::select! { - biased; - _ = shutdown.clone() => { - tracing::info!("DataSetDownloaderDaemon shutting down"); - break; - } - _ = tokio::time::sleep(poll_duration.to_std()?) => { - self.check_for_available_data_sets().await?; - self.process_data_sets().await?; - continue; - } - } + self.check_for_available_data_sets().await?; + self.process_data_sets().await?; + tokio::time::sleep(poll_duration.to_std()?).await; } - - Ok(()) } } diff --git a/mobile_verifier/src/coverage.rs b/mobile_verifier/src/coverage.rs index a5d348b15..5fb4957a8 100644 --- a/mobile_verifier/src/coverage.rs +++ b/mobile_verifier/src/coverage.rs @@ -157,51 +157,6 @@ where } } -/* -#[derive(FromRow)] -pub struct UnassignedHex { - uuid: Uuid, - #[sqlx(try_from = "i64")] - hex: u64, - signal_level: SignalLevel, - signal_power: i32, -} - -impl UnassignedHex { - pub fn fetch(pool: &PgPool) -> impl Stream> + '_ { - sqlx::query_as( - "SELECT uuid, hex, signal_level, signal_power FROM hexes WHERE urbanized IS NULL OR footfall IS NULL", - ) - .fetch(pool) - } - - fn to_location_string(&self) -> String { - format!("{:x}", self.hex) - } -} - -pub async fn set_oracle_boosting_assignments( - unassigned_urbinization_hexes: impl Stream>, - hex_boost_data: &HexBoostData, - pool: &PgPool, -) -> anyhow::Result> { - let now = Utc::now(); - - let boost_results = - initialize_unassigned_hexes(unassigned_urbinization_hexes, hex_boost_data, pool).await?; - - Ok(boost_results - .into_iter() - .map( - move |(coverage_object, assignments)| proto::OracleBoostingReportV1 { - coverage_object: Vec::from(coverage_object.into_bytes()), - assignments, - timestamp: now.encode_timestamp(), - }, - )) -} -*/ - impl ManagedTask for CoverageDaemon where Urban: DataSet, From b0f602a4ed1641460caa87adf9164b634e2008d4 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Wed, 24 Apr 2024 10:43:27 -0400 Subject: [PATCH 11/53] Refactor code a bit --- mobile_verifier/src/boosting_oracles/mod.rs | 189 +++++++++++++------- mobile_verifier/src/rewarder.rs | 2 +- 2 files changed, 127 insertions(+), 64 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/mod.rs b/mobile_verifier/src/boosting_oracles/mod.rs index 11e1dbe31..e6a37259e 100644 --- a/mobile_verifier/src/boosting_oracles/mod.rs +++ b/mobile_verifier/src/boosting_oracles/mod.rs @@ -106,11 +106,7 @@ where oracle_boosting_sink: FileSinkClient, data_set_directory: PathBuf, ) -> anyhow::Result { - let latest_file_date: Option> = - sqlx::query_scalar("SELECT time_to_use FROM data_sets WHERE data_set = $1 ORDER BY time_to_use DESC LIMIT 1") - .bind(T::TYPE) - .fetch_optional(&pool) - .await?; + let latest_file_date = db::fetch_latest_file_date(&pool, T::TYPE).await?; Ok(Self { pool, data_set, @@ -143,16 +139,12 @@ where new_data_set.key, new_data_set ); - sqlx::query( - r#" - INSERT INTO data_sets (filename, data_set, time_to_use, status) - VALUES ($1, $2, $3, 'pending') - "#, + db::insert_new_data_set( + &self.pool, + &new_data_set.key, + T::TYPE, + new_data_set.timestamp, ) - .bind(new_data_set.key) - .bind(T::TYPE) - .bind(new_data_set.timestamp) - .execute(&self.pool) .await?; self.latest_file_date = Some(new_data_set.timestamp); } @@ -162,54 +154,63 @@ where pub async fn process_data_sets(&self) -> anyhow::Result<()> { tracing::info!("Checking for new data sets"); let mut data_set = self.data_set.lock().await; - let curr_data_set = data_set.timestamp(); - let latest_data_set: Option = sqlx::query_as( - "SELECT filename, time_to_use, status FROM data_sets WHERE status != 'processed' AND data_set = $1 AND COALESCE(time_to_use > $2, TRUE) AND time_to_use <= $3 ORDER BY time_to_use DESC LIMIT 1" - ) - .bind(T::TYPE) - .bind(curr_data_set) - .bind(Utc::now()) - .fetch_optional(&self.pool) + let latest_unprocessed_data_set = + db::fetch_latest_unprocessed_data_set(&self.pool, T::TYPE, data_set.timestamp()) .await?; - if let Some(latest_data_set) = latest_data_set { - let path = self.get_data_set_path(latest_data_set.time_to_use); - - // Download the file if it hasn't been downloaded already: - if !latest_data_set.status.is_downloaded() { - download_data_set(&self.store, &latest_data_set.filename, &path).await?; - // Set the status to be downloaded - sqlx::query("UPDATE data_sets SET status = 'downloaded' WHERE filename = $1") - .bind(&latest_data_set.filename) - .execute(&self.pool) - .await?; - } + let Some(latest_unprocessed_data_set) = latest_unprocessed_data_set else { + return Ok(()); + }; - // Now that we've downloaded the file, load it into the data set - data_set.update(Path::new(&path), latest_data_set.time_to_use)?; + // If there is an unprocessed data set, download it (if we need to) and process it. - // Release the lock as it is no longer needed - drop(data_set); + let path = self.get_data_set_path(latest_unprocessed_data_set.time_to_use); - // Update the hexes - let boosting_reports = set_oracle_boosting_assignments( - UnassignedHex::fetch_all(&self.pool), - &self.data_sets, + // Download the file if it hasn't been downloaded already: + if !latest_unprocessed_data_set.status.is_downloaded() { + download_data_set(&self.store, &latest_unprocessed_data_set.filename, &path).await?; + db::set_data_set_status( &self.pool, + &latest_unprocessed_data_set.filename, + DataSetStatus::Downloaded, ) .await?; + tracing::info!( + data_set = latest_unprocessed_data_set.filename, + "Data set download complete" + ); + } - sqlx::query("UPDATE data_sets SET status = 'processed' WHERE filename = $1") - .bind(latest_data_set.filename) - .execute(&self.pool) - .await?; + // Now that we've downloaded the file, load it into the data set + data_set.update(Path::new(&path), latest_unprocessed_data_set.time_to_use)?; + + // Release the lock as it is no longer needed + drop(data_set); + + // Update the hexes + let boosting_reports = set_oracle_boosting_assignments( + UnassignedHex::fetch_all(&self.pool), + &self.data_sets, + &self.pool, + ) + .await?; + + db::set_data_set_status( + &self.pool, + &latest_unprocessed_data_set.filename, + DataSetStatus::Processed, + ) + .await?; + + self.oracle_boosting_sink + .write_all(boosting_reports) + .await?; + self.oracle_boosting_sink.commit().await?; + tracing::info!( + data_set = latest_unprocessed_data_set.filename, + "Data set processing complete" + ); - self.oracle_boosting_sink - .write_all(boosting_reports) - .await?; - self.oracle_boosting_sink.commit().await?; - tracing::info!("Data set download complete"); - } Ok(()) } @@ -278,17 +279,79 @@ impl DataSetType { } } -/// Check if there are any pending or downloaded files prior to the given reward period -pub async fn check_for_unprocessed_data_sets( - pool: &PgPool, - period_end: DateTime, -) -> sqlx::Result { - sqlx::query_scalar( - "SELECT COUNT(*) > 0 FROM data_sets WHERE time_to_use <= $1 AND status != 'processed'", - ) - .bind(period_end) - .fetch_one(pool) - .await +pub mod db { + use super::*; + + pub async fn fetch_latest_file_date( + pool: &PgPool, + data_set_type: DataSetType, + ) -> sqlx::Result>> { + sqlx::query_scalar("SELECT time_to_use FROM data_sets WHERE data_set = $1 ORDER BY time_to_use DESC LIMIT 1") + .bind(data_set_type) + .fetch_optional(pool) + .await + } + + pub async fn insert_new_data_set( + pool: &PgPool, + filename: &str, + data_set_type: DataSetType, + time_to_use: DateTime, + ) -> sqlx::Result<()> { + sqlx::query( + r#" + INSERT INTO data_sets (filename, data_set, time_to_use, status) + VALUES ($1, $2, $3, 'pending') + "#, + ) + .bind(filename) + .bind(data_set_type) + .bind(time_to_use) + .execute(pool) + .await?; + Ok(()) + } + + pub async fn fetch_latest_unprocessed_data_set( + pool: &PgPool, + data_set_type: DataSetType, + since: Option>, + ) -> sqlx::Result> { + sqlx::query_as( + "SELECT filename, time_to_use, status FROM data_sets WHERE status != 'processed' AND data_set = $1 AND COALESCE(time_to_use > $2, TRUE) AND time_to_use <= $3 ORDER BY time_to_use DESC LIMIT 1" + ) + .bind(data_set_type) + .bind(since) + .bind(Utc::now()) + .fetch_optional(pool) + .await + } + + pub async fn set_data_set_status( + pool: &PgPool, + filename: &str, + status: DataSetStatus, + ) -> sqlx::Result<()> { + sqlx::query("UPDATE data_sets SET status = $1 WHERE filename = $2") + .bind(status) + .bind(filename) + .execute(pool) + .await?; + Ok(()) + } + + /// Check if there are any pending or downloaded files prior to the given reward period + pub async fn check_for_unprocessed_data_sets( + pool: &PgPool, + period_end: DateTime, + ) -> sqlx::Result { + sqlx::query_scalar( + "SELECT COUNT(*) > 0 FROM data_sets WHERE time_to_use <= $1 AND status != 'processed'", + ) + .bind(period_end) + .fetch_one(pool) + .await + } } pub trait HexAssignment: Send + Sync + 'static { diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index d0a7e14b4..ed5a2f76a 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -1,5 +1,5 @@ use crate::{ - boosting_oracles::check_for_unprocessed_data_sets, + boosting_oracles::db::check_for_unprocessed_data_sets, coverage, data_session, heartbeats::{self, HeartbeatReward}, radio_threshold, From 065e6cc82c341635e00eac270972e12686b7c5cf Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Wed, 24 Apr 2024 10:46:52 -0400 Subject: [PATCH 12/53] Move one more fn to db mod --- mobile_verifier/src/boosting_oracles/mod.rs | 30 +++++++++++++++------ 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/mod.rs b/mobile_verifier/src/boosting_oracles/mod.rs index e6a37259e..6c6b9dc55 100644 --- a/mobile_verifier/src/boosting_oracles/mod.rs +++ b/mobile_verifier/src/boosting_oracles/mod.rs @@ -217,16 +217,18 @@ where pub async fn run(mut self) -> anyhow::Result<()> { // Get the first data set: if let Some(time_to_use) = - sqlx::query_scalar( - "SELECT time_to_use FROM data_sets WHERE status = 'processed' AND data_set = $1 ORDER BY time_to_use DESC LIMIT 1" - ) - .bind(T::TYPE) - .fetch_optional(&self.pool) - .await? + db::fetch_time_of_latest_processed_data_set(&self.pool, T::TYPE).await? { let data_set_path = self.get_data_set_path(time_to_use); - tracing::info!("Found initial {} data set: {}", T::TYPE.to_prefix(), data_set_path.to_string_lossy()); - self.data_set.lock().await.update(&data_set_path, time_to_use)?; + tracing::info!( + "Found initial {} data set: {}", + T::TYPE.to_prefix(), + data_set_path.to_string_lossy() + ); + self.data_set + .lock() + .await + .update(&data_set_path, time_to_use)?; } let poll_duration = Duration::minutes(5); @@ -340,6 +342,18 @@ pub mod db { Ok(()) } + pub async fn fetch_time_of_latest_processed_data_set( + pool: &PgPool, + data_set_type: DataSetType, + ) -> sqlx::Result>> { + sqlx::query_scalar( + "SELECT time_to_use FROM data_sets WHERE status = 'processed' AND data_set = $1 ORDER BY time_to_use DESC LIMIT 1" + ) + .bind(data_set_type) + .fetch_optional(pool) + .await + } + /// Check if there are any pending or downloaded files prior to the given reward period pub async fn check_for_unprocessed_data_sets( pool: &PgPool, From ec569fe54d7dd581842ffaa37823675e140416c4 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Wed, 24 Apr 2024 16:34:03 -0400 Subject: [PATCH 13/53] Commit missed file --- .../src/boosting_oracles/landtype.rs | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 mobile_verifier/src/boosting_oracles/landtype.rs diff --git a/mobile_verifier/src/boosting_oracles/landtype.rs b/mobile_verifier/src/boosting_oracles/landtype.rs new file mode 100644 index 000000000..0c862163c --- /dev/null +++ b/mobile_verifier/src/boosting_oracles/landtype.rs @@ -0,0 +1,154 @@ +use std::path::Path; + +use chrono::{DateTime, Utc}; +use hextree::disktree::DiskTreeMap; + +use super::{Assignment, DataSet, DataSetType, DiskTreeLike, HexAssignment}; + +pub struct Landtype { + landtype: Option, + timestamp: Option>, +} + +impl Landtype { + pub fn new() -> Self { + Self { + landtype: None, + timestamp: None, + } + } + + pub fn new_mock(landtype: L) -> Self { + Self { + landtype: Some(landtype), + timestamp: None, + } + } +} + +impl Default for Landtype { + fn default() -> Self { + Self::new() + } +} + +impl DataSet for Landtype { + const TYPE: DataSetType = DataSetType::Landtype; + + fn timestamp(&self) -> Option> { + self.timestamp + } + + fn update(&mut self, path: &Path, time_to_use: DateTime) -> anyhow::Result<()> { + self.landtype = Some(DiskTreeMap::open(path)?); + self.timestamp = Some(time_to_use); + Ok(()) + } + + fn is_ready(&self) -> bool { + self.landtype.is_some() + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LandtypeValue { + Tree = 10, + Shrub = 20, + Grass = 30, + Crop = 40, + Built = 50, + Bare = 60, + Frozen = 70, + Water = 80, + Wet = 90, + Mangrove = 95, + Moss = 100, +} + +impl std::fmt::Display for LandtypeValue { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.to_str()) + } +} + +impl LandtypeValue { + pub(crate) fn to_str(self) -> &'static str { + match self { + LandtypeValue::Tree => "TreeCover", + LandtypeValue::Shrub => "Shrubland", + LandtypeValue::Grass => "Grassland", + LandtypeValue::Crop => "Cropland", + LandtypeValue::Built => "BuiltUp", + LandtypeValue::Bare => "BareOrSparseVeg", + LandtypeValue::Frozen => "SnowAndIce", + LandtypeValue::Water => "Water", + LandtypeValue::Wet => "HerbaceousWetland", + LandtypeValue::Mangrove => "Mangroves", + LandtypeValue::Moss => "MossAndLichen", + } + } +} + +impl TryFrom for LandtypeValue { + type Error = anyhow::Error; + fn try_from(other: u8) -> anyhow::Result { + let val = match other { + 10 => LandtypeValue::Tree, + 20 => LandtypeValue::Shrub, + 30 => LandtypeValue::Grass, + 40 => LandtypeValue::Crop, + 50 => LandtypeValue::Built, + 60 => LandtypeValue::Bare, + 70 => LandtypeValue::Frozen, + 80 => LandtypeValue::Water, + 90 => LandtypeValue::Wet, + 95 => LandtypeValue::Mangrove, + 100 => LandtypeValue::Moss, + other => anyhow::bail!("unexpected landtype disktree value: {other:?}"), + }; + Ok(val) + } +} + +impl From for Assignment { + fn from(value: LandtypeValue) -> Self { + match value { + LandtypeValue::Built => Assignment::A, + // + LandtypeValue::Tree => Assignment::B, + LandtypeValue::Shrub => Assignment::B, + LandtypeValue::Grass => Assignment::B, + // + LandtypeValue::Bare => Assignment::C, + LandtypeValue::Crop => Assignment::C, + LandtypeValue::Frozen => Assignment::C, + LandtypeValue::Water => Assignment::C, + LandtypeValue::Wet => Assignment::C, + LandtypeValue::Mangrove => Assignment::C, + LandtypeValue::Moss => Assignment::C, + } + } +} + +impl HexAssignment for Landtype +where + Land: DiskTreeLike + Send + Sync + 'static, +{ + fn assignment(&self, cell: hextree::Cell) -> anyhow::Result { + let Some(ref landtype) = self.landtype else { + anyhow::bail!("No landtype data set has been loaded"); + }; + + let Some((_, vals)) = landtype.get(cell)? else { + return Ok(Assignment::C); + }; + + anyhow::ensure!( + vals.len() == 1, + "unexpected landtype disktree data: {cell:?} {vals:?}" + ); + + let cover = LandtypeValue::try_from(vals[0])?; + Ok(cover.into()) + } +} From 789c93fb81c802dee7702d22a2700ebfd5fb6458 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Fri, 26 Apr 2024 13:06:12 -0400 Subject: [PATCH 14/53] Move data set downloading to separate module --- .../src/boosting_oracles/data_sets.rs | 483 ++++++++++++++++++ mobile_verifier/src/boosting_oracles/mod.rs | 482 +---------------- 2 files changed, 488 insertions(+), 477 deletions(-) create mode 100644 mobile_verifier/src/boosting_oracles/data_sets.rs diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs new file mode 100644 index 000000000..6e22a3d0a --- /dev/null +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -0,0 +1,483 @@ +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + pin::pin, + sync::Arc, +}; + +use chrono::{DateTime, Duration, Utc}; +use file_store::{file_sink::FileSinkClient, traits::TimestampEncode, FileStore}; +use futures_util::{Stream, StreamExt, TryFutureExt, TryStreamExt}; +use helium_proto::services::poc_mobile as proto; +use rust_decimal::prelude::ToPrimitive; +use rust_decimal_macros::dec; +use sqlx::{FromRow, PgPool, QueryBuilder}; +use task_manager::ManagedTask; +use tokio::{fs::File, io::AsyncWriteExt, sync::Mutex}; + +use crate::{boosting_oracles::assignment::HexAssignments, coverage::SignalLevel}; + +use super::{HexAssignment, HexBoostData}; + +pub trait DataSet: HexAssignment + Send + Sync + 'static { + const TYPE: DataSetType; + + fn timestamp(&self) -> Option>; + + fn update(&mut self, path: &Path, time_to_use: DateTime) -> anyhow::Result<()>; + + fn is_ready(&self) -> bool; +} + +pub struct DataSetDownloaderDaemon { + pool: PgPool, + data_set: Arc>, + data_sets: HexBoostData, + store: FileStore, + oracle_boosting_sink: FileSinkClient, + data_set_directory: PathBuf, + latest_file_date: Option>, +} + +#[derive(FromRow)] +pub struct NewDataSet { + filename: String, + time_to_use: DateTime, + status: DataSetStatus, +} + +#[derive(Copy, Clone, sqlx::Type)] +#[sqlx(type_name = "data_set_status")] +#[sqlx(rename_all = "lowercase")] +pub enum DataSetStatus { + Pending, + Downloaded, + Processed, +} + +impl DataSetStatus { + pub fn is_downloaded(&self) -> bool { + matches!(self, Self::Downloaded) + } +} + +impl ManagedTask for DataSetDownloaderDaemon +where + T: DataSet, + A: HexAssignment, + B: HexAssignment, + C: HexAssignment, +{ + fn start_task( + self: Box, + shutdown: triggered::Listener, + ) -> futures::prelude::future::LocalBoxFuture<'static, anyhow::Result<()>> { + let handle = tokio::spawn(async move { + #[rustfmt::skip] + tokio::select! { + biased; + _ = shutdown.clone() => Ok(()), + result = self.run() => result, + } + }); + Box::pin( + handle + .map_err(anyhow::Error::from) + .and_then(|result| async move { result.map_err(anyhow::Error::from) }), + ) + } +} + +impl DataSetDownloaderDaemon +where + T: DataSet, + A: HexAssignment, + B: HexAssignment, + C: HexAssignment, +{ + pub async fn new( + pool: PgPool, + data_set: Arc>, + data_sets: HexBoostData, + store: FileStore, + oracle_boosting_sink: FileSinkClient, + data_set_directory: PathBuf, + ) -> anyhow::Result { + let latest_file_date = db::fetch_latest_file_date(&pool, T::TYPE).await?; + Ok(Self { + pool, + data_set, + data_sets, + store, + oracle_boosting_sink, + data_set_directory, + latest_file_date, + }) + } + + fn get_data_set_path(&self, time_to_use: DateTime) -> PathBuf { + let path = PathBuf::from(format!( + "{}.{}.res10.h3tree", + T::TYPE.to_prefix(), + time_to_use.timestamp() + )); + let mut dir = self.data_set_directory.clone(); + dir.push(path); + dir + } + + pub async fn check_for_available_data_sets(&mut self) -> anyhow::Result<()> { + let mut new_data_sets = self + .store + .list(T::TYPE.to_prefix(), self.latest_file_date, None); + while let Some(new_data_set) = new_data_sets.next().await.transpose()? { + tracing::info!( + "Found new data set: {}, {:#?}", + new_data_set.key, + new_data_set + ); + db::insert_new_data_set( + &self.pool, + &new_data_set.key, + T::TYPE, + new_data_set.timestamp, + ) + .await?; + self.latest_file_date = Some(new_data_set.timestamp); + } + Ok(()) + } + + pub async fn process_data_sets(&self) -> anyhow::Result<()> { + tracing::info!("Checking for new data sets"); + let mut data_set = self.data_set.lock().await; + let latest_unprocessed_data_set = + db::fetch_latest_unprocessed_data_set(&self.pool, T::TYPE, data_set.timestamp()) + .await?; + + let Some(latest_unprocessed_data_set) = latest_unprocessed_data_set else { + return Ok(()); + }; + + // If there is an unprocessed data set, download it (if we need to) and process it. + + let path = self.get_data_set_path(latest_unprocessed_data_set.time_to_use); + + // Download the file if it hasn't been downloaded already: + if !latest_unprocessed_data_set.status.is_downloaded() { + download_data_set(&self.store, &latest_unprocessed_data_set.filename, &path).await?; + db::set_data_set_status( + &self.pool, + &latest_unprocessed_data_set.filename, + DataSetStatus::Downloaded, + ) + .await?; + tracing::info!( + data_set = latest_unprocessed_data_set.filename, + "Data set download complete" + ); + } + + // Now that we've downloaded the file, load it into the data set + data_set.update(Path::new(&path), latest_unprocessed_data_set.time_to_use)?; + + // Release the lock as it is no longer needed + drop(data_set); + + // Update the hexes + let boosting_reports = set_oracle_boosting_assignments( + UnassignedHex::fetch_all(&self.pool), + &self.data_sets, + &self.pool, + ) + .await?; + + db::set_data_set_status( + &self.pool, + &latest_unprocessed_data_set.filename, + DataSetStatus::Processed, + ) + .await?; + + self.oracle_boosting_sink + .write_all(boosting_reports) + .await?; + self.oracle_boosting_sink.commit().await?; + tracing::info!( + data_set = latest_unprocessed_data_set.filename, + "Data set processing complete" + ); + + Ok(()) + } + + pub async fn run(mut self) -> anyhow::Result<()> { + // Get the first data set: + if let Some(time_to_use) = + db::fetch_time_of_latest_processed_data_set(&self.pool, T::TYPE).await? + { + let data_set_path = self.get_data_set_path(time_to_use); + tracing::info!( + "Found initial {} data set: {}", + T::TYPE.to_prefix(), + data_set_path.to_string_lossy() + ); + self.data_set + .lock() + .await + .update(&data_set_path, time_to_use)?; + } + + let poll_duration = Duration::minutes(5); + + loop { + self.check_for_available_data_sets().await?; + self.process_data_sets().await?; + tokio::time::sleep(poll_duration.to_std()?).await; + } + } +} + +async fn download_data_set( + store: &FileStore, + in_file_name: &str, + out_path: &Path, +) -> anyhow::Result<()> { + tracing::info!("Downloading new data set: {}", out_path.to_string_lossy()); + // TODO: abstract this out to a function + let stream = store.get_raw(in_file_name).await?; + let mut bytes = tokio_util::codec::FramedRead::new( + async_compression::tokio::bufread::GzipDecoder::new(tokio_util::io::StreamReader::new( + stream, + )), + tokio_util::codec::BytesCodec::new(), + ); + let mut file = File::create(&out_path).await?; + while let Some(bytes) = bytes.next().await.transpose()? { + file.write_all(&bytes).await?; + } + Ok(()) +} + +#[derive(Copy, Clone, sqlx::Type)] +#[sqlx(type_name = "data_set_type")] +#[sqlx(rename_all = "lowercase")] +pub enum DataSetType { + Urbanization, + Footfall, + Landtype, +} + +impl DataSetType { + pub fn to_prefix(self) -> &'static str { + match self { + Self::Urbanization => "urbanization", + Self::Footfall => "footfall", + Self::Landtype => "landtype", + } + } +} + +pub mod db { + use super::*; + + pub async fn fetch_latest_file_date( + pool: &PgPool, + data_set_type: DataSetType, + ) -> sqlx::Result>> { + sqlx::query_scalar("SELECT time_to_use FROM data_sets WHERE data_set = $1 ORDER BY time_to_use DESC LIMIT 1") + .bind(data_set_type) + .fetch_optional(pool) + .await + } + + pub async fn insert_new_data_set( + pool: &PgPool, + filename: &str, + data_set_type: DataSetType, + time_to_use: DateTime, + ) -> sqlx::Result<()> { + sqlx::query( + r#" + INSERT INTO data_sets (filename, data_set, time_to_use, status) + VALUES ($1, $2, $3, 'pending') + "#, + ) + .bind(filename) + .bind(data_set_type) + .bind(time_to_use) + .execute(pool) + .await?; + Ok(()) + } + + pub async fn fetch_latest_unprocessed_data_set( + pool: &PgPool, + data_set_type: DataSetType, + since: Option>, + ) -> sqlx::Result> { + sqlx::query_as( + "SELECT filename, time_to_use, status FROM data_sets WHERE status != 'processed' AND data_set = $1 AND COALESCE(time_to_use > $2, TRUE) AND time_to_use <= $3 ORDER BY time_to_use DESC LIMIT 1" + ) + .bind(data_set_type) + .bind(since) + .bind(Utc::now()) + .fetch_optional(pool) + .await + } + + pub async fn set_data_set_status( + pool: &PgPool, + filename: &str, + status: DataSetStatus, + ) -> sqlx::Result<()> { + sqlx::query("UPDATE data_sets SET status = $1 WHERE filename = $2") + .bind(status) + .bind(filename) + .execute(pool) + .await?; + Ok(()) + } + + pub async fn fetch_time_of_latest_processed_data_set( + pool: &PgPool, + data_set_type: DataSetType, + ) -> sqlx::Result>> { + sqlx::query_scalar( + "SELECT time_to_use FROM data_sets WHERE status = 'processed' AND data_set = $1 ORDER BY time_to_use DESC LIMIT 1" + ) + .bind(data_set_type) + .fetch_optional(pool) + .await + } + + /// Check if there are any pending or downloaded files prior to the given reward period + pub async fn check_for_unprocessed_data_sets( + pool: &PgPool, + period_end: DateTime, + ) -> sqlx::Result { + sqlx::query_scalar( + "SELECT COUNT(*) > 0 FROM data_sets WHERE time_to_use <= $1 AND status != 'processed'", + ) + .bind(period_end) + .fetch_one(pool) + .await + } +} + +#[derive(FromRow)] +pub struct UnassignedHex { + uuid: uuid::Uuid, + #[sqlx(try_from = "i64")] + hex: u64, + signal_level: SignalLevel, + signal_power: i32, +} + +impl UnassignedHex { + pub fn fetch_all(pool: &PgPool) -> impl Stream> + '_ { + sqlx::query_as("SELECT uuid, hex, signal_level, signal_power FROM hexes").fetch(pool) + } + + pub fn fetch_unassigned(pool: &PgPool) -> impl Stream> + '_ { + sqlx::query_as( + "SELECT + uuid, hex, signal_level, signal_power + FROM + hexes + WHERE + urbanized IS NULL + OR footfall IS NULL + OR landtype IS NULL", + ) + .fetch(pool) + } +} + +pub async fn set_oracle_boosting_assignments<'a>( + unassigned_hexes: impl Stream>, + data_sets: &HexBoostData, + pool: &'a PgPool, +) -> anyhow::Result> { + const NUMBER_OF_FIELDS_IN_QUERY: u16 = 5; + const ASSIGNMENTS_MAX_BATCH_ENTRIES: usize = (u16::MAX / NUMBER_OF_FIELDS_IN_QUERY) as usize; + + let now = Utc::now(); + let mut boost_results = HashMap::>::new(); + let mut unassigned_hexes = pin!(unassigned_hexes.try_chunks(ASSIGNMENTS_MAX_BATCH_ENTRIES)); + + let urbanization = data_sets.urbanization.lock().await; + let footfall = data_sets.footfall.lock().await; + let landtype = data_sets.landtype.lock().await; + + while let Some(hexes) = unassigned_hexes.try_next().await? { + let hexes: anyhow::Result> = hexes + .into_iter() + .map(|hex| { + let cell = hextree::Cell::try_from(hex.hex)?; + let assignments = + HexAssignments::from_data_sets(cell, &*footfall, &*landtype, &*urbanization)?; + let location = format!("{:x}", hex.hex); + let assignment_multiplier = (assignments.boosting_multiplier() * dec!(1000)) + .to_u32() + .unwrap_or(0); + + boost_results.entry(hex.uuid).or_default().push( + proto::OracleBoostingHexAssignment { + location, + urbanized: assignments.urbanized.into(), + footfall: assignments.footfall.into(), + landtype: assignments.landtype.into(), + assignment_multiplier, + }, + ); + + Ok(( + hex, + assignments.footfall, + assignments.landtype, + assignments.urbanized, + )) + }) + .collect(); + + let mut transaction = pool.begin().await?; + sqlx::query("LOCK TABLE hexes") + .execute(&mut transaction) + .await?; + QueryBuilder::new( + "INSERT INTO hexes (uuid, hex, signal_level, signal_power, footfall, landtype, urbanized)", + ) + .push_values(hexes?, |mut b, (hex, footfall, landtype, urbanized)| { + b.push_bind(hex.uuid) + .push_bind(hex.hex as i64) + .push_bind(hex.signal_level) + .push_bind(hex.signal_power) + .push_bind(footfall) + .push_bind(landtype) + .push_bind(urbanized); + }) + .push( + r#" + ON CONFLICT (uuid, hex) DO UPDATE SET + footfall = EXCLUDED.footfall, + landtype = EXCLUDED.landtype, + urbanized = EXCLUDED.urbanized + "#, + ) + .build() + .execute(&mut transaction) + .await?; + transaction.commit().await?; + } + + Ok(boost_results + .into_iter() + .map( + move |(coverage_object, assignments)| proto::OracleBoostingReportV1 { + coverage_object: Vec::from(coverage_object.into_bytes()), + assignments, + timestamp: now.encode_timestamp(), + }, + )) +} diff --git a/mobile_verifier/src/boosting_oracles/mod.rs b/mobile_verifier/src/boosting_oracles/mod.rs index e3c00a3af..b34901918 100644 --- a/mobile_verifier/src/boosting_oracles/mod.rs +++ b/mobile_verifier/src/boosting_oracles/mod.rs @@ -1,374 +1,19 @@ pub mod assignment; +pub mod data_sets; pub mod footfall; pub mod landtype; pub mod urbanization; -use std::path::Path; -use std::pin::pin; +use std::collections::HashMap; use std::sync::Arc; -use std::{collections::HashMap, path::PathBuf}; use crate::boosting_oracles::assignment::HexAssignments; -use crate::coverage::SignalLevel; pub use assignment::Assignment; -use chrono::{DateTime, Duration, Utc}; -use file_store::{file_sink::FileSinkClient, traits::TimestampEncode, FileStore}; -use futures_util::{Stream, StreamExt, TryFutureExt, TryStreamExt}; -use helium_proto::services::poc_mobile as proto; +pub use data_sets::*; + use hextree::disktree::DiskTreeMap; -use rust_decimal::prelude::ToPrimitive; -use rust_decimal_macros::dec; -use sqlx::{FromRow, PgPool, QueryBuilder}; -use task_manager::ManagedTask; -use tokio::{fs::File, io::AsyncWriteExt, sync::Mutex}; +use tokio::sync::Mutex; pub use urbanization::Urbanization; -use uuid::Uuid; - -pub trait DataSet: HexAssignment + Send + Sync + 'static { - const TYPE: DataSetType; - - fn timestamp(&self) -> Option>; - - fn update(&mut self, path: &Path, time_to_use: DateTime) -> anyhow::Result<()>; - - fn is_ready(&self) -> bool; -} - -pub struct DataSetDownloaderDaemon { - pool: PgPool, - data_set: Arc>, - data_sets: HexBoostData, - store: FileStore, - oracle_boosting_sink: FileSinkClient, - data_set_directory: PathBuf, - latest_file_date: Option>, -} - -#[derive(FromRow)] -pub struct NewDataSet { - filename: String, - time_to_use: DateTime, - status: DataSetStatus, -} - -#[derive(Copy, Clone, sqlx::Type)] -#[sqlx(type_name = "data_set_status")] -#[sqlx(rename_all = "lowercase")] -pub enum DataSetStatus { - Pending, - Downloaded, - Processed, -} - -impl DataSetStatus { - pub fn is_downloaded(&self) -> bool { - matches!(self, Self::Downloaded) - } -} - -impl ManagedTask for DataSetDownloaderDaemon -where - T: DataSet, - A: HexAssignment, - B: HexAssignment, - C: HexAssignment, -{ - fn start_task( - self: Box, - shutdown: triggered::Listener, - ) -> futures::prelude::future::LocalBoxFuture<'static, anyhow::Result<()>> { - let handle = tokio::spawn(async move { - #[rustfmt::skip] - tokio::select! { - biased; - _ = shutdown.clone() => Ok(()), - result = self.run() => result, - } - }); - Box::pin( - handle - .map_err(anyhow::Error::from) - .and_then(|result| async move { result.map_err(anyhow::Error::from) }), - ) - } -} - -impl DataSetDownloaderDaemon -where - T: DataSet, - A: HexAssignment, - B: HexAssignment, - C: HexAssignment, -{ - pub async fn new( - pool: PgPool, - data_set: Arc>, - data_sets: HexBoostData, - store: FileStore, - oracle_boosting_sink: FileSinkClient, - data_set_directory: PathBuf, - ) -> anyhow::Result { - let latest_file_date = db::fetch_latest_file_date(&pool, T::TYPE).await?; - Ok(Self { - pool, - data_set, - data_sets, - store, - oracle_boosting_sink, - data_set_directory, - latest_file_date, - }) - } - - fn get_data_set_path(&self, time_to_use: DateTime) -> PathBuf { - let path = PathBuf::from(format!( - "{}.{}.res10.h3tree", - T::TYPE.to_prefix(), - time_to_use.timestamp() - )); - let mut dir = self.data_set_directory.clone(); - dir.push(path); - dir - } - - pub async fn check_for_available_data_sets(&mut self) -> anyhow::Result<()> { - let mut new_data_sets = self - .store - .list(T::TYPE.to_prefix(), self.latest_file_date, None); - while let Some(new_data_set) = new_data_sets.next().await.transpose()? { - tracing::info!( - "Found new data set: {}, {:#?}", - new_data_set.key, - new_data_set - ); - db::insert_new_data_set( - &self.pool, - &new_data_set.key, - T::TYPE, - new_data_set.timestamp, - ) - .await?; - self.latest_file_date = Some(new_data_set.timestamp); - } - Ok(()) - } - - pub async fn process_data_sets(&self) -> anyhow::Result<()> { - tracing::info!("Checking for new data sets"); - let mut data_set = self.data_set.lock().await; - let latest_unprocessed_data_set = - db::fetch_latest_unprocessed_data_set(&self.pool, T::TYPE, data_set.timestamp()) - .await?; - - let Some(latest_unprocessed_data_set) = latest_unprocessed_data_set else { - return Ok(()); - }; - - // If there is an unprocessed data set, download it (if we need to) and process it. - - let path = self.get_data_set_path(latest_unprocessed_data_set.time_to_use); - - // Download the file if it hasn't been downloaded already: - if !latest_unprocessed_data_set.status.is_downloaded() { - download_data_set(&self.store, &latest_unprocessed_data_set.filename, &path).await?; - db::set_data_set_status( - &self.pool, - &latest_unprocessed_data_set.filename, - DataSetStatus::Downloaded, - ) - .await?; - tracing::info!( - data_set = latest_unprocessed_data_set.filename, - "Data set download complete" - ); - } - - // Now that we've downloaded the file, load it into the data set - data_set.update(Path::new(&path), latest_unprocessed_data_set.time_to_use)?; - - // Release the lock as it is no longer needed - drop(data_set); - - // Update the hexes - let boosting_reports = set_oracle_boosting_assignments( - UnassignedHex::fetch_all(&self.pool), - &self.data_sets, - &self.pool, - ) - .await?; - - db::set_data_set_status( - &self.pool, - &latest_unprocessed_data_set.filename, - DataSetStatus::Processed, - ) - .await?; - - self.oracle_boosting_sink - .write_all(boosting_reports) - .await?; - self.oracle_boosting_sink.commit().await?; - tracing::info!( - data_set = latest_unprocessed_data_set.filename, - "Data set processing complete" - ); - - Ok(()) - } - - pub async fn run(mut self) -> anyhow::Result<()> { - // Get the first data set: - if let Some(time_to_use) = - db::fetch_time_of_latest_processed_data_set(&self.pool, T::TYPE).await? - { - let data_set_path = self.get_data_set_path(time_to_use); - tracing::info!( - "Found initial {} data set: {}", - T::TYPE.to_prefix(), - data_set_path.to_string_lossy() - ); - self.data_set - .lock() - .await - .update(&data_set_path, time_to_use)?; - } - - let poll_duration = Duration::minutes(5); - - loop { - self.check_for_available_data_sets().await?; - self.process_data_sets().await?; - tokio::time::sleep(poll_duration.to_std()?).await; - } - } -} - -async fn download_data_set( - store: &FileStore, - in_file_name: &str, - out_path: &Path, -) -> anyhow::Result<()> { - tracing::info!("Downloading new data set: {}", out_path.to_string_lossy()); - // TODO: abstract this out to a function - let stream = store.get_raw(in_file_name).await?; - let mut bytes = tokio_util::codec::FramedRead::new( - async_compression::tokio::bufread::GzipDecoder::new(tokio_util::io::StreamReader::new( - stream, - )), - tokio_util::codec::BytesCodec::new(), - ); - let mut file = File::create(&out_path).await?; - while let Some(bytes) = bytes.next().await.transpose()? { - file.write_all(&bytes).await?; - } - Ok(()) -} - -#[derive(Copy, Clone, sqlx::Type)] -#[sqlx(type_name = "data_set_type")] -#[sqlx(rename_all = "lowercase")] -pub enum DataSetType { - Urbanization, - Footfall, - Landtype, -} - -impl DataSetType { - pub fn to_prefix(self) -> &'static str { - match self { - Self::Urbanization => "urbanization", - Self::Footfall => "footfall", - Self::Landtype => "landtype", - } - } -} - -pub mod db { - use super::*; - - pub async fn fetch_latest_file_date( - pool: &PgPool, - data_set_type: DataSetType, - ) -> sqlx::Result>> { - sqlx::query_scalar("SELECT time_to_use FROM data_sets WHERE data_set = $1 ORDER BY time_to_use DESC LIMIT 1") - .bind(data_set_type) - .fetch_optional(pool) - .await - } - - pub async fn insert_new_data_set( - pool: &PgPool, - filename: &str, - data_set_type: DataSetType, - time_to_use: DateTime, - ) -> sqlx::Result<()> { - sqlx::query( - r#" - INSERT INTO data_sets (filename, data_set, time_to_use, status) - VALUES ($1, $2, $3, 'pending') - "#, - ) - .bind(filename) - .bind(data_set_type) - .bind(time_to_use) - .execute(pool) - .await?; - Ok(()) - } - - pub async fn fetch_latest_unprocessed_data_set( - pool: &PgPool, - data_set_type: DataSetType, - since: Option>, - ) -> sqlx::Result> { - sqlx::query_as( - "SELECT filename, time_to_use, status FROM data_sets WHERE status != 'processed' AND data_set = $1 AND COALESCE(time_to_use > $2, TRUE) AND time_to_use <= $3 ORDER BY time_to_use DESC LIMIT 1" - ) - .bind(data_set_type) - .bind(since) - .bind(Utc::now()) - .fetch_optional(pool) - .await - } - - pub async fn set_data_set_status( - pool: &PgPool, - filename: &str, - status: DataSetStatus, - ) -> sqlx::Result<()> { - sqlx::query("UPDATE data_sets SET status = $1 WHERE filename = $2") - .bind(status) - .bind(filename) - .execute(pool) - .await?; - Ok(()) - } - - pub async fn fetch_time_of_latest_processed_data_set( - pool: &PgPool, - data_set_type: DataSetType, - ) -> sqlx::Result>> { - sqlx::query_scalar( - "SELECT time_to_use FROM data_sets WHERE status = 'processed' AND data_set = $1 ORDER BY time_to_use DESC LIMIT 1" - ) - .bind(data_set_type) - .fetch_optional(pool) - .await - } - - /// Check if there are any pending or downloaded files prior to the given reward period - pub async fn check_for_unprocessed_data_sets( - pool: &PgPool, - period_end: DateTime, - ) -> sqlx::Result { - sqlx::query_scalar( - "SELECT COUNT(*) > 0 FROM data_sets WHERE time_to_use <= $1 AND status != 'processed'", - ) - .bind(period_end) - .fetch_one(pool) - .await - } -} pub trait HexAssignment: Send + Sync + 'static { fn assignment(&self, cell: hextree::Cell) -> anyhow::Result; @@ -463,123 +108,6 @@ impl DiskTreeLike for MockDiskTree { } } -#[derive(FromRow)] -pub struct UnassignedHex { - uuid: uuid::Uuid, - #[sqlx(try_from = "i64")] - hex: u64, - signal_level: SignalLevel, - signal_power: i32, -} - -impl UnassignedHex { - pub fn fetch_all(pool: &PgPool) -> impl Stream> + '_ { - sqlx::query_as("SELECT uuid, hex, signal_level, signal_power FROM hexes").fetch(pool) - } - - pub fn fetch_unassigned(pool: &PgPool) -> impl Stream> + '_ { - sqlx::query_as( - "SELECT - uuid, hex, signal_level, signal_power - FROM - hexes - WHERE - urbanized IS NULL - OR footfall IS NULL - OR landtype IS NULL", - ) - .fetch(pool) - } -} - -pub async fn set_oracle_boosting_assignments<'a>( - unassigned_hexes: impl Stream>, - data_sets: &HexBoostData, - pool: &'a PgPool, -) -> anyhow::Result> { - const NUMBER_OF_FIELDS_IN_QUERY: u16 = 5; - const ASSIGNMENTS_MAX_BATCH_ENTRIES: usize = (u16::MAX / NUMBER_OF_FIELDS_IN_QUERY) as usize; - - let now = Utc::now(); - let mut boost_results = HashMap::>::new(); - let mut unassigned_hexes = pin!(unassigned_hexes.try_chunks(ASSIGNMENTS_MAX_BATCH_ENTRIES)); - - let urbanization = data_sets.urbanization.lock().await; - let footfall = data_sets.footfall.lock().await; - let landtype = data_sets.landtype.lock().await; - - while let Some(hexes) = unassigned_hexes.try_next().await? { - let hexes: anyhow::Result> = hexes - .into_iter() - .map(|hex| { - let cell = hextree::Cell::try_from(hex.hex)?; - let assignments = - HexAssignments::from_data_sets(cell, &*footfall, &*landtype, &*urbanization)?; - let location = format!("{:x}", hex.hex); - let assignment_multiplier = (assignments.boosting_multiplier() * dec!(1000)) - .to_u32() - .unwrap_or(0); - - boost_results.entry(hex.uuid).or_default().push( - proto::OracleBoostingHexAssignment { - location, - urbanized: assignments.urbanized.into(), - footfall: assignments.footfall.into(), - landtype: assignments.landtype.into(), - assignment_multiplier, - }, - ); - - Ok(( - hex, - assignments.footfall, - assignments.landtype, - assignments.urbanized, - )) - }) - .collect(); - - let mut transaction = pool.begin().await?; - sqlx::query("LOCK TABLE hexes") - .execute(&mut transaction) - .await?; - QueryBuilder::new( - "INSERT INTO hexes (uuid, hex, signal_level, signal_power, footfall, landtype, urbanized)", - ) - .push_values(hexes?, |mut b, (hex, footfall, landtype, urbanized)| { - b.push_bind(hex.uuid) - .push_bind(hex.hex as i64) - .push_bind(hex.signal_level) - .push_bind(hex.signal_power) - .push_bind(footfall) - .push_bind(landtype) - .push_bind(urbanized); - }) - .push( - r#" - ON CONFLICT (uuid, hex) DO UPDATE SET - footfall = EXCLUDED.footfall, - landtype = EXCLUDED.landtype, - urbanized = EXCLUDED.urbanized - "#, - ) - .build() - .execute(&mut transaction) - .await?; - transaction.commit().await?; - } - - Ok(boost_results - .into_iter() - .map( - move |(coverage_object, assignments)| proto::OracleBoostingReportV1 { - coverage_object: Vec::from(coverage_object.into_bytes()), - assignments, - timestamp: now.encode_timestamp(), - }, - )) -} - #[cfg(test)] mod tests { From f3519c671cbd470730d8b85b9aee5c8a99852d6f Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Fri, 26 Apr 2024 13:12:16 -0400 Subject: [PATCH 15/53] Remove outdated comment --- mobile_verifier/src/boosting_oracles/data_sets.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index 6e22a3d0a..92a645108 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -244,7 +244,6 @@ async fn download_data_set( out_path: &Path, ) -> anyhow::Result<()> { tracing::info!("Downloading new data set: {}", out_path.to_string_lossy()); - // TODO: abstract this out to a function let stream = store.get_raw(in_file_name).await?; let mut bytes = tokio_util::codec::FramedRead::new( async_compression::tokio::bufread::GzipDecoder::new(tokio_util::io::StreamReader::new( From 49a742f0055a9d77df5c1dab91a0ccdef9d21093 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Mon, 29 Apr 2024 10:17:13 -0400 Subject: [PATCH 16/53] Address comments, fix some subtle bugs --- Cargo.lock | 1 + Cargo.toml | 1 + file_store/Cargo.toml | 2 +- mobile_verifier/Cargo.toml | 1 + .../src/boosting_oracles/data_sets.rs | 65 ++++++++++--------- mobile_verifier/src/boosting_oracles/mod.rs | 37 ++++++++--- mobile_verifier/src/cli/server.rs | 6 +- mobile_verifier/tests/boosting_oracles.rs | 12 +++- 8 files changed, 84 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 59ccf4a88..6d5c0ed1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4581,6 +4581,7 @@ dependencies = [ "clap 4.4.8", "config", "db-store", + "derive_builder", "file-store", "flate2", "futures", diff --git a/Cargo.toml b/Cargo.toml index 721b4919c..16a643a89 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,6 +102,7 @@ itertools = "*" tokio-util = "0" uuid = {version = "1", features = ["v4", "serde"]} tower-http = {version = "0", features = ["trace"]} +derive_builder = "0" [patch.crates-io] sqlx = { git = "https://github.com/helium/sqlx.git", rev = "92a2268f02e0cac6fccb34d3e926347071dbb88d" } diff --git a/file_store/Cargo.toml b/file_store/Cargo.toml index 601143fa9..f5ffbef93 100644 --- a/file_store/Cargo.toml +++ b/file_store/Cargo.toml @@ -45,7 +45,7 @@ base64 = {workspace = true} beacon = {workspace = true} sqlx = {workspace = true, optional = true} async-trait = {workspace = true} -derive_builder = "0" +derive_builder = {workspace = true} retainer = {workspace = true} uuid = {workspace = true} h3o = {workspace = true} diff --git a/mobile_verifier/Cargo.toml b/mobile_verifier/Cargo.toml index 9c0c6c3c0..415682f93 100644 --- a/mobile_verifier/Cargo.toml +++ b/mobile_verifier/Cargo.toml @@ -53,6 +53,7 @@ retainer = {workspace = true} uuid = {workspace = true} task-manager = {path = "../task_manager"} solana-sdk = {workspace = true} +derive_builder = {workspace = true} [dev-dependencies] backon = "0" diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index 92a645108..2f57b9a54 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -103,7 +103,22 @@ where oracle_boosting_sink: FileSinkClient, data_set_directory: PathBuf, ) -> anyhow::Result { - let latest_file_date = db::fetch_latest_file_date(&pool, T::TYPE).await?; + // Get the first data set: + let latest_file_date = if let Some(time_to_use) = + db::fetch_time_of_latest_processed_data_set(&pool, T::TYPE).await? + { + let data_set_path = get_data_set_path(&data_set_directory, T::TYPE, time_to_use); + tracing::info!( + "Found initial {} data set: {}", + T::TYPE.to_prefix(), + data_set_path.to_string_lossy() + ); + data_set.lock().await.update(&data_set_path, time_to_use)?; + Some(time_to_use) + } else { + db::fetch_latest_file_date(&pool, T::TYPE).await? + }; + Ok(Self { pool, data_set, @@ -115,17 +130,6 @@ where }) } - fn get_data_set_path(&self, time_to_use: DateTime) -> PathBuf { - let path = PathBuf::from(format!( - "{}.{}.res10.h3tree", - T::TYPE.to_prefix(), - time_to_use.timestamp() - )); - let mut dir = self.data_set_directory.clone(); - dir.push(path); - dir - } - pub async fn check_for_available_data_sets(&mut self) -> anyhow::Result<()> { let mut new_data_sets = self .store @@ -161,7 +165,11 @@ where // If there is an unprocessed data set, download it (if we need to) and process it. - let path = self.get_data_set_path(latest_unprocessed_data_set.time_to_use); + let path = get_data_set_path( + &self.data_set_directory, + T::TYPE, + latest_unprocessed_data_set.time_to_use, + ); // Download the file if it hasn't been downloaded already: if !latest_unprocessed_data_set.status.is_downloaded() { @@ -212,22 +220,6 @@ where } pub async fn run(mut self) -> anyhow::Result<()> { - // Get the first data set: - if let Some(time_to_use) = - db::fetch_time_of_latest_processed_data_set(&self.pool, T::TYPE).await? - { - let data_set_path = self.get_data_set_path(time_to_use); - tracing::info!( - "Found initial {} data set: {}", - T::TYPE.to_prefix(), - data_set_path.to_string_lossy() - ); - self.data_set - .lock() - .await - .update(&data_set_path, time_to_use)?; - } - let poll_duration = Duration::minutes(5); loop { @@ -238,6 +230,21 @@ where } } +fn get_data_set_path( + data_set_directory: &Path, + data_set_type: DataSetType, + time_to_use: DateTime, +) -> PathBuf { + let path = PathBuf::from(format!( + "{}.{}.res10.h3tree", + data_set_type.to_prefix(), + time_to_use.timestamp() + )); + let mut dir = data_set_directory.to_path_buf(); + dir.push(path); + dir +} + async fn download_data_set( store: &FileStore, in_file_name: &str, diff --git a/mobile_verifier/src/boosting_oracles/mod.rs b/mobile_verifier/src/boosting_oracles/mod.rs index b34901918..9a68423bb 100644 --- a/mobile_verifier/src/boosting_oracles/mod.rs +++ b/mobile_verifier/src/boosting_oracles/mod.rs @@ -31,11 +31,21 @@ impl HexAssignment for Assignment { } } +#[derive(derive_builder::Builder)] +#[builder(pattern = "owned")] pub struct HexBoostData { + #[builder(setter(custom))] pub footfall: Arc>, + #[builder(setter(custom))] pub landtype: Arc>, + #[builder(setter(custom))] pub urbanization: Arc>, } +impl HexBoostData { + pub fn builder() -> HexBoostDataBuilder { + HexBoostDataBuilder::default() + } +} impl Clone for HexBoostData { fn clone(&self) -> Self { @@ -47,13 +57,20 @@ impl Clone for HexBoostData { } } -impl HexBoostData { - pub fn new(footfall: Foot, landtype: Land, urbanization: Urban) -> Self { - Self { - urbanization: Arc::new(Mutex::new(urbanization)), - footfall: Arc::new(Mutex::new(footfall)), - landtype: Arc::new(Mutex::new(landtype)), - } +impl HexBoostDataBuilder { + pub fn footfall(mut self, foot: Foot) -> Self { + self.footfall = Some(Arc::new(Mutex::new(foot))); + self + } + + pub fn landtype(mut self, land: Land) -> Self { + self.landtype = Some(Arc::new(Mutex::new(land))); + self + } + + pub fn urbanization(mut self, urban: Urban) -> Self { + self.urbanization = Some(Arc::new(Mutex::new(urban))); + self } } @@ -274,7 +291,11 @@ mod tests { Urbanization::new_mock(DiskTreeMap::with_buf(urbanized_buf)?, usa_geofence); // Let the testing commence - let data = HexBoostData::new(footfall, landtype, urbanization); + let data = HexBoostData::builder() + .footfall(footfall) + .landtype(landtype) + .urbanization(urbanization) + .build()?; // NOTE(mj): formatting ignored to make it easier to see the expected change in assignments. // NOTE(mj): The semicolon at the end of the block is there to keep rust from diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index ce9dd80b0..45447f264 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -119,7 +119,11 @@ impl Cmd { let urbanization: Urbanization = Urbanization::new(usa_geofence.clone()); let footfall: Footfall = Footfall::new(); let landtype: Landtype = Landtype::new(); - let hex_boost_data = HexBoostData::new(footfall, landtype, urbanization); + let hex_boost_data = HexBoostData::builder() + .footfall(footfall) + .landtype(landtype) + .urbanization(urbanization) + .build()?; // Data sets and downloaders let data_sets_file_store = FileStore::from_settings(&settings.data_sets).await?; diff --git a/mobile_verifier/tests/boosting_oracles.rs b/mobile_verifier/tests/boosting_oracles.rs index b393e7fe8..ea918fcae 100644 --- a/mobile_verifier/tests/boosting_oracles.rs +++ b/mobile_verifier/tests/boosting_oracles.rs @@ -217,7 +217,11 @@ async fn test_footfall_and_urbanization_report(pool: PgPool) -> anyhow::Result<( transaction.commit().await?; let unassigned_hexes = UnassignedHex::fetch_unassigned(&pool); - let hex_boost_data = HexBoostData::new(footfall, landtype, urbanized); + let hex_boost_data = HexBoostData::builder() + .footfall(footfall) + .landtype(landtype) + .urbanization(urbanized) + .build()?; let oba = set_oracle_boosting_assignments(unassigned_hexes, &hex_boost_data, &pool) .await? .collect::>(); @@ -344,7 +348,11 @@ async fn test_footfall_and_urbanization_and_landtype(pool: PgPool) -> anyhow::Re transaction.commit().await?; let unassigned_hexes = UnassignedHex::fetch_unassigned(&pool); - let hex_boost_data = HexBoostData::new(footfall, landtype, urbanized); + let hex_boost_data = HexBoostData::builder() + .footfall(footfall) + .landtype(landtype) + .urbanization(urbanized) + .build()?; let _ = set_oracle_boosting_assignments(unassigned_hexes, &hex_boost_data, &pool).await?; let heartbeats = heartbeats(12, start, &owner, &cbsd_id, 0.0, 0.0, uuid); From f54b6237a1b9cec8b9c87b1cd02be19452994fd8 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Mon, 29 Apr 2024 10:18:43 -0400 Subject: [PATCH 17/53] Fix incorrect constant --- mobile_verifier/src/boosting_oracles/data_sets.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index 2f57b9a54..7d4078f15 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -405,7 +405,7 @@ pub async fn set_oracle_boosting_assignments<'a>( data_sets: &HexBoostData, pool: &'a PgPool, ) -> anyhow::Result> { - const NUMBER_OF_FIELDS_IN_QUERY: u16 = 5; + const NUMBER_OF_FIELDS_IN_QUERY: u16 = 7; const ASSIGNMENTS_MAX_BATCH_ENTRIES: usize = (u16::MAX / NUMBER_OF_FIELDS_IN_QUERY) as usize; let now = Utc::now(); From a4634f0dcc305e54675f04a304c30b19d12e5cc1 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Mon, 29 Apr 2024 10:38:37 -0400 Subject: [PATCH 18/53] Add builder for HexAssignments --- .../src/boosting_oracles/assignment.rs | 62 +++++++++++++++---- .../src/boosting_oracles/data_sets.rs | 7 ++- mobile_verifier/src/boosting_oracles/mod.rs | 6 +- 3 files changed, 60 insertions(+), 15 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/assignment.rs b/mobile_verifier/src/boosting_oracles/assignment.rs index 36a63934c..8d8a37321 100644 --- a/mobile_verifier/src/boosting_oracles/assignment.rs +++ b/mobile_verifier/src/boosting_oracles/assignment.rs @@ -1,3 +1,4 @@ +use anyhow::Result; use helium_proto::services::poc_mobile::oracle_boosting_hex_assignment::Assignment as ProtoAssignment; use rust_decimal::Decimal; use rust_decimal_macros::dec; @@ -60,24 +61,20 @@ impl fmt::Display for Assignment { } impl HexAssignments { - pub fn from_data_sets( - cell: hextree::Cell, - footfall: &impl HexAssignment, - landtype: &impl HexAssignment, - urbanized: &impl HexAssignment, - ) -> anyhow::Result { - Ok(Self { - footfall: footfall.assignment(cell)?, - landtype: landtype.assignment(cell)?, - urbanized: urbanized.assignment(cell)?, - }) + pub fn builder(cell: hextree::Cell) -> HexAssignmentsBuilder { + HexAssignmentsBuilder { + cell, + footfall: None, + landtype: None, + urbanized: None, + } } pub fn boosting_multiplier(&self) -> Decimal { let HexAssignments { footfall, - urbanized, landtype, + urbanized, } = self; use Assignment::*; @@ -112,6 +109,47 @@ impl HexAssignments { } } +pub struct HexAssignmentsBuilder { + cell: hextree::Cell, + footfall: Option>, + landtype: Option>, + urbanized: Option>, +} + +impl HexAssignmentsBuilder { + pub fn footfall(mut self, footfall: &impl HexAssignment) -> Self { + self.footfall = Some(footfall.assignment(self.cell)); + self + } + + pub fn landtype(mut self, landtype: &impl HexAssignment) -> Self { + self.landtype = Some(landtype.assignment(self.cell)); + self + } + + pub fn urbanized(mut self, urbanized: &impl HexAssignment) -> Self { + self.urbanized = Some(urbanized.assignment(self.cell)); + self + } + + pub fn build(self) -> anyhow::Result { + let Some(footfall) = self.footfall else { + anyhow::bail!("footfall assignment not set"); + }; + let Some(landtype) = self.landtype else { + anyhow::bail!("landtype assignment not set"); + }; + let Some(urbanized) = self.urbanized else { + anyhow::bail!("urbanized assignment not set"); + }; + Ok(HexAssignments { + footfall: footfall?, + urbanized: urbanized?, + landtype: landtype?, + }) + } +} + #[cfg(test)] impl HexAssignments { pub fn test_best() -> Self { diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index 7d4078f15..04489cbe9 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -421,8 +421,11 @@ pub async fn set_oracle_boosting_assignments<'a>( .into_iter() .map(|hex| { let cell = hextree::Cell::try_from(hex.hex)?; - let assignments = - HexAssignments::from_data_sets(cell, &*footfall, &*landtype, &*urbanization)?; + let assignments = HexAssignments::builder(cell) + .footfall(&*footfall) + .landtype(&*landtype) + .urbanized(&*urbanization) + .build()?; let location = format!("{:x}", hex.hex); let assignment_multiplier = (assignments.boosting_multiplier() * dec!(1000)) .to_u32() diff --git a/mobile_verifier/src/boosting_oracles/mod.rs b/mobile_verifier/src/boosting_oracles/mod.rs index 9a68423bb..feef4be9c 100644 --- a/mobile_verifier/src/boosting_oracles/mod.rs +++ b/mobile_verifier/src/boosting_oracles/mod.rs @@ -97,7 +97,11 @@ where let footfall = self.footfall.lock().await; let landtype = self.landtype.lock().await; let urbanization = self.urbanization.lock().await; - HexAssignments::from_data_sets(cell, &*footfall, &*landtype, &*urbanization) + HexAssignments::builder(cell) + .footfall(&*footfall) + .landtype(&*landtype) + .urbanized(&*urbanization) + .build() } } From 2afa4974e71dc7b98a323127ce87c2692bf3383b Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Mon, 29 Apr 2024 10:40:42 -0400 Subject: [PATCH 19/53] Add comments to settings --- mobile_verifier/src/settings.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mobile_verifier/src/settings.rs b/mobile_verifier/src/settings.rs index 2efcb388a..02e1d0958 100644 --- a/mobile_verifier/src/settings.rs +++ b/mobile_verifier/src/settings.rs @@ -20,6 +20,8 @@ pub struct Settings { pub ingest: file_store::Settings, pub data_transfer_ingest: file_store::Settings, pub output: file_store::Settings, + /// S3 bucket from which new data sets are downloaded for oracle boosting + /// assignments pub data_sets: file_store::Settings, pub metrics: poc_metrics::Settings, pub price_tracker: price::price_tracker::Settings, @@ -36,6 +38,7 @@ pub struct Settings { /// beyond which its location weight will be reduced #[serde(default = "default_max_asserted_distance_deviation")] pub max_asserted_distance_deviation: u32, + /// Directory in which new oracle boosting data sets are downloaded into pub data_sets_directory: PathBuf, // Geofencing settings pub usa_and_mexico_geofence_regions: String, From bc484659be898923829d84d4814b201f123a1de5 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Thu, 2 May 2024 19:40:12 -0400 Subject: [PATCH 20/53] Remove geofence validation specialization --- .../src/boosting_oracles/urbanization.rs | 6 ++++++ mobile_verifier/src/geofence.rs | 20 ++++--------------- mobile_verifier/src/heartbeats/cbrs.rs | 4 ++-- mobile_verifier/src/heartbeats/mod.rs | 4 ++-- mobile_verifier/src/heartbeats/wifi.rs | 4 ++-- mobile_verifier/tests/boosting_oracles.rs | 8 +------- mobile_verifier/tests/common/mod.rs | 5 ----- mobile_verifier/tests/heartbeats.rs | 8 +------- mobile_verifier/tests/modeled_coverage.rs | 8 +------- mobile_verifier/tests/rewarder_poc_dc.rs | 10 ---------- 10 files changed, 19 insertions(+), 58 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/urbanization.rs b/mobile_verifier/src/boosting_oracles/urbanization.rs index 0565cfe4a..950068461 100644 --- a/mobile_verifier/src/boosting_oracles/urbanization.rs +++ b/mobile_verifier/src/boosting_oracles/urbanization.rs @@ -26,6 +26,12 @@ impl
Urbanization
{ } } +impl
Default for Urbanization
{ + fn default() -> Self { + Self::new() + } +} + impl DataSet for Urbanization { const TYPE: DataSetType = DataSetType::Urbanization; diff --git a/mobile_verifier/src/geofence.rs b/mobile_verifier/src/geofence.rs index 100aded3e..a944f2ef3 100644 --- a/mobile_verifier/src/geofence.rs +++ b/mobile_verifier/src/geofence.rs @@ -1,18 +1,12 @@ use base64::{engine::general_purpose, Engine as _}; use h3o::{LatLng, Resolution}; use hextree::{Cell, HexTreeSet}; -use std::{collections::HashSet, fs, io::Read, path, sync::Arc}; +use std::{fs, io::Read, path, sync::Arc}; use crate::heartbeats::Heartbeat; -pub trait GeofenceValidator: Clone + Send + Sync + 'static { - fn in_valid_region(&self, t: &T) -> bool; -} - -impl GeofenceValidator for HashSet { - fn in_valid_region(&self, cell: &hextree::Cell) -> bool { - self.contains(cell) - } +pub trait GeofenceValidator: Clone + Send + Sync + 'static { + fn in_valid_region(&self, t: &Heartbeat) -> bool; } #[derive(Clone)] @@ -38,7 +32,7 @@ impl Geofence { } } -impl GeofenceValidator for Geofence { +impl GeofenceValidator for Geofence { fn in_valid_region(&self, heartbeat: &Heartbeat) -> bool { let Ok(lat_lon) = LatLng::new(heartbeat.lat, heartbeat.lon) else { return false; @@ -50,12 +44,6 @@ impl GeofenceValidator for Geofence { } } -impl GeofenceValidator for Geofence { - fn in_valid_region(&self, cell: &hextree::Cell) -> bool { - self.regions.contains(*cell) - } -} - pub fn valid_mapping_regions(encoded_files: Vec) -> anyhow::Result { let mut combined_regions: Vec = Vec::new(); for file in encoded_files { diff --git a/mobile_verifier/src/heartbeats/cbrs.rs b/mobile_verifier/src/heartbeats/cbrs.rs index e8b0a097e..91ae48e34 100644 --- a/mobile_verifier/src/heartbeats/cbrs.rs +++ b/mobile_verifier/src/heartbeats/cbrs.rs @@ -39,7 +39,7 @@ pub struct CbrsHeartbeatDaemon { impl CbrsHeartbeatDaemon where GIR: GatewayResolver, - GFV: GeofenceValidator, + GFV: GeofenceValidator, { #[allow(clippy::too_many_arguments)] pub async fn create_managed_task( @@ -197,7 +197,7 @@ where impl ManagedTask for CbrsHeartbeatDaemon where GIR: GatewayResolver, - GFV: GeofenceValidator, + GFV: GeofenceValidator, { fn start_task( self: Box, diff --git a/mobile_verifier/src/heartbeats/mod.rs b/mobile_verifier/src/heartbeats/mod.rs index d807366f9..1ea6d669e 100644 --- a/mobile_verifier/src/heartbeats/mod.rs +++ b/mobile_verifier/src/heartbeats/mod.rs @@ -388,7 +388,7 @@ impl ValidatedHeartbeat { max_distance_to_asserted: u32, max_distance_to_coverage: u32, epoch: &Range>, - geofence: &impl GeofenceValidator, + geofence: &impl GeofenceValidator, ) -> anyhow::Result { let Some(coverage_object) = heartbeat.coverage_object else { return Ok(Self::new( @@ -589,7 +589,7 @@ impl ValidatedHeartbeat { max_distance_to_asserted: u32, max_distance_to_coverage: u32, epoch: &'a Range>, - geofence: &'a impl GeofenceValidator, + geofence: &'a impl GeofenceValidator, ) -> impl Stream> + 'a { heartbeats.then(move |heartbeat| async move { Self::validate( diff --git a/mobile_verifier/src/heartbeats/wifi.rs b/mobile_verifier/src/heartbeats/wifi.rs index bd5337a6d..667b3c4e7 100644 --- a/mobile_verifier/src/heartbeats/wifi.rs +++ b/mobile_verifier/src/heartbeats/wifi.rs @@ -38,7 +38,7 @@ pub struct WifiHeartbeatDaemon { impl WifiHeartbeatDaemon where GIR: GatewayResolver, - GFV: GeofenceValidator, + GFV: GeofenceValidator, { #[allow(clippy::too_many_arguments)] pub async fn create_managed_task( @@ -188,7 +188,7 @@ where impl ManagedTask for WifiHeartbeatDaemon where GIR: GatewayResolver, - GFV: GeofenceValidator, + GFV: GeofenceValidator, { fn start_task( self: Box, diff --git a/mobile_verifier/tests/boosting_oracles.rs b/mobile_verifier/tests/boosting_oracles.rs index ea918fcae..4b8ee8c01 100644 --- a/mobile_verifier/tests/boosting_oracles.rs +++ b/mobile_verifier/tests/boosting_oracles.rs @@ -33,18 +33,12 @@ use uuid::Uuid; #[derive(Clone)] struct MockGeofence; -impl GeofenceValidator for MockGeofence { +impl GeofenceValidator for MockGeofence { fn in_valid_region(&self, _heartbeat: &Heartbeat) -> bool { true } } -impl GeofenceValidator for MockGeofence { - fn in_valid_region(&self, _cell: &u64) -> bool { - true - } -} - #[derive(Copy, Clone)] struct AllOwnersValid; diff --git a/mobile_verifier/tests/common/mod.rs b/mobile_verifier/tests/common/mod.rs index 43c6c1ce8..82c969919 100644 --- a/mobile_verifier/tests/common/mod.rs +++ b/mobile_verifier/tests/common/mod.rs @@ -21,11 +21,6 @@ pub struct MockCarrierServiceClient { pub valid_sps: ValidSpMap, } -#[derive(Debug, Clone)] -pub struct MockHexBoostingClient { - pub boosted_hexes: Vec, -} - pub struct MockFileSinkReceiver { pub receiver: tokio::sync::mpsc::Receiver, } diff --git a/mobile_verifier/tests/heartbeats.rs b/mobile_verifier/tests/heartbeats.rs index 7b759d45b..61601bb08 100644 --- a/mobile_verifier/tests/heartbeats.rs +++ b/mobile_verifier/tests/heartbeats.rs @@ -385,18 +385,12 @@ fn signal_level(hex: &str, signal_level: SignalLevel) -> anyhow::Result for MockGeofence { +impl GeofenceValidator for MockGeofence { fn in_valid_region(&self, _heartbeat: &Heartbeat) -> bool { true } } -impl GeofenceValidator for MockGeofence { - fn in_valid_region(&self, _cell: &u64) -> bool { - true - } -} - #[derive(Copy, Clone)] struct AllOwnersValid; diff --git a/mobile_verifier/tests/modeled_coverage.rs b/mobile_verifier/tests/modeled_coverage.rs index f202b916c..94df89be0 100644 --- a/mobile_verifier/tests/modeled_coverage.rs +++ b/mobile_verifier/tests/modeled_coverage.rs @@ -38,18 +38,12 @@ use uuid::Uuid; #[derive(Clone)] struct MockGeofence; -impl GeofenceValidator for MockGeofence { +impl GeofenceValidator for MockGeofence { fn in_valid_region(&self, _heartbeat: &Heartbeat) -> bool { true } } -impl GeofenceValidator for MockGeofence { - fn in_valid_region(&self, _cell: &hextree::Cell) -> bool { - true - } -} - const BOOST_HEX_PUBKEY: &str = "J9JiLTpjaShxL8eMvUs8txVw6TZ36E38SiJ89NxnMbLU"; const BOOST_CONFIG_PUBKEY: &str = "BZM1QTud72B2cpTW7PhEnFmRX7ZWzvY7DpPpNJJuDrWG"; diff --git a/mobile_verifier/tests/rewarder_poc_dc.rs b/mobile_verifier/tests/rewarder_poc_dc.rs index 4e675a634..d7f7146cd 100644 --- a/mobile_verifier/tests/rewarder_poc_dc.rs +++ b/mobile_verifier/tests/rewarder_poc_dc.rs @@ -22,7 +22,6 @@ use mobile_verifier::{ cell_type::CellType, coverage::CoverageObject, data_session, - geofence::GeofenceValidator, heartbeats::{HbType, Heartbeat, ValidatedHeartbeat}, reward_shares, rewarder, speedtests, }; @@ -58,15 +57,6 @@ impl HexBoostingInfoResolver for MockHexBoostingClient { } } -#[derive(Clone)] -struct MockGeofence; - -impl GeofenceValidator for MockGeofence { - fn in_valid_region(&self, _cell: &hextree::Cell) -> bool { - true - } -} - #[sqlx::test] async fn test_poc_and_dc_rewards(pool: PgPool) -> anyhow::Result<()> { let (mobile_rewards_client, mut mobile_rewards) = common::create_file_sink(); From e745e42226cf70b5412428af30c74f7dec34e52b Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Thu, 2 May 2024 19:53:04 -0400 Subject: [PATCH 21/53] Fix weird warning --- mobile_verifier/tests/common/mod.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mobile_verifier/tests/common/mod.rs b/mobile_verifier/tests/common/mod.rs index 82c969919..8818c90a5 100644 --- a/mobile_verifier/tests/common/mod.rs +++ b/mobile_verifier/tests/common/mod.rs @@ -21,6 +21,12 @@ pub struct MockCarrierServiceClient { pub valid_sps: ValidSpMap, } +#[derive(Debug, Clone)] +#[allow(dead_code)] +pub struct MockHexBoostingClient { + pub boosted_hexes: Vec, +} + pub struct MockFileSinkReceiver { pub receiver: tokio::sync::mpsc::Receiver, } From e3ef992ac2366d41b5a0c5639ead71e7288a15e4 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Thu, 2 May 2024 20:05:33 -0400 Subject: [PATCH 22/53] ugh --- boost_manager/tests/common/mod.rs | 1 + mobile_verifier/tests/common/mod.rs | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/boost_manager/tests/common/mod.rs b/boost_manager/tests/common/mod.rs index 1c5d267b1..f18032129 100644 --- a/boost_manager/tests/common/mod.rs +++ b/boost_manager/tests/common/mod.rs @@ -6,6 +6,7 @@ use mobile_config::boosted_hex_info::BoostedHexInfo; use tokio::{sync::mpsc::error::TryRecvError, time::timeout}; #[derive(Debug, Clone)] +#[allow(dead_code)] pub struct MockHexBoostingClient { pub boosted_hexes: Vec, } diff --git a/mobile_verifier/tests/common/mod.rs b/mobile_verifier/tests/common/mod.rs index 8818c90a5..43c6c1ce8 100644 --- a/mobile_verifier/tests/common/mod.rs +++ b/mobile_verifier/tests/common/mod.rs @@ -22,7 +22,6 @@ pub struct MockCarrierServiceClient { } #[derive(Debug, Clone)] -#[allow(dead_code)] pub struct MockHexBoostingClient { pub boosted_hexes: Vec, } From 4996d1b74cab5df8e00967bb407b9cf7c38271b5 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Thu, 2 May 2024 20:20:52 -0400 Subject: [PATCH 23/53] I am shouting at my computer rn --- mobile_verifier/tests/common/mod.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mobile_verifier/tests/common/mod.rs b/mobile_verifier/tests/common/mod.rs index 43c6c1ce8..85bac1189 100644 --- a/mobile_verifier/tests/common/mod.rs +++ b/mobile_verifier/tests/common/mod.rs @@ -14,14 +14,17 @@ use tokio::{ time::timeout, }; +#[allow(dead_code)] pub type ValidSpMap = HashMap; #[derive(Debug, Clone)] +#[allow(dead_code)] pub struct MockCarrierServiceClient { pub valid_sps: ValidSpMap, } #[derive(Debug, Clone)] +#[allow(dead_code)] pub struct MockHexBoostingClient { pub boosted_hexes: Vec, } From 31433de28e81ea6c6a0d48dd70e661d0b9d5af79 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Fri, 3 May 2024 12:45:43 -0400 Subject: [PATCH 24/53] Clean up --- .../src/boosting_oracles/data_sets.rs | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index 04489cbe9..502a6f13a 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -153,7 +153,7 @@ where } pub async fn process_data_sets(&self) -> anyhow::Result<()> { - tracing::info!("Checking for new data sets"); + tracing::info!("Checking for new {} data sets", T::TYPE.to_prefix()); let mut data_set = self.data_set.lock().await; let latest_unprocessed_data_set = db::fetch_latest_unprocessed_data_set(&self.pool, T::TYPE, data_set.timestamp()) @@ -305,9 +305,9 @@ pub mod db { ) -> sqlx::Result<()> { sqlx::query( r#" - INSERT INTO data_sets (filename, data_set, time_to_use, status) - VALUES ($1, $2, $3, 'pending') - "#, + INSERT INTO data_sets (filename, data_set, time_to_use, status) + VALUES ($1, $2, $3, 'pending') + "#, ) .bind(filename) .bind(data_set_type) @@ -323,13 +323,13 @@ pub mod db { since: Option>, ) -> sqlx::Result> { sqlx::query_as( - "SELECT filename, time_to_use, status FROM data_sets WHERE status != 'processed' AND data_set = $1 AND COALESCE(time_to_use > $2, TRUE) AND time_to_use <= $3 ORDER BY time_to_use DESC LIMIT 1" - ) - .bind(data_set_type) - .bind(since) - .bind(Utc::now()) - .fetch_optional(pool) - .await + "SELECT filename, time_to_use, status FROM data_sets WHERE status != 'processed' AND data_set = $1 AND COALESCE(time_to_use > $2, TRUE) AND time_to_use <= $3 ORDER BY time_to_use DESC LIMIT 1" + ) + .bind(data_set_type) + .bind(since) + .bind(Utc::now()) + .fetch_optional(pool) + .await } pub async fn set_data_set_status( @@ -350,11 +350,11 @@ pub mod db { data_set_type: DataSetType, ) -> sqlx::Result>> { sqlx::query_scalar( - "SELECT time_to_use FROM data_sets WHERE status = 'processed' AND data_set = $1 ORDER BY time_to_use DESC LIMIT 1" - ) - .bind(data_set_type) - .fetch_optional(pool) - .await + "SELECT time_to_use FROM data_sets WHERE status = 'processed' AND data_set = $1 ORDER BY time_to_use DESC LIMIT 1" + ) + .bind(data_set_type) + .fetch_optional(pool) + .await } /// Check if there are any pending or downloaded files prior to the given reward period From b39839037ef12aff565571cabaa880fcc2f26a30 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Fri, 3 May 2024 13:31:44 -0400 Subject: [PATCH 25/53] add migration --- mobile_verifier/migrations/33_data_sets.sql | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 mobile_verifier/migrations/33_data_sets.sql diff --git a/mobile_verifier/migrations/33_data_sets.sql b/mobile_verifier/migrations/33_data_sets.sql new file mode 100644 index 000000000..01f44d668 --- /dev/null +++ b/mobile_verifier/migrations/33_data_sets.sql @@ -0,0 +1,18 @@ +CREATE TYPE data_set_status AS enum ( + 'pending', + 'downloaded', + 'processed' +); + +CREATE TYPE data_set_type AS enum ( + 'urbanization', + 'footfall', + 'landtype' +); + +CREATE TABLE data_sets ( + filename TEXT PRIMARY KEY, + data_set data_set_type NOT NULL, + time_to_use TIMESTAMPTZ NOT NULL, + status data_set_status NOT NULL +); From 6468175b9b4910c4f75809a90cf4d423cdf50e34 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Fri, 3 May 2024 14:51:43 -0400 Subject: [PATCH 26/53] Don't lock table, don't process data sets when one is missing --- .../src/boosting_oracles/data_sets.rs | 69 ++++++++++++------- 1 file changed, 46 insertions(+), 23 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index 502a6f13a..544512aa6 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -6,7 +6,7 @@ use std::{ }; use chrono::{DateTime, Duration, Utc}; -use file_store::{file_sink::FileSinkClient, traits::TimestampEncode, FileStore}; +use file_store::{file_sink::FileSinkClient, traits::TimestampEncode, FileInfo, FileStore}; use futures_util::{Stream, StreamExt, TryFutureExt, TryStreamExt}; use helium_proto::services::poc_mobile as proto; use rust_decimal::prelude::ToPrimitive; @@ -64,9 +64,9 @@ impl DataSetStatus { impl ManagedTask for DataSetDownloaderDaemon where T: DataSet, - A: HexAssignment, - B: HexAssignment, - C: HexAssignment, + A: DataSet, + B: DataSet, + C: DataSet, { fn start_task( self: Box, @@ -91,9 +91,9 @@ where impl DataSetDownloaderDaemon where T: DataSet, - A: HexAssignment, - B: HexAssignment, - C: HexAssignment, + A: DataSet, + B: DataSet, + C: DataSet, { pub async fn new( pool: PgPool, @@ -184,6 +184,14 @@ where data_set = latest_unprocessed_data_set.filename, "Data set download complete" ); + /* + delete_old_data_sets( + &self.data_set_directory, + T::TYPE, + latest_unprocessed_data_set.time_to_use, + ) + .await?; + */ } // Now that we've downloaded the file, load it into the data set @@ -193,12 +201,18 @@ where drop(data_set); // Update the hexes - let boosting_reports = set_oracle_boosting_assignments( - UnassignedHex::fetch_all(&self.pool), - &self.data_sets, - &self.pool, - ) - .await?; + if self.data_sets.is_ready().await { + let boosting_reports = set_oracle_boosting_assignments( + UnassignedHex::fetch_all(&self.pool), + &self.data_sets, + &self.pool, + ) + .await?; + self.oracle_boosting_sink + .write_all(boosting_reports) + .await?; + self.oracle_boosting_sink.commit().await?; + } db::set_data_set_status( &self.pool, @@ -207,10 +221,6 @@ where ) .await?; - self.oracle_boosting_sink - .write_all(boosting_reports) - .await?; - self.oracle_boosting_sink.commit().await?; tracing::info!( data_set = latest_unprocessed_data_set.filename, "Data set processing complete" @@ -245,6 +255,24 @@ fn get_data_set_path( dir } +/* +async fn delete_old_data_sets( + data_set_directory: &Path, + data_set_type: DataSetType, + time_to_use: DateTime, +) -> anyhow::Result<()> { + let mut data_sets = tokio::fs::read_dir(data_set_directory).await?; + while let Some(data_set) = data_sets.next_entry().await? { + let file_info: FileInfo = data_set.file_name().to_string_lossy().parse()?; + if file_info.prefix == data_set_type.to_prefix() && file_info.timestamp < time_to_use { + tracing::info!(data_set = file_info.key, "Deleting old data set file"); + tokio::fs::remove_file(data_set.file_name()).await?; + } + } + Ok(()) +} +*/ + async fn download_data_set( store: &FileStore, in_file_name: &str, @@ -450,10 +478,6 @@ pub async fn set_oracle_boosting_assignments<'a>( }) .collect(); - let mut transaction = pool.begin().await?; - sqlx::query("LOCK TABLE hexes") - .execute(&mut transaction) - .await?; QueryBuilder::new( "INSERT INTO hexes (uuid, hex, signal_level, signal_power, footfall, landtype, urbanized)", ) @@ -475,9 +499,8 @@ pub async fn set_oracle_boosting_assignments<'a>( "#, ) .build() - .execute(&mut transaction) + .execute(pool) .await?; - transaction.commit().await?; } Ok(boost_results From f334f38cb851ced0a0ac44e607a32451ce1aa9d6 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Fri, 3 May 2024 14:53:31 -0400 Subject: [PATCH 27/53] Remove clippy --- mobile_verifier/src/boosting_oracles/data_sets.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index 544512aa6..e4b5f4887 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -6,7 +6,7 @@ use std::{ }; use chrono::{DateTime, Duration, Utc}; -use file_store::{file_sink::FileSinkClient, traits::TimestampEncode, FileInfo, FileStore}; +use file_store::{file_sink::FileSinkClient, traits::TimestampEncode, FileStore}; use futures_util::{Stream, StreamExt, TryFutureExt, TryStreamExt}; use helium_proto::services::poc_mobile as proto; use rust_decimal::prelude::ToPrimitive; From cd5fa35df9610f1a76007d5115a7f277a5cbbb45 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Fri, 3 May 2024 17:33:06 -0400 Subject: [PATCH 28/53] Remove is ready check for processing, delete old data sets --- .../src/boosting_oracles/data_sets.rs | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index e4b5f4887..efa4b9044 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -6,7 +6,7 @@ use std::{ }; use chrono::{DateTime, Duration, Utc}; -use file_store::{file_sink::FileSinkClient, traits::TimestampEncode, FileStore}; +use file_store::{file_sink::FileSinkClient, traits::TimestampEncode, FileInfo, FileStore}; use futures_util::{Stream, StreamExt, TryFutureExt, TryStreamExt}; use helium_proto::services::poc_mobile as proto; use rust_decimal::prelude::ToPrimitive; @@ -184,14 +184,12 @@ where data_set = latest_unprocessed_data_set.filename, "Data set download complete" ); - /* delete_old_data_sets( &self.data_set_directory, T::TYPE, latest_unprocessed_data_set.time_to_use, ) .await?; - */ } // Now that we've downloaded the file, load it into the data set @@ -201,18 +199,12 @@ where drop(data_set); // Update the hexes - if self.data_sets.is_ready().await { - let boosting_reports = set_oracle_boosting_assignments( - UnassignedHex::fetch_all(&self.pool), - &self.data_sets, - &self.pool, - ) - .await?; - self.oracle_boosting_sink - .write_all(boosting_reports) - .await?; - self.oracle_boosting_sink.commit().await?; - } + let boosting_reports = set_oracle_boosting_assignments( + UnassignedHex::fetch_all(&self.pool), + &self.data_sets, + &self.pool, + ) + .await?; db::set_data_set_status( &self.pool, @@ -226,6 +218,11 @@ where "Data set processing complete" ); + self.oracle_boosting_sink + .write_all(boosting_reports) + .await?; + self.oracle_boosting_sink.commit().await?; + Ok(()) } @@ -255,7 +252,6 @@ fn get_data_set_path( dir } -/* async fn delete_old_data_sets( data_set_directory: &Path, data_set_type: DataSetType, @@ -271,7 +267,6 @@ async fn delete_old_data_sets( } Ok(()) } -*/ async fn download_data_set( store: &FileStore, From 6837d4850f16a5be2ea38ed50c3bb0cb58cae334 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Mon, 6 May 2024 12:10:19 -0400 Subject: [PATCH 29/53] Refactor --- .../src/boosting_oracles/data_sets.rs | 382 ++++++++++++------ .../src/boosting_oracles/footfall.rs | 1 + .../src/boosting_oracles/landtype.rs | 1 + mobile_verifier/src/boosting_oracles/mod.rs | 53 +-- .../src/boosting_oracles/urbanization.rs | 1 + mobile_verifier/src/cli/server.rs | 75 +--- mobile_verifier/src/coverage.rs | 66 +-- 7 files changed, 288 insertions(+), 291 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index efa4b9044..80bae6220 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -2,23 +2,29 @@ use std::{ collections::HashMap, path::{Path, PathBuf}, pin::pin, - sync::Arc, }; use chrono::{DateTime, Duration, Utc}; -use file_store::{file_sink::FileSinkClient, traits::TimestampEncode, FileInfo, FileStore}; +use file_store::{ + file_sink::{self, FileSinkClient}, + file_upload::FileUpload, + traits::TimestampEncode, + FileInfo, FileStore, FileType, +}; use futures_util::{Stream, StreamExt, TryFutureExt, TryStreamExt}; use helium_proto::services::poc_mobile as proto; +use hextree::disktree::DiskTreeMap; use rust_decimal::prelude::ToPrimitive; use rust_decimal_macros::dec; use sqlx::{FromRow, PgPool, QueryBuilder}; -use task_manager::ManagedTask; -use tokio::{fs::File, io::AsyncWriteExt, sync::Mutex}; +use task_manager::{ManagedTask, TaskManager}; +use tokio::{fs::File, io::AsyncWriteExt, sync::mpsc::Receiver}; -use crate::{boosting_oracles::assignment::HexAssignments, coverage::SignalLevel}; +use crate::{boosting_oracles::assignment::HexAssignments, coverage::SignalLevel, Settings}; -use super::{HexAssignment, HexBoostData}; +use super::{footfall::Footfall, landtype::Landtype, HexAssignment, HexBoostData, Urbanization}; +#[async_trait::async_trait] pub trait DataSet: HexAssignment + Send + Sync + 'static { const TYPE: DataSetType; @@ -27,16 +33,78 @@ pub trait DataSet: HexAssignment + Send + Sync + 'static { fn update(&mut self, path: &Path, time_to_use: DateTime) -> anyhow::Result<()>; fn is_ready(&self) -> bool; + + async fn fetch_first_data_set( + &mut self, + pool: &PgPool, + data_set_directory: &Path, + ) -> anyhow::Result<()> { + let Some(first_data_set) = db::fetch_latest_processed_data_set(pool, Self::TYPE).await? + else { + return Ok(()); + }; + let path = get_data_set_path(data_set_directory, Self::TYPE, first_data_set.time_to_use); + self.update(Path::new(&path), first_data_set.time_to_use)?; + Ok(()) + } + + async fn check_for_available_data_sets( + &self, + store: &FileStore, + pool: &PgPool, + ) -> anyhow::Result<()> { + let mut new_data_sets = store.list(Self::TYPE.to_prefix(), self.timestamp(), None); + while let Some(new_data_set) = new_data_sets.next().await.transpose()? { + db::insert_new_data_set(pool, &new_data_set.key, Self::TYPE, new_data_set.timestamp) + .await?; + } + Ok(()) + } + + async fn fetch_next_available_data_set( + &mut self, + store: &FileStore, + pool: &PgPool, + data_set_directory: &Path, + ) -> anyhow::Result> { + self.check_for_available_data_sets(store, pool).await?; + + tracing::info!("Checking for new {} data sets", Self::TYPE.to_prefix()); + let latest_unprocessed_data_set = + db::fetch_latest_unprocessed_data_set(pool, Self::TYPE, self.timestamp()).await?; + + let Some(latest_unprocessed_data_set) = latest_unprocessed_data_set else { + return Ok(None); + }; + + let path = get_data_set_path( + data_set_directory, + Self::TYPE, + latest_unprocessed_data_set.time_to_use, + ); + + if !latest_unprocessed_data_set.status.is_downloaded() { + download_data_set(store, &latest_unprocessed_data_set.filename, &path).await?; + latest_unprocessed_data_set.mark_as_downloaded(pool).await?; + tracing::info!( + data_set = latest_unprocessed_data_set.filename, + "Data set download complete" + ); + } + + self.update(Path::new(&path), latest_unprocessed_data_set.time_to_use)?; + + Ok(Some(latest_unprocessed_data_set)) + } } -pub struct DataSetDownloaderDaemon { +pub struct DataSetDownloaderDaemon { pool: PgPool, - data_set: Arc>, data_sets: HexBoostData, store: FileStore, oracle_boosting_sink: FileSinkClient, data_set_directory: PathBuf, - latest_file_date: Option>, + new_coverage_object_signal: Receiver<()>, } #[derive(FromRow)] @@ -46,6 +114,18 @@ pub struct NewDataSet { status: DataSetStatus, } +impl NewDataSet { + async fn mark_as_downloaded(&self, pool: &PgPool) -> anyhow::Result<()> { + db::set_data_set_status(pool, &self.filename, DataSetStatus::Downloaded).await?; + Ok(()) + } + + async fn mark_as_processed(&self, pool: &PgPool) -> anyhow::Result<()> { + db::set_data_set_status(pool, &self.filename, DataSetStatus::Processed).await?; + Ok(()) + } +} + #[derive(Copy, Clone, sqlx::Type)] #[sqlx(type_name = "data_set_status")] #[sqlx(rename_all = "lowercase")] @@ -61,9 +141,8 @@ impl DataSetStatus { } } -impl ManagedTask for DataSetDownloaderDaemon +impl ManagedTask for DataSetDownloaderDaemon where - T: DataSet, A: DataSet, B: DataSet, C: DataSet, @@ -88,140 +167,138 @@ where } } -impl DataSetDownloaderDaemon +impl + DataSetDownloaderDaemon, Landtype, Urbanization> +{ + pub async fn create_managed_task( + pool: PgPool, + settings: &Settings, + file_upload: FileUpload, + new_coverage_object_signal: Receiver<()>, + ) -> anyhow::Result { + let (oracle_boosting_reports, oracle_boosting_reports_server) = + file_sink::FileSinkBuilder::new( + FileType::OracleBoostingReport, + settings.store_base_path(), + file_upload.clone(), + concat!(env!("CARGO_PKG_NAME"), "_oracle_boosting_report"), + ) + .auto_commit(false) + .roll_time(Duration::minutes(15)) + .create() + .await?; + + let urbanization: Urbanization = Urbanization::new(); + let footfall: Footfall = Footfall::new(); + let landtype: Landtype = Landtype::new(); + let hex_boost_data = HexBoostData::builder() + .footfall(footfall) + .landtype(landtype) + .urbanization(urbanization) + .build()?; + + let data_set_downloader = Self::new( + pool, + hex_boost_data, + FileStore::from_settings(&settings.data_sets).await?, + oracle_boosting_reports, + settings.data_sets_directory.clone(), + new_coverage_object_signal, + ); + + Ok(TaskManager::builder() + .add_task(oracle_boosting_reports_server) + .add_task(data_set_downloader) + .build()) + } +} + +impl DataSetDownloaderDaemon where - T: DataSet, A: DataSet, B: DataSet, C: DataSet, { - pub async fn new( + pub fn new( pool: PgPool, - data_set: Arc>, data_sets: HexBoostData, store: FileStore, oracle_boosting_sink: FileSinkClient, data_set_directory: PathBuf, - ) -> anyhow::Result { - // Get the first data set: - let latest_file_date = if let Some(time_to_use) = - db::fetch_time_of_latest_processed_data_set(&pool, T::TYPE).await? - { - let data_set_path = get_data_set_path(&data_set_directory, T::TYPE, time_to_use); - tracing::info!( - "Found initial {} data set: {}", - T::TYPE.to_prefix(), - data_set_path.to_string_lossy() - ); - data_set.lock().await.update(&data_set_path, time_to_use)?; - Some(time_to_use) - } else { - db::fetch_latest_file_date(&pool, T::TYPE).await? - }; - - Ok(Self { + new_coverage_object_signal: Receiver<()>, + ) -> Self { + Self { pool, - data_set, data_sets, store, oracle_boosting_sink, data_set_directory, - latest_file_date, - }) + new_coverage_object_signal, + } } - pub async fn check_for_available_data_sets(&mut self) -> anyhow::Result<()> { - let mut new_data_sets = self - .store - .list(T::TYPE.to_prefix(), self.latest_file_date, None); - while let Some(new_data_set) = new_data_sets.next().await.transpose()? { - tracing::info!( - "Found new data set: {}, {:#?}", - new_data_set.key, - new_data_set - ); - db::insert_new_data_set( + async fn check_for_new_data_sets(&mut self) -> anyhow::Result<()> { + let new_urbanized = self + .data_sets + .urbanization + .fetch_next_available_data_set(&self.store, &self.pool, &self.data_set_directory) + .await?; + let new_footfall = self + .data_sets + .footfall + .fetch_next_available_data_set(&self.store, &self.pool, &self.data_set_directory) + .await?; + let new_landtype = self + .data_sets + .landtype + .fetch_next_available_data_set(&self.store, &self.pool, &self.data_set_directory) + .await?; + + // If all of the data sets are ready and there is at least one new one, re-process all + // hex assignments: + let new_data_set = + new_urbanized.is_some() || new_footfall.is_some() || new_landtype.is_some(); + if self.data_sets.is_ready() && new_data_set { + tracing::info!("Processing new data sets"); + let boosting_reports = set_oracle_boosting_assignments( + UnassignedHex::fetch_all(&self.pool), + &self.data_sets, &self.pool, - &new_data_set.key, - T::TYPE, - new_data_set.timestamp, ) .await?; - self.latest_file_date = Some(new_data_set.timestamp); - } - Ok(()) - } - - pub async fn process_data_sets(&self) -> anyhow::Result<()> { - tracing::info!("Checking for new {} data sets", T::TYPE.to_prefix()); - let mut data_set = self.data_set.lock().await; - let latest_unprocessed_data_set = - db::fetch_latest_unprocessed_data_set(&self.pool, T::TYPE, data_set.timestamp()) + self.oracle_boosting_sink + .write_all(boosting_reports) .await?; + } - let Some(latest_unprocessed_data_set) = latest_unprocessed_data_set else { - return Ok(()); - }; - - // If there is an unprocessed data set, download it (if we need to) and process it. - - let path = get_data_set_path( - &self.data_set_directory, - T::TYPE, - latest_unprocessed_data_set.time_to_use, - ); - - // Download the file if it hasn't been downloaded already: - if !latest_unprocessed_data_set.status.is_downloaded() { - download_data_set(&self.store, &latest_unprocessed_data_set.filename, &path).await?; - db::set_data_set_status( - &self.pool, - &latest_unprocessed_data_set.filename, - DataSetStatus::Downloaded, + // Mark the new data sets as processed and delete the old ones + if let Some(new_urbanized) = new_urbanized { + new_urbanized.mark_as_processed(&self.pool).await?; + delete_old_data_sets( + &self.data_set_directory, + DataSetType::Urbanization, + new_urbanized.time_to_use, ) .await?; - tracing::info!( - data_set = latest_unprocessed_data_set.filename, - "Data set download complete" - ); + } + if let Some(new_footfall) = new_footfall { + new_footfall.mark_as_processed(&self.pool).await?; delete_old_data_sets( &self.data_set_directory, - T::TYPE, - latest_unprocessed_data_set.time_to_use, + DataSetType::Footfall, + new_footfall.time_to_use, ) .await?; } - - // Now that we've downloaded the file, load it into the data set - data_set.update(Path::new(&path), latest_unprocessed_data_set.time_to_use)?; - - // Release the lock as it is no longer needed - drop(data_set); - - // Update the hexes - let boosting_reports = set_oracle_boosting_assignments( - UnassignedHex::fetch_all(&self.pool), - &self.data_sets, - &self.pool, - ) - .await?; - - db::set_data_set_status( - &self.pool, - &latest_unprocessed_data_set.filename, - DataSetStatus::Processed, - ) - .await?; - - tracing::info!( - data_set = latest_unprocessed_data_set.filename, - "Data set processing complete" - ); - - self.oracle_boosting_sink - .write_all(boosting_reports) + if let Some(new_landtype) = new_landtype { + new_landtype.mark_as_processed(&self.pool).await?; + delete_old_data_sets( + &self.data_set_directory, + DataSetType::Landtype, + new_landtype.time_to_use, + ) .await?; - self.oracle_boosting_sink.commit().await?; + } Ok(()) } @@ -229,10 +306,39 @@ where pub async fn run(mut self) -> anyhow::Result<()> { let poll_duration = Duration::minutes(5); + self.data_sets + .urbanization + .fetch_first_data_set(&self.pool, &self.data_set_directory) + .await?; + self.data_sets + .footfall + .fetch_first_data_set(&self.pool, &self.data_set_directory) + .await?; + self.data_sets + .landtype + .fetch_first_data_set(&self.pool, &self.data_set_directory) + .await?; + loop { - self.check_for_available_data_sets().await?; - self.process_data_sets().await?; - tokio::time::sleep(poll_duration.to_std()?).await; + #[rustfmt::skip] + tokio::select! { + _ = self.new_coverage_object_signal.recv() => { + // If we see a new coverage object, we want to assign only those hexes + // that don't have an assignment + let boosting_reports = set_oracle_boosting_assignments( + UnassignedHex::fetch_unassigned(&self.pool), + &self.data_sets, + &self.pool, + ) + .await?; + self.oracle_boosting_sink + .write_all(boosting_reports) + .await?; + }, + _ = tokio::time::sleep(poll_duration.to_std()?) => { + self.check_for_new_data_sets().await?; + } + } } } } @@ -330,6 +436,7 @@ pub mod db { r#" INSERT INTO data_sets (filename, data_set, time_to_use, status) VALUES ($1, $2, $3, 'pending') + ON CONFLICT DO NOTHING "#, ) .bind(filename) @@ -355,6 +462,18 @@ pub mod db { .await } + pub async fn fetch_latest_processed_data_set( + pool: &PgPool, + data_set_type: DataSetType, + ) -> sqlx::Result> { + sqlx::query_as( + "SELECT filename, time_to_use, status FROM data_sets WHERE status = 'processed' AND data_set = $1 ORDER BY time_to_use DESC LIMIT 1" + ) + .bind(data_set_type) + .fetch_optional(pool) + .await + } + pub async fn set_data_set_status( pool: &PgPool, filename: &str, @@ -385,12 +504,19 @@ pub mod db { pool: &PgPool, period_end: DateTime, ) -> sqlx::Result { - sqlx::query_scalar( - "SELECT COUNT(*) > 0 FROM data_sets WHERE time_to_use <= $1 AND status != 'processed'", + Ok( + sqlx::query_scalar( + "SELECT COUNT(*) > 0 FROM data_sets WHERE time_to_use <= $1 AND status != 'processed'", + ) + .bind(period_end) + .fetch_one(pool) + .await? + || sqlx::query_scalar( + "SELECT COUNT(*) > 0 from hexes where urbanized IS NULL OR footfall IS NULL OR landtype IS NULL" + ) + .fetch_one(pool) + .await? ) - .bind(period_end) - .fetch_one(pool) - .await } } @@ -435,19 +561,15 @@ pub async fn set_oracle_boosting_assignments<'a>( let mut boost_results = HashMap::>::new(); let mut unassigned_hexes = pin!(unassigned_hexes.try_chunks(ASSIGNMENTS_MAX_BATCH_ENTRIES)); - let urbanization = data_sets.urbanization.lock().await; - let footfall = data_sets.footfall.lock().await; - let landtype = data_sets.landtype.lock().await; - while let Some(hexes) = unassigned_hexes.try_next().await? { let hexes: anyhow::Result> = hexes .into_iter() .map(|hex| { let cell = hextree::Cell::try_from(hex.hex)?; let assignments = HexAssignments::builder(cell) - .footfall(&*footfall) - .landtype(&*landtype) - .urbanized(&*urbanization) + .footfall(&data_sets.footfall) + .landtype(&data_sets.landtype) + .urbanized(&data_sets.urbanization) .build()?; let location = format!("{:x}", hex.hex); let assignment_multiplier = (assignments.boosting_multiplier() * dec!(1000)) diff --git a/mobile_verifier/src/boosting_oracles/footfall.rs b/mobile_verifier/src/boosting_oracles/footfall.rs index 829050202..1b134bca7 100644 --- a/mobile_verifier/src/boosting_oracles/footfall.rs +++ b/mobile_verifier/src/boosting_oracles/footfall.rs @@ -32,6 +32,7 @@ impl Default for Footfall { } } +#[async_trait::async_trait] impl DataSet for Footfall { const TYPE: DataSetType = DataSetType::Footfall; diff --git a/mobile_verifier/src/boosting_oracles/landtype.rs b/mobile_verifier/src/boosting_oracles/landtype.rs index 0c862163c..6493bde3c 100644 --- a/mobile_verifier/src/boosting_oracles/landtype.rs +++ b/mobile_verifier/src/boosting_oracles/landtype.rs @@ -32,6 +32,7 @@ impl Default for Landtype { } } +#[async_trait::async_trait] impl DataSet for Landtype { const TYPE: DataSetType = DataSetType::Landtype; diff --git a/mobile_verifier/src/boosting_oracles/mod.rs b/mobile_verifier/src/boosting_oracles/mod.rs index 59aeb9839..6c071fe5a 100644 --- a/mobile_verifier/src/boosting_oracles/mod.rs +++ b/mobile_verifier/src/boosting_oracles/mod.rs @@ -5,14 +5,12 @@ pub mod landtype; pub mod urbanization; use std::collections::HashMap; -use std::sync::Arc; use crate::boosting_oracles::assignment::HexAssignments; pub use assignment::Assignment; pub use data_sets::*; use hextree::disktree::DiskTreeMap; -use tokio::sync::Mutex; pub use urbanization::Urbanization; pub trait HexAssignment: Send + Sync + 'static { @@ -34,12 +32,9 @@ impl HexAssignment for Assignment { #[derive(derive_builder::Builder)] #[builder(pattern = "owned")] pub struct HexBoostData { - #[builder(setter(custom))] - pub footfall: Arc>, - #[builder(setter(custom))] - pub landtype: Arc>, - #[builder(setter(custom))] - pub urbanization: Arc>, + pub footfall: Foot, + pub landtype: Land, + pub urbanization: Urban, } impl HexBoostData { pub fn builder() -> HexBoostDataBuilder { @@ -47,43 +42,14 @@ impl HexBoostData { } } -impl Clone for HexBoostData { - fn clone(&self) -> Self { - Self { - footfall: self.footfall.clone(), - landtype: self.landtype.clone(), - urbanization: self.urbanization.clone(), - } - } -} - -impl HexBoostDataBuilder { - pub fn footfall(mut self, foot: Foot) -> Self { - self.footfall = Some(Arc::new(Mutex::new(foot))); - self - } - - pub fn landtype(mut self, land: Land) -> Self { - self.landtype = Some(Arc::new(Mutex::new(land))); - self - } - - pub fn urbanization(mut self, urban: Urban) -> Self { - self.urbanization = Some(Arc::new(Mutex::new(urban))); - self - } -} - impl HexBoostData where Foot: DataSet, Land: DataSet, Urban: DataSet, { - pub async fn is_ready(&self) -> bool { - self.urbanization.lock().await.is_ready() - && self.footfall.lock().await.is_ready() - && self.landtype.lock().await.is_ready() + pub fn is_ready(&self) -> bool { + self.urbanization.is_ready() && self.footfall.is_ready() && self.landtype.is_ready() } } @@ -94,13 +60,10 @@ where Urban: HexAssignment, { pub async fn assignments(&self, cell: hextree::Cell) -> anyhow::Result { - let footfall = self.footfall.lock().await; - let landtype = self.landtype.lock().await; - let urbanization = self.urbanization.lock().await; HexAssignments::builder(cell) - .footfall(&*footfall) - .landtype(&*landtype) - .urbanized(&*urbanization) + .footfall(&self.footfall) + .landtype(&self.landtype) + .urbanized(&self.urbanization) .build() } } diff --git a/mobile_verifier/src/boosting_oracles/urbanization.rs b/mobile_verifier/src/boosting_oracles/urbanization.rs index 950068461..506598943 100644 --- a/mobile_verifier/src/boosting_oracles/urbanization.rs +++ b/mobile_verifier/src/boosting_oracles/urbanization.rs @@ -32,6 +32,7 @@ impl
Default for Urbanization
{ } } +#[async_trait::async_trait] impl DataSet for Urbanization { const TYPE: DataSetType = DataSetType::Urbanization; diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index 6b6e2db5d..4c038b452 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -1,8 +1,5 @@ use crate::{ - boosting_oracles::{ - footfall::Footfall, landtype::Landtype, urbanization::Urbanization, - DataSetDownloaderDaemon, HexBoostData, - }, + boosting_oracles::DataSetDownloaderDaemon, coverage::CoverageDaemon, data_session::DataSessionIngestor, geofence::Geofence, @@ -20,12 +17,12 @@ use file_store::{ file_upload::{self}, FileStore, FileType, }; -use hextree::disktree::DiskTreeMap; use mobile_config::client::{ entity_client::EntityClient, hex_boosting_client::HexBoostingClient, AuthorizationClient, CarrierServiceClient, GatewayClient, }; use task_manager::TaskManager; +use tokio::sync::mpsc::channel; #[derive(Debug, clap::Args)] pub struct Cmd {} @@ -104,67 +101,12 @@ impl Cmd { settings.usa_and_mexico_fencing_resolution()?, )?; - let (oracle_boosting_reports, oracle_boosting_reports_server) = - file_sink::FileSinkBuilder::new( - FileType::OracleBoostingReport, - store_base_path, - file_upload.clone(), - concat!(env!("CARGO_PKG_NAME"), "_oracle_boosting_report"), - ) - .auto_commit(false) - .roll_time(Duration::minutes(15)) - .create() - .await?; - - let urbanization: Urbanization = Urbanization::new(); - let footfall: Footfall = Footfall::new(); - let landtype: Landtype = Landtype::new(); - let hex_boost_data = HexBoostData::builder() - .footfall(footfall) - .landtype(landtype) - .urbanization(urbanization) - .build()?; - - // Data sets and downloaders - let data_sets_file_store = FileStore::from_settings(&settings.data_sets).await?; - let footfall_data_set_downloader = DataSetDownloaderDaemon::new( - pool.clone(), - hex_boost_data.footfall.clone(), - hex_boost_data.clone(), - data_sets_file_store.clone(), - oracle_boosting_reports.clone(), - settings.data_sets_directory.clone(), - ) - .await?; - - let landtype_data_set_downloader = DataSetDownloaderDaemon::new( - pool.clone(), - hex_boost_data.landtype.clone(), - hex_boost_data.clone(), - data_sets_file_store.clone(), - oracle_boosting_reports.clone(), - settings.data_sets_directory.clone(), - ) - .await?; - - let urbanization_data_set_downloader = DataSetDownloaderDaemon::new( - pool.clone(), - hex_boost_data.urbanization.clone(), - hex_boost_data.clone(), - data_sets_file_store.clone(), - oracle_boosting_reports, - settings.data_sets_directory.clone(), - ) - .await?; + let (new_coverage_obj_signal_tx, new_coverage_obj_signal_rx) = channel(10); TaskManager::builder() .add_task(file_upload_server) .add_task(valid_heartbeats_server) .add_task(seniority_updates_server) - .add_task(oracle_boosting_reports_server) - .add_task(landtype_data_set_downloader) - .add_task(footfall_data_set_downloader) - .add_task(urbanization_data_set_downloader) .add_task(speedtests_avg_server) .add_task( CbrsHeartbeatDaemon::create_managed_task( @@ -208,7 +150,16 @@ impl Cmd { file_upload.clone(), report_ingest.clone(), auth_client.clone(), - hex_boost_data, + new_coverage_obj_signal_tx, + ) + .await?, + ) + .add_task( + DataSetDownloaderDaemon::create_managed_task( + pool.clone(), + settings, + file_upload.clone(), + new_coverage_obj_signal_rx, ) .await?, ) diff --git a/mobile_verifier/src/coverage.rs b/mobile_verifier/src/coverage.rs index c3a15702a..d94d134d6 100644 --- a/mobile_verifier/src/coverage.rs +++ b/mobile_verifier/src/coverage.rs @@ -1,8 +1,5 @@ use crate::{ - boosting_oracles::{ - assignment::HexAssignments, set_oracle_boosting_assignments, DataSet, HexBoostData, - UnassignedHex, - }, + boosting_oracles::assignment::HexAssignments, heartbeats::{HbType, KeyType, OwnedKeyType}, IsAuthorized, Settings, }; @@ -44,7 +41,7 @@ use std::{ time::Instant, }; use task_manager::{ManagedTask, TaskManager}; -use tokio::sync::mpsc::Receiver; +use tokio::sync::mpsc::{Receiver, Sender}; use uuid::Uuid; #[derive(Copy, Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Type)] @@ -68,28 +65,22 @@ impl From for SignalLevel { } } -pub struct CoverageDaemon { +pub struct CoverageDaemon { pool: Pool, auth_client: AuthorizationClient, - hex_boost_data: HexBoostData, coverage_objs: Receiver>, coverage_obj_sink: FileSinkClient, - oracle_boosting_sink: FileSinkClient, + new_coverage_object_signal: Sender<()>, } -impl CoverageDaemon -where - Foot: DataSet, - Land: DataSet, - Urban: DataSet, -{ +impl CoverageDaemon { pub async fn create_managed_task( pool: Pool, settings: &Settings, file_upload: FileUpload, file_store: FileStore, auth_client: AuthorizationClient, - hex_boost_data: HexBoostData, + new_coverage_object_signal: Sender<()>, ) -> anyhow::Result { let (valid_coverage_objs, valid_coverage_objs_server) = file_sink::FileSinkBuilder::new( FileType::CoverageObject, @@ -102,19 +93,6 @@ where .create() .await?; - // Oracle boosting reports - let (oracle_boosting_reports, oracle_boosting_reports_server) = - file_sink::FileSinkBuilder::new( - FileType::OracleBoostingReport, - settings.store_base_path(), - file_upload, - concat!(env!("CARGO_PKG_NAME"), "_oracle_boosting_report"), - ) - .auto_commit(false) - .roll_time(Duration::minutes(15)) - .create() - .await?; - let (coverage_objs, coverage_objs_server) = file_source::continuous_source::() .state(pool.clone()) @@ -128,15 +106,13 @@ where let coverage_daemon = CoverageDaemon::new( pool, auth_client, - hex_boost_data, coverage_objs, valid_coverage_objs, - oracle_boosting_reports, + new_coverage_object_signal, ); Ok(TaskManager::builder() .add_task(valid_coverage_objs_server) - .add_task(oracle_boosting_reports_server) .add_task(coverage_objs_server) .add_task(coverage_daemon) .build()) @@ -145,18 +121,16 @@ where pub fn new( pool: PgPool, auth_client: AuthorizationClient, - hex_boost_data: HexBoostData, coverage_objs: Receiver>, coverage_obj_sink: FileSinkClient, - oracle_boosting_sink: FileSinkClient, + new_coverage_object_signal: Sender<()>, ) -> Self { Self { pool, auth_client, - hex_boost_data, coverage_objs, coverage_obj_sink, - oracle_boosting_sink, + new_coverage_object_signal, } } @@ -203,30 +177,14 @@ where self.coverage_obj_sink.commit().await?; transaction.commit().await?; - // After writing all of the coverage objects, we set their oracle boosting assignments. - // If the data sets are not ready, we leave everything as NULL. - if !self.hex_boost_data.is_ready().await { - return Ok(()); - } - let unassigned_hexes = UnassignedHex::fetch_unassigned(&self.pool); - let boosting_reports = - set_oracle_boosting_assignments(unassigned_hexes, &self.hex_boost_data, &self.pool) - .await?; - self.oracle_boosting_sink - .write_all(boosting_reports) - .await?; - self.oracle_boosting_sink.commit().await?; + // Tell the data set manager to update the assignments + self.new_coverage_object_signal.send(()).await?; Ok(()) } } -impl ManagedTask for CoverageDaemon -where - Foot: DataSet, - Land: DataSet, - Urban: DataSet, -{ +impl ManagedTask for CoverageDaemon { fn start_task( self: Box, shutdown: triggered::Listener, From fc5d5d4d0eecaf4eeb1e21084e93ba1c0004e157 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Mon, 6 May 2024 15:16:04 -0400 Subject: [PATCH 30/53] Use try_send instead of send --- mobile_verifier/src/cli/server.rs | 2 +- mobile_verifier/src/coverage.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index 4c038b452..83fa3a35e 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -101,7 +101,7 @@ impl Cmd { settings.usa_and_mexico_fencing_resolution()?, )?; - let (new_coverage_obj_signal_tx, new_coverage_obj_signal_rx) = channel(10); + let (new_coverage_obj_signal_tx, new_coverage_obj_signal_rx) = channel(1); TaskManager::builder() .add_task(file_upload_server) diff --git a/mobile_verifier/src/coverage.rs b/mobile_verifier/src/coverage.rs index d94d134d6..bc486794f 100644 --- a/mobile_verifier/src/coverage.rs +++ b/mobile_verifier/src/coverage.rs @@ -177,8 +177,8 @@ impl CoverageDaemon { self.coverage_obj_sink.commit().await?; transaction.commit().await?; - // Tell the data set manager to update the assignments - self.new_coverage_object_signal.send(()).await?; + // Tell the data set manager to update the assignments. + self.new_coverage_object_signal.try_send(())?; Ok(()) } From f95aef90588c83ae73ae582718c35e128d048da4 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Mon, 6 May 2024 15:25:48 -0400 Subject: [PATCH 31/53] Clippy and tests --- mobile_verifier/src/coverage.rs | 2 +- .../tests/integrations/boosting_oracles.rs | 1 - .../tests/integrations/common/mod.rs | 36 ++++++++++--------- .../tests/integrations/hex_boosting.rs | 8 +++-- .../tests/integrations/modeled_coverage.rs | 9 +++-- .../tests/integrations/rewarder_poc_dc.rs | 9 +++-- 6 files changed, 38 insertions(+), 27 deletions(-) diff --git a/mobile_verifier/src/coverage.rs b/mobile_verifier/src/coverage.rs index bc486794f..0b2c51c95 100644 --- a/mobile_verifier/src/coverage.rs +++ b/mobile_verifier/src/coverage.rs @@ -177,7 +177,7 @@ impl CoverageDaemon { self.coverage_obj_sink.commit().await?; transaction.commit().await?; - // Tell the data set manager to update the assignments. + // Tell the data set manager to update the assignments. self.new_coverage_object_signal.try_send(())?; Ok(()) diff --git a/mobile_verifier/tests/integrations/boosting_oracles.rs b/mobile_verifier/tests/integrations/boosting_oracles.rs index e61d8e660..bc0126fa6 100644 --- a/mobile_verifier/tests/integrations/boosting_oracles.rs +++ b/mobile_verifier/tests/integrations/boosting_oracles.rs @@ -1,4 +1,3 @@ -use crate::common; use anyhow::Context; use chrono::{DateTime, Duration, Utc}; use file_store::{ diff --git a/mobile_verifier/tests/integrations/common/mod.rs b/mobile_verifier/tests/integrations/common/mod.rs index 9c5b6193d..f165495ff 100644 --- a/mobile_verifier/tests/integrations/common/mod.rs +++ b/mobile_verifier/tests/integrations/common/mod.rs @@ -12,7 +12,7 @@ use mobile_config::{ boosted_hex_info::{BoostedHexInfo, BoostedHexInfoStream}, client::{hex_boosting_client::HexBoostingInfoResolver, ClientError}, }; -use mobile_verifier::boosting_oracles::{Assignment, BoostedHexAssignments, HexAssignments}; +use mobile_verifier::boosting_oracles::{Assignment, HexBoostData}; use std::collections::HashMap; use tokio::{sync::mpsc::error::TryRecvError, time::timeout}; @@ -182,24 +182,26 @@ pub fn seconds(s: u64) -> std::time::Duration { type MockAssignmentMap = HashMap; -#[derive(Default)] +pub fn mock_hex_boost_data_default( +) -> HexBoostData { + HexBoostData::builder() + .urbanization(MockAssignmentMap::default()) + .footfall(MockAssignmentMap::default()) + .landtype(MockAssignmentMap::default()) + .build() + .unwrap() +} + #[allow(dead_code)] -pub struct MockHexAssignments { +pub fn mock_hex_boost_data( footfall: MockAssignmentMap, urbanized: MockAssignmentMap, landtype: MockAssignmentMap, -} - -impl MockHexAssignments { - pub fn new( - footfall: MockAssignmentMap, - urbanized: MockAssignmentMap, - landtype: MockAssignmentMap, - ) -> Self { - Self { - footfall, - urbanized, - landtype, - } - } +) -> HexBoostData { + HexBoostData::builder() + .footfall(footfall) + .urbanization(urbanized) + .landtype(landtype) + .build() + .unwrap() } diff --git a/mobile_verifier/tests/integrations/hex_boosting.rs b/mobile_verifier/tests/integrations/hex_boosting.rs index b031e32ec..8dcbb823d 100644 --- a/mobile_verifier/tests/integrations/hex_boosting.rs +++ b/mobile_verifier/tests/integrations/hex_boosting.rs @@ -36,8 +36,12 @@ const BOOST_CONFIG_PUBKEY: &str = "BZM1QTud72B2cpTW7PhEnFmRX7ZWzvY7DpPpNJJuDrWG" async fn update_assignments(pool: &PgPool) -> anyhow::Result<()> { let unassigned_hexes = UnassignedHex::fetch_unassigned(pool); - let _ = set_oracle_boosting_assignments(unassigned_hexes, &MockHexAssignments::best(), pool) - .await?; + let _ = set_oracle_boosting_assignments( + unassigned_hexes, + &common::mock_hex_boost_data_default(), + pool, + ) + .await?; Ok(()) } diff --git a/mobile_verifier/tests/integrations/modeled_coverage.rs b/mobile_verifier/tests/integrations/modeled_coverage.rs index baa0e8c60..392801766 100644 --- a/mobile_verifier/tests/integrations/modeled_coverage.rs +++ b/mobile_verifier/tests/integrations/modeled_coverage.rs @@ -1,6 +1,5 @@ use crate::common; use chrono::{DateTime, Duration, Utc}; -use common::MockHexAssignments; use file_store::{ coverage::{CoverageObjectIngestReport, RadioHexSignalLevel}, heartbeat::{CbrsHeartbeat, CbrsHeartbeatIngestReport}, @@ -407,8 +406,12 @@ async fn process_input( transaction.commit().await?; let unassigned_hexes = UnassignedHex::fetch_unassigned(pool); - let _ = set_oracle_boosting_assignments(unassigned_hexes, &MockHexAssignments::best(), pool) - .await?; + let _ = set_oracle_boosting_assignments( + unassigned_hexes, + &common::mock_hex_boost_data_default(), + pool, + ) + .await?; let mut transaction = pool.begin().await?; let mut heartbeats = pin!(ValidatedHeartbeat::validate_heartbeats( diff --git a/mobile_verifier/tests/integrations/rewarder_poc_dc.rs b/mobile_verifier/tests/integrations/rewarder_poc_dc.rs index daf7721b7..2812a06be 100644 --- a/mobile_verifier/tests/integrations/rewarder_poc_dc.rs +++ b/mobile_verifier/tests/integrations/rewarder_poc_dc.rs @@ -1,6 +1,5 @@ use crate::common::{self, MockFileSinkReceiver, MockHexBoostingClient}; use chrono::{DateTime, Duration as ChronoDuration, Utc}; -use common::MockHexAssignments; use file_store::{ coverage::{CoverageObject as FSCoverageObject, KeyType, RadioHexSignalLevel}, speedtest::CellSpeedtest, @@ -268,8 +267,12 @@ async fn seed_heartbeats( async fn update_assignments(pool: &PgPool) -> anyhow::Result<()> { let unassigned_hexes = UnassignedHex::fetch_unassigned(pool); - let _ = set_oracle_boosting_assignments(unassigned_hexes, &MockHexAssignments::best(), pool) - .await?; + let _ = set_oracle_boosting_assignments( + unassigned_hexes, + &common::mock_hex_boost_data_default(), + pool, + ) + .await?; Ok(()) } From fcbd3aef10ceaa402174d2e6665134b3a0ef3580 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Mon, 6 May 2024 16:31:10 -0400 Subject: [PATCH 32/53] Fix tests --- mobile_verifier/tests/integrations/common/mod.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/mobile_verifier/tests/integrations/common/mod.rs b/mobile_verifier/tests/integrations/common/mod.rs index f165495ff..1ec3a0a94 100644 --- a/mobile_verifier/tests/integrations/common/mod.rs +++ b/mobile_verifier/tests/integrations/common/mod.rs @@ -180,18 +180,17 @@ pub fn seconds(s: u64) -> std::time::Duration { std::time::Duration::from_secs(s) } -type MockAssignmentMap = HashMap; - -pub fn mock_hex_boost_data_default( -) -> HexBoostData { +pub fn mock_hex_boost_data_default() -> HexBoostData { HexBoostData::builder() - .urbanization(MockAssignmentMap::default()) - .footfall(MockAssignmentMap::default()) - .landtype(MockAssignmentMap::default()) + .urbanization(Assignment::A) + .footfall(Assignment::A) + .landtype(Assignment::A) .build() .unwrap() } +type MockAssignmentMap = HashMap; + #[allow(dead_code)] pub fn mock_hex_boost_data( footfall: MockAssignmentMap, From b2ed62ced90bad8befa4fc1f8496f461d1b5d40a Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Mon, 6 May 2024 21:06:37 -0400 Subject: [PATCH 33/53] Address comments --- .../src/boosting_oracles/data_sets.rs | 12 ++++ mobile_verifier/src/boosting_oracles/mod.rs | 62 +++++++++---------- mobile_verifier/src/coverage.rs | 2 +- 3 files changed, 44 insertions(+), 32 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index 80bae6220..3a3eee302 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -319,6 +319,18 @@ where .fetch_first_data_set(&self.pool, &self.data_set_directory) .await?; + // Attempt to fill in any unassigned hexes. This is for the edge case in + // which we shutdown before a coverage object updates. + let boosting_reports = set_oracle_boosting_assignments( + UnassignedHex::fetch_unassigned(&self.pool), + &self.data_sets, + &self.pool, + ) + .await?; + self.oracle_boosting_sink + .write_all(boosting_reports) + .await?; + loop { #[rustfmt::skip] tokio::select! { diff --git a/mobile_verifier/src/boosting_oracles/mod.rs b/mobile_verifier/src/boosting_oracles/mod.rs index 6c071fe5a..773c86cf7 100644 --- a/mobile_verifier/src/boosting_oracles/mod.rs +++ b/mobile_verifier/src/boosting_oracles/mod.rs @@ -59,7 +59,7 @@ where Land: HexAssignment, Urban: HexAssignment, { - pub async fn assignments(&self, cell: hextree::Cell) -> anyhow::Result { + pub fn assignments(&self, cell: hextree::Cell) -> anyhow::Result { HexAssignments::builder(cell) .footfall(&self.footfall) .landtype(&self.landtype) @@ -103,8 +103,8 @@ mod tests { use super::*; - #[tokio::test] - async fn test_hex_boost_data() -> anyhow::Result<()> { + #[test] + fn test_hex_boost_data() -> anyhow::Result<()> { // This test will break if any of the logic deriving Assignments from // the underlying DiskTreeMap's changes. @@ -268,41 +268,41 @@ mod tests { { use Assignment::*; // yellow - assert_eq!(HexAssignments { footfall: A, landtype: A, urbanized: A }, data.assignments(poi_built_urbanized).await?); - assert_eq!(HexAssignments { footfall: A, landtype: B, urbanized: A }, data.assignments(poi_grass_urbanized).await?); - assert_eq!(HexAssignments { footfall: A, landtype: C, urbanized: A }, data.assignments(poi_water_urbanized).await?); + assert_eq!(HexAssignments { footfall: A, landtype: A, urbanized: A }, data.assignments(poi_built_urbanized)?); + assert_eq!(HexAssignments { footfall: A, landtype: B, urbanized: A }, data.assignments(poi_grass_urbanized)?); + assert_eq!(HexAssignments { footfall: A, landtype: C, urbanized: A }, data.assignments(poi_water_urbanized)?); // orange - assert_eq!(HexAssignments { footfall: A, landtype: A, urbanized: B }, data.assignments(poi_built_not_urbanized).await?); - assert_eq!(HexAssignments { footfall: A, landtype: B, urbanized: B }, data.assignments(poi_grass_not_urbanized).await?); - assert_eq!(HexAssignments { footfall: A, landtype: C, urbanized: B }, data.assignments(poi_water_not_urbanized).await?); + assert_eq!(HexAssignments { footfall: A, landtype: A, urbanized: B }, data.assignments(poi_built_not_urbanized)?); + assert_eq!(HexAssignments { footfall: A, landtype: B, urbanized: B }, data.assignments(poi_grass_not_urbanized)?); + assert_eq!(HexAssignments { footfall: A, landtype: C, urbanized: B }, data.assignments(poi_water_not_urbanized)?); // light green - assert_eq!(HexAssignments { footfall: B, landtype: A, urbanized: A }, data.assignments(poi_no_data_built_urbanized).await?); - assert_eq!(HexAssignments { footfall: B, landtype: B, urbanized: A }, data.assignments(poi_no_data_grass_urbanized).await?); - assert_eq!(HexAssignments { footfall: B, landtype: C, urbanized: A }, data.assignments(poi_no_data_water_urbanized).await?); + assert_eq!(HexAssignments { footfall: B, landtype: A, urbanized: A }, data.assignments(poi_no_data_built_urbanized)?); + assert_eq!(HexAssignments { footfall: B, landtype: B, urbanized: A }, data.assignments(poi_no_data_grass_urbanized)?); + assert_eq!(HexAssignments { footfall: B, landtype: C, urbanized: A }, data.assignments(poi_no_data_water_urbanized)?); // green - assert_eq!(HexAssignments { footfall: B, landtype: A, urbanized: B }, data.assignments(poi_no_data_built_not_urbanized).await?); - assert_eq!(HexAssignments { footfall: B, landtype: B, urbanized: B }, data.assignments(poi_no_data_grass_not_urbanized).await?); - assert_eq!(HexAssignments { footfall: B, landtype: C, urbanized: B }, data.assignments(poi_no_data_water_not_urbanized).await?); + assert_eq!(HexAssignments { footfall: B, landtype: A, urbanized: B }, data.assignments(poi_no_data_built_not_urbanized)?); + assert_eq!(HexAssignments { footfall: B, landtype: B, urbanized: B }, data.assignments(poi_no_data_grass_not_urbanized)?); + assert_eq!(HexAssignments { footfall: B, landtype: C, urbanized: B }, data.assignments(poi_no_data_water_not_urbanized)?); // light blue - assert_eq!(HexAssignments { footfall: C, landtype: A, urbanized: A }, data.assignments(no_poi_built_urbanized).await?); - assert_eq!(HexAssignments { footfall: C, landtype: B, urbanized: A }, data.assignments(no_poi_grass_urbanized).await?); - assert_eq!(HexAssignments { footfall: C, landtype: C, urbanized: A }, data.assignments(no_poi_water_urbanized).await?); + assert_eq!(HexAssignments { footfall: C, landtype: A, urbanized: A }, data.assignments(no_poi_built_urbanized)?); + assert_eq!(HexAssignments { footfall: C, landtype: B, urbanized: A }, data.assignments(no_poi_grass_urbanized)?); + assert_eq!(HexAssignments { footfall: C, landtype: C, urbanized: A }, data.assignments(no_poi_water_urbanized)?); // dark blue - assert_eq!(HexAssignments { footfall: C, landtype: A, urbanized: B }, data.assignments(no_poi_built_not_urbanized).await?); - assert_eq!(HexAssignments { footfall: C, landtype: B, urbanized: B }, data.assignments(no_poi_grass_not_urbanized).await?); - assert_eq!(HexAssignments { footfall: C, landtype: C, urbanized: B }, data.assignments(no_poi_water_not_urbanized).await?); + assert_eq!(HexAssignments { footfall: C, landtype: A, urbanized: B }, data.assignments(no_poi_built_not_urbanized)?); + assert_eq!(HexAssignments { footfall: C, landtype: B, urbanized: B }, data.assignments(no_poi_grass_not_urbanized)?); + assert_eq!(HexAssignments { footfall: C, landtype: C, urbanized: B }, data.assignments(no_poi_water_not_urbanized)?); // gray - assert_eq!(HexAssignments { footfall: A, landtype: A, urbanized: C }, data.assignments(poi_built_outside_us).await?); - assert_eq!(HexAssignments { footfall: A, landtype: B, urbanized: C }, data.assignments(poi_grass_outside_us).await?); - assert_eq!(HexAssignments { footfall: A, landtype: C, urbanized: C }, data.assignments(poi_water_outside_us).await?); - assert_eq!(HexAssignments { footfall: B, landtype: A, urbanized: C }, data.assignments(poi_no_data_built_outside_us).await?); - assert_eq!(HexAssignments { footfall: B, landtype: B, urbanized: C }, data.assignments(poi_no_data_grass_outside_us).await?); - assert_eq!(HexAssignments { footfall: B, landtype: C, urbanized: C }, data.assignments(poi_no_data_water_outside_us).await?); - assert_eq!(HexAssignments { footfall: C, landtype: A, urbanized: C }, data.assignments(no_poi_built_outside_us).await?); - assert_eq!(HexAssignments { footfall: C, landtype: B, urbanized: C }, data.assignments(no_poi_grass_outside_us).await?); - assert_eq!(HexAssignments { footfall: C, landtype: C, urbanized: C }, data.assignments(no_poi_water_outside_us).await?); + assert_eq!(HexAssignments { footfall: A, landtype: A, urbanized: C }, data.assignments(poi_built_outside_us)?); + assert_eq!(HexAssignments { footfall: A, landtype: B, urbanized: C }, data.assignments(poi_grass_outside_us)?); + assert_eq!(HexAssignments { footfall: A, landtype: C, urbanized: C }, data.assignments(poi_water_outside_us)?); + assert_eq!(HexAssignments { footfall: B, landtype: A, urbanized: C }, data.assignments(poi_no_data_built_outside_us)?); + assert_eq!(HexAssignments { footfall: B, landtype: B, urbanized: C }, data.assignments(poi_no_data_grass_outside_us)?); + assert_eq!(HexAssignments { footfall: B, landtype: C, urbanized: C }, data.assignments(poi_no_data_water_outside_us)?); + assert_eq!(HexAssignments { footfall: C, landtype: A, urbanized: C }, data.assignments(no_poi_built_outside_us)?); + assert_eq!(HexAssignments { footfall: C, landtype: B, urbanized: C }, data.assignments(no_poi_grass_outside_us)?); + assert_eq!(HexAssignments { footfall: C, landtype: C, urbanized: C }, data.assignments(no_poi_water_outside_us)?); // never inserted - assert_eq!(HexAssignments { footfall: C, landtype: C, urbanized: C }, data.assignments(unknown_cell).await?); + assert_eq!(HexAssignments { footfall: C, landtype: C, urbanized: C }, data.assignments(unknown_cell)?); }; Ok(()) diff --git a/mobile_verifier/src/coverage.rs b/mobile_verifier/src/coverage.rs index 0b2c51c95..0a315279f 100644 --- a/mobile_verifier/src/coverage.rs +++ b/mobile_verifier/src/coverage.rs @@ -178,7 +178,7 @@ impl CoverageDaemon { transaction.commit().await?; // Tell the data set manager to update the assignments. - self.new_coverage_object_signal.try_send(())?; + let _ = self.new_coverage_object_signal.try_send(()); Ok(()) } From 9f381eee00f46c9a27ec8e5dd42f0703da0ba253 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Mon, 6 May 2024 21:07:55 -0400 Subject: [PATCH 34/53] Move first fill to inside is_ready check --- .../src/boosting_oracles/data_sets.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index 3a3eee302..f5c6c14c3 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -321,15 +321,17 @@ where // Attempt to fill in any unassigned hexes. This is for the edge case in // which we shutdown before a coverage object updates. - let boosting_reports = set_oracle_boosting_assignments( - UnassignedHex::fetch_unassigned(&self.pool), - &self.data_sets, - &self.pool, - ) - .await?; - self.oracle_boosting_sink - .write_all(boosting_reports) + if self.data_sets.is_ready() { + let boosting_reports = set_oracle_boosting_assignments( + UnassignedHex::fetch_unassigned(&self.pool), + &self.data_sets, + &self.pool, + ) .await?; + self.oracle_boosting_sink + .write_all(boosting_reports) + .await?; + } loop { #[rustfmt::skip] From 94ce2a2bcdac43fd7565c0af8eee36361691b1d3 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Mon, 6 May 2024 21:08:32 -0400 Subject: [PATCH 35/53] Move second in as well --- .../src/boosting_oracles/data_sets.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index f5c6c14c3..80ab2b931 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -339,15 +339,17 @@ where _ = self.new_coverage_object_signal.recv() => { // If we see a new coverage object, we want to assign only those hexes // that don't have an assignment - let boosting_reports = set_oracle_boosting_assignments( - UnassignedHex::fetch_unassigned(&self.pool), - &self.data_sets, - &self.pool, - ) - .await?; - self.oracle_boosting_sink - .write_all(boosting_reports) + if self.data_sets.is_ready() { + let boosting_reports = set_oracle_boosting_assignments( + UnassignedHex::fetch_unassigned(&self.pool), + &self.data_sets, + &self.pool, + ) .await?; + self.oracle_boosting_sink + .write_all(boosting_reports) + .await?; + } }, _ = tokio::time::sleep(poll_duration.to_std()?) => { self.check_for_new_data_sets().await?; From 65de9abf49fefa0559f7217f5313b48bfef1c78f Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Tue, 7 May 2024 16:59:18 -0400 Subject: [PATCH 36/53] Change check_for_unprocessed_data_sets query --- .../src/boosting_oracles/data_sets.rs | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index 80ab2b931..a9b080116 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -520,19 +520,30 @@ pub mod db { pool: &PgPool, period_end: DateTime, ) -> sqlx::Result { - Ok( - sqlx::query_scalar( - "SELECT COUNT(*) > 0 FROM data_sets WHERE time_to_use <= $1 AND status != 'processed'", - ) - .bind(period_end) - .fetch_one(pool) - .await? + Ok(sqlx::query_scalar( + "SELECT COUNT(*) > 0 FROM data_sets WHERE time_to_use <= $1 AND status != 'processed'", + ) + .bind(period_end) + .fetch_one(pool) + .await? || sqlx::query_scalar( - "SELECT COUNT(*) > 0 from hexes where urbanized IS NULL OR footfall IS NULL OR landtype IS NULL" + r#" + SELECT COUNT(*) > 0 FROM coverage_objects + WHERE inserted_at < $1 AND uuid IN ( + SELECT + DISTINCT uuid + FROM + hexes + WHERE + urbanized IS NULL + OR footfall IS NULL + OR landtype IS NULL + ) + "#, ) + .bind(period_end) .fetch_one(pool) - .await? - ) + .await?) } } From 7938cdf2b6ccd83bee4dfc202c373dbab454811b Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Wed, 8 May 2024 23:24:06 -0400 Subject: [PATCH 37/53] Fix up a bit --- Cargo.lock | 1 + mobile_verifier/Cargo.toml | 1 + .../src/boosting_oracles/data_sets.rs | 25 ++++++++++++++----- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1a8cf1e69..98df54329 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4686,6 +4686,7 @@ dependencies = [ "price", "prost", "rand 0.8.5", + "regex", "retainer", "reward-scheduler", "rust_decimal", diff --git a/mobile_verifier/Cargo.toml b/mobile_verifier/Cargo.toml index 415682f93..2bc69762b 100644 --- a/mobile_verifier/Cargo.toml +++ b/mobile_verifier/Cargo.toml @@ -54,6 +54,7 @@ uuid = {workspace = true} task-manager = {path = "../task_manager"} solana-sdk = {workspace = true} derive_builder = {workspace = true} +regex = "1" [dev-dependencies] backon = "0" diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index a9b080116..0f38893e2 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -8,12 +8,14 @@ use chrono::{DateTime, Duration, Utc}; use file_store::{ file_sink::{self, FileSinkClient}, file_upload::FileUpload, - traits::TimestampEncode, - FileInfo, FileStore, FileType, + traits::{TimestampDecode, TimestampEncode}, + FileStore, FileType, }; use futures_util::{Stream, StreamExt, TryFutureExt, TryStreamExt}; use helium_proto::services::poc_mobile as proto; use hextree::disktree::DiskTreeMap; +use lazy_static::lazy_static; +use regex::Regex; use rust_decimal::prelude::ToPrimitive; use rust_decimal_macros::dec; use sqlx::{FromRow, PgPool, QueryBuilder}; @@ -183,7 +185,7 @@ impl file_upload.clone(), concat!(env!("CARGO_PKG_NAME"), "_oracle_boosting_report"), ) - .auto_commit(false) + .auto_commit(true) .roll_time(Duration::minutes(15)) .create() .await?; @@ -374,6 +376,10 @@ fn get_data_set_path( dir } +lazy_static! { + static ref RE: Regex = Regex::new(r"([a-z,_]+).(\d+)(.res10.h3tree)?").unwrap(); +} + async fn delete_old_data_sets( data_set_directory: &Path, data_set_type: DataSetType, @@ -381,9 +387,16 @@ async fn delete_old_data_sets( ) -> anyhow::Result<()> { let mut data_sets = tokio::fs::read_dir(data_set_directory).await?; while let Some(data_set) = data_sets.next_entry().await? { - let file_info: FileInfo = data_set.file_name().to_string_lossy().parse()?; - if file_info.prefix == data_set_type.to_prefix() && file_info.timestamp < time_to_use { - tracing::info!(data_set = file_info.key, "Deleting old data set file"); + let file_name = data_set.file_name(); + let file_name = file_name.to_string_lossy(); + let Some(cap) = RE.captures(&*file_name) else { + tracing::warn!("Could not determine data set file type: {}", file_name); + continue; + }; + let prefix = &cap[1]; + let timestamp = cap[2].parse::()?.to_timestamp_millis()?; + if prefix == data_set_type.to_prefix() && timestamp < time_to_use { + tracing::info!(data_set = &*file_name, "Deleting old data set file"); tokio::fs::remove_file(data_set.file_name()).await?; } } From 1367c27cbb0e89d58406446561848b0aeab87908 Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Tue, 14 May 2024 13:55:01 -0400 Subject: [PATCH 38/53] Make clippy happy --- mobile_verifier/src/boosting_oracles/data_sets.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index 0f38893e2..c2b119f14 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -389,7 +389,7 @@ async fn delete_old_data_sets( while let Some(data_set) = data_sets.next_entry().await? { let file_name = data_set.file_name(); let file_name = file_name.to_string_lossy(); - let Some(cap) = RE.captures(&*file_name) else { + let Some(cap) = RE.captures(&file_name) else { tracing::warn!("Could not determine data set file type: {}", file_name); continue; }; From 9cc4cc6ff7e6c620e7cfd5c5eef03bc0c1ffd2cb Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Tue, 14 May 2024 14:56:00 -0400 Subject: [PATCH 39/53] Fix downloading --- mobile_verifier/src/boosting_oracles/data_sets.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index c2b119f14..fee9ec8db 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -306,7 +306,7 @@ where } pub async fn run(mut self) -> anyhow::Result<()> { - let poll_duration = Duration::minutes(5); + let poll_duration = Duration::minutes(1); self.data_sets .urbanization @@ -369,7 +369,7 @@ fn get_data_set_path( let path = PathBuf::from(format!( "{}.{}.res10.h3tree", data_set_type.to_prefix(), - time_to_use.timestamp() + time_to_use.timestamp_millis() )); let mut dir = data_set_directory.to_path_buf(); dir.push(path); @@ -396,8 +396,10 @@ async fn delete_old_data_sets( let prefix = &cap[1]; let timestamp = cap[2].parse::()?.to_timestamp_millis()?; if prefix == data_set_type.to_prefix() && timestamp < time_to_use { + let mut path = data_set_directory.to_path_buf(); + path.push(data_set.path()); tracing::info!(data_set = &*file_name, "Deleting old data set file"); - tokio::fs::remove_file(data_set.file_name()).await?; + tokio::fs::remove_file(path).await?; } } Ok(()) From 7e447ab09c891a1d9df83e4e25eb5a114850d0c1 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Tue, 14 May 2024 16:01:14 -0400 Subject: [PATCH 40/53] Switch to not exists and fix path deletion --- mobile_verifier/migrations/33_data_sets.sql | 6 +++--- mobile_verifier/src/boosting_oracles/data_sets.rs | 4 +--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/mobile_verifier/migrations/33_data_sets.sql b/mobile_verifier/migrations/33_data_sets.sql index 01f44d668..043be74a4 100644 --- a/mobile_verifier/migrations/33_data_sets.sql +++ b/mobile_verifier/migrations/33_data_sets.sql @@ -1,16 +1,16 @@ -CREATE TYPE data_set_status AS enum ( +CREATE TYPE IF NOT EXISTS data_set_status AS enum ( 'pending', 'downloaded', 'processed' ); -CREATE TYPE data_set_type AS enum ( +CREATE TYPE IF NOT EXISTS data_set_type AS enum ( 'urbanization', 'footfall', 'landtype' ); -CREATE TABLE data_sets ( +CREATE TABLE IF NOT EXISTS data_sets ( filename TEXT PRIMARY KEY, data_set data_set_type NOT NULL, time_to_use TIMESTAMPTZ NOT NULL, diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index fee9ec8db..dc4be1b33 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -396,10 +396,8 @@ async fn delete_old_data_sets( let prefix = &cap[1]; let timestamp = cap[2].parse::()?.to_timestamp_millis()?; if prefix == data_set_type.to_prefix() && timestamp < time_to_use { - let mut path = data_set_directory.to_path_buf(); - path.push(data_set.path()); tracing::info!(data_set = &*file_name, "Deleting old data set file"); - tokio::fs::remove_file(path).await?; + tokio::fs::remove_file(data_set.path()).await?; } } Ok(()) From bf83fb46e045a6fd094d1e5ad24c7c0aa9d21089 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Wed, 15 May 2024 10:26:21 -0400 Subject: [PATCH 41/53] Fix migration --- mobile_verifier/migrations/33_data_sets.sql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mobile_verifier/migrations/33_data_sets.sql b/mobile_verifier/migrations/33_data_sets.sql index 043be74a4..d76d99a37 100644 --- a/mobile_verifier/migrations/33_data_sets.sql +++ b/mobile_verifier/migrations/33_data_sets.sql @@ -1,10 +1,12 @@ -CREATE TYPE IF NOT EXISTS data_set_status AS enum ( +DROP TYPE IF EXISTS data_set_status; +CREATE TYPE data_set_status AS enum ( 'pending', 'downloaded', 'processed' ); -CREATE TYPE IF NOT EXISTS data_set_type AS enum ( +DROP TYPE IF EXISTS data_set_type; +CREATE TYPE data_set_type AS enum ( 'urbanization', 'footfall', 'landtype' From 198a838e54f7eedcb37fc5c73f17133373d0dd59 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Wed, 15 May 2024 15:56:54 -0400 Subject: [PATCH 42/53] Move things around --- .../src/boosting_oracles/data_sets.rs | 272 +++++++++++------- 1 file changed, 166 insertions(+), 106 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index dc4be1b33..c3074b9c5 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -262,15 +262,12 @@ where new_urbanized.is_some() || new_footfall.is_some() || new_landtype.is_some(); if self.data_sets.is_ready() && new_data_set { tracing::info!("Processing new data sets"); - let boosting_reports = set_oracle_boosting_assignments( - UnassignedHex::fetch_all(&self.pool), - &self.data_sets, + set_all_oracle_boosting_assignments( &self.pool, + &self.data_sets, + &self.oracle_boosting_sink, ) .await?; - self.oracle_boosting_sink - .write_all(boosting_reports) - .await?; } // Mark the new data sets as processed and delete the old ones @@ -324,15 +321,12 @@ where // Attempt to fill in any unassigned hexes. This is for the edge case in // which we shutdown before a coverage object updates. if self.data_sets.is_ready() { - let boosting_reports = set_oracle_boosting_assignments( - UnassignedHex::fetch_unassigned(&self.pool), - &self.data_sets, + set_unassigned_oracle_boosting_assignments( &self.pool, + &self.data_sets, + &self.oracle_boosting_sink, ) .await?; - self.oracle_boosting_sink - .write_all(boosting_reports) - .await?; } loop { @@ -342,15 +336,11 @@ where // If we see a new coverage object, we want to assign only those hexes // that don't have an assignment if self.data_sets.is_ready() { - let boosting_reports = set_oracle_boosting_assignments( - UnassignedHex::fetch_unassigned(&self.pool), - &self.data_sets, + set_unassigned_oracle_boosting_assignments( &self.pool, - ) - .await?; - self.oracle_boosting_sink - .write_all(boosting_reports) - .await?; + &self.data_sets, + &self.oracle_boosting_sink + ).await?; } }, _ = tokio::time::sleep(poll_duration.to_std()?) => { @@ -560,23 +550,42 @@ pub mod db { } } -#[derive(FromRow)] -pub struct UnassignedHex { - uuid: uuid::Uuid, - #[sqlx(try_from = "i64")] - hex: u64, - signal_level: SignalLevel, - signal_power: i32, +pub struct AssignedCoverageObjects { + coverage_objs: HashMap>, } -impl UnassignedHex { - pub fn fetch_all(pool: &PgPool) -> impl Stream> + '_ { - sqlx::query_as("SELECT uuid, hex, signal_level, signal_power FROM hexes").fetch(pool) +impl AssignedCoverageObjects { + async fn from_stream( + stream: impl Stream>, + data_sets: &HexBoostData, + ) -> anyhow::Result { + let mut coverage_objs = HashMap::>::new(); + let mut stream = pin!(stream); + while let Some(hex) = stream.try_next().await? { + let hex = hex.assign(data_sets)?; + coverage_objs.entry(hex.uuid).or_default().push(hex); + } + Ok(Self { coverage_objs }) } - pub fn fetch_unassigned(pool: &PgPool) -> impl Stream> + '_ { - sqlx::query_as( - "SELECT + async fn fetch_all( + pool: &PgPool, + data_sets: &HexBoostData, + ) -> anyhow::Result { + Self::from_stream( + sqlx::query_as("SELECT uuid, hex, signal_level, signal_power FROM hexes").fetch(pool), + data_sets, + ) + .await + } + + async fn fetch_unassigned( + pool: &PgPool, + data_sets: &HexBoostData, + ) -> anyhow::Result { + Self::from_stream( + sqlx::query_as( + "SELECT uuid, hex, signal_level, signal_power FROM hexes @@ -584,89 +593,140 @@ impl UnassignedHex { urbanized IS NULL OR footfall IS NULL OR landtype IS NULL", + ) + .fetch(pool), + data_sets, ) - .fetch(pool) + .await } -} -pub async fn set_oracle_boosting_assignments<'a>( - unassigned_hexes: impl Stream>, - data_sets: &HexBoostData, - pool: &'a PgPool, -) -> anyhow::Result> { - const NUMBER_OF_FIELDS_IN_QUERY: u16 = 7; - const ASSIGNMENTS_MAX_BATCH_ENTRIES: usize = (u16::MAX / NUMBER_OF_FIELDS_IN_QUERY) as usize; - - let now = Utc::now(); - let mut boost_results = HashMap::>::new(); - let mut unassigned_hexes = pin!(unassigned_hexes.try_chunks(ASSIGNMENTS_MAX_BATCH_ENTRIES)); - - while let Some(hexes) = unassigned_hexes.try_next().await? { - let hexes: anyhow::Result> = hexes - .into_iter() - .map(|hex| { - let cell = hextree::Cell::try_from(hex.hex)?; - let assignments = HexAssignments::builder(cell) - .footfall(&data_sets.footfall) - .landtype(&data_sets.landtype) - .urbanized(&data_sets.urbanization) - .build()?; - let location = format!("{:x}", hex.hex); - let assignment_multiplier = (assignments.boosting_multiplier() * dec!(1000)) + async fn write(&self, boosting_reports: &FileSinkClient) -> file_store::Result { + let timestamp = Utc::now().encode_timestamp(); + for (uuid, hexes) in self.coverage_objs.iter() { + let assignments: Vec<_> = hexes + .iter() + .map(|hex| { + let location = format!("{:x}", hex.hex); + let assignment_multiplier = (hex.assignments.boosting_multiplier() + * dec!(1000)) .to_u32() .unwrap_or(0); - - boost_results.entry(hex.uuid).or_default().push( proto::OracleBoostingHexAssignment { location, - urbanized: assignments.urbanized.into(), - footfall: assignments.footfall.into(), - landtype: assignments.landtype.into(), + urbanized: hex.assignments.urbanized.into(), + footfall: hex.assignments.footfall.into(), + landtype: hex.assignments.landtype.into(), assignment_multiplier, + } + }) + .collect(); + boosting_reports + .write( + proto::OracleBoostingReportV1 { + coverage_object: Vec::from(uuid.into_bytes()), + assignments, + timestamp, }, - ); - - Ok(( - hex, - assignments.footfall, - assignments.landtype, - assignments.urbanized, - )) - }) - .collect(); - - QueryBuilder::new( - "INSERT INTO hexes (uuid, hex, signal_level, signal_power, footfall, landtype, urbanized)", - ) - .push_values(hexes?, |mut b, (hex, footfall, landtype, urbanized)| { - b.push_bind(hex.uuid) - .push_bind(hex.hex as i64) - .push_bind(hex.signal_level) - .push_bind(hex.signal_power) - .push_bind(footfall) - .push_bind(landtype) - .push_bind(urbanized); + &[], + ) + .await?; + } + + Ok(()) + } + + async fn save(self, pool: &PgPool) -> anyhow::Result<()> { + const NUMBER_OF_FIELDS_IN_QUERY: u16 = 7; + const ASSIGNMENTS_MAX_BATCH_ENTRIES: usize = + (u16::MAX / NUMBER_OF_FIELDS_IN_QUERY) as usize; + + let assigned_hexes: Vec<_> = self.coverage_objs.into_values().flatten().collect(); + for assigned_hexes in assigned_hexes.chunks(ASSIGNMENTS_MAX_BATCH_ENTRIES) { + QueryBuilder::new( + "INSERT INTO hexes (uuid, hex, signal_level, signal_power, footfall, landtype, urbanized)", + ) + .push_values(assigned_hexes, |mut b, hex| { + b.push_bind(hex.uuid) + .push_bind(hex.hex as i64) + .push_bind(hex.signal_level) + .push_bind(hex.signal_power) + .push_bind(hex.assignments.footfall) + .push_bind(hex.assignments.landtype) + .push_bind(hex.assignments.urbanized); + }) + .push( + r#" + ON CONFLICT (uuid, hex) DO UPDATE SET + footfall = EXCLUDED.footfall, + landtype = EXCLUDED.landtype, + urbanized = EXCLUDED.urbanized + "#, + ) + .build() + .execute(pool) + .await?; + } + + Ok(()) + } +} + +#[derive(FromRow)] +pub struct UnassignedHex { + uuid: uuid::Uuid, + #[sqlx(try_from = "i64")] + hex: u64, + signal_level: SignalLevel, + signal_power: i32, +} + +impl UnassignedHex { + fn assign( + self, + data_sets: &HexBoostData, + ) -> anyhow::Result { + let cell = hextree::Cell::try_from(self.hex)?; + let assignments = HexAssignments::builder(cell) + .footfall(&data_sets.footfall) + .landtype(&data_sets.landtype) + .urbanized(&data_sets.urbanization) + .build()?; + Ok(AssignedHex { + uuid: self.uuid, + hex: self.hex, + signal_level: self.signal_level, + signal_power: self.signal_power, + assignments, }) - .push( - r#" - ON CONFLICT (uuid, hex) DO UPDATE SET - footfall = EXCLUDED.footfall, - landtype = EXCLUDED.landtype, - urbanized = EXCLUDED.urbanized - "#, - ) - .build() - .execute(pool) - .await?; } +} + +pub struct AssignedHex { + uuid: uuid::Uuid, + hex: u64, + signal_level: SignalLevel, + signal_power: i32, + assignments: HexAssignments, +} + +pub async fn set_all_oracle_boosting_assignments( + pool: &PgPool, + data_sets: &HexBoostData, + file_sink: &FileSinkClient, +) -> anyhow::Result<()> { + let assigned_coverage_objs = AssignedCoverageObjects::fetch_all(pool, data_sets).await?; + assigned_coverage_objs.write(file_sink).await?; + assigned_coverage_objs.save(pool).await?; + Ok(()) +} - Ok(boost_results - .into_iter() - .map( - move |(coverage_object, assignments)| proto::OracleBoostingReportV1 { - coverage_object: Vec::from(coverage_object.into_bytes()), - assignments, - timestamp: now.encode_timestamp(), - }, - )) +pub async fn set_unassigned_oracle_boosting_assignments( + pool: &PgPool, + data_sets: &HexBoostData, + file_sink: &FileSinkClient, +) -> anyhow::Result<()> { + let assigned_coverage_objs = AssignedCoverageObjects::fetch_unassigned(pool, data_sets).await?; + assigned_coverage_objs.write(file_sink).await?; + assigned_coverage_objs.save(pool).await?; + Ok(()) } From f46c818b4087202a682dd7eb241bc824d277ea97 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Thu, 16 May 2024 11:23:46 -0400 Subject: [PATCH 43/53] Fix up tests, address comments --- .../src/boosting_oracles/data_sets.rs | 92 +++++++++---------- .../src/boosting_oracles/footfall.rs | 19 ++-- .../src/boosting_oracles/landtype.rs | 19 ++-- mobile_verifier/src/boosting_oracles/mod.rs | 31 +------ .../src/boosting_oracles/urbanization.rs | 19 ++-- .../tests/integrations/boosting_oracles.rs | 12 +-- .../tests/integrations/common/mod.rs | 55 ++++++++++- .../tests/integrations/hex_boosting.rs | 7 +- .../tests/integrations/modeled_coverage.rs | 10 +- .../tests/integrations/rewarder_poc_dc.rs | 7 +- 10 files changed, 132 insertions(+), 139 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index c3074b9c5..7ef92a371 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -13,7 +13,6 @@ use file_store::{ }; use futures_util::{Stream, StreamExt, TryFutureExt, TryStreamExt}; use helium_proto::services::poc_mobile as proto; -use hextree::disktree::DiskTreeMap; use lazy_static::lazy_static; use regex::Regex; use rust_decimal::prelude::ToPrimitive; @@ -24,7 +23,9 @@ use tokio::{fs::File, io::AsyncWriteExt, sync::mpsc::Receiver}; use crate::{boosting_oracles::assignment::HexAssignments, coverage::SignalLevel, Settings}; -use super::{footfall::Footfall, landtype::Landtype, HexAssignment, HexBoostData, Urbanization}; +use super::{ + footfall::Footfall, landtype::Landtype, urbanization::Urbanization, HexAssignment, HexBoostData, +}; #[async_trait::async_trait] pub trait DataSet: HexAssignment + Send + Sync + 'static { @@ -169,9 +170,7 @@ where } } -impl - DataSetDownloaderDaemon, Landtype, Urbanization> -{ +impl DataSetDownloaderDaemon { pub async fn create_managed_task( pool: PgPool, settings: &Settings, @@ -190,9 +189,9 @@ impl .create() .await?; - let urbanization: Urbanization = Urbanization::new(); - let footfall: Footfall = Footfall::new(); - let landtype: Landtype = Landtype::new(); + let urbanization = Urbanization::new(); + let footfall = Footfall::new(); + let landtype = Landtype::new(); let hex_boost_data = HexBoostData::builder() .footfall(footfall) .landtype(landtype) @@ -548,14 +547,34 @@ pub mod db { .fetch_one(pool) .await?) } + + pub fn fetch_all_hexes(pool: &PgPool) -> impl Stream> + '_ { + sqlx::query_as("SELECT uuid, hex, signal_level, signal_power FROM hexes").fetch(pool) + } + + pub fn fetch_hexes_with_null_assignments( + pool: &PgPool, + ) -> impl Stream> + '_ { + sqlx::query_as( + "SELECT + uuid, hex, signal_level, signal_power + FROM + hexes + WHERE + urbanized IS NULL + OR footfall IS NULL + OR landtype IS NULL", + ) + .fetch(pool) + } } pub struct AssignedCoverageObjects { - coverage_objs: HashMap>, + pub coverage_objs: HashMap>, } impl AssignedCoverageObjects { - async fn from_stream( + pub async fn assign_hex_stream( stream: impl Stream>, data_sets: &HexBoostData, ) -> anyhow::Result { @@ -568,38 +587,6 @@ impl AssignedCoverageObjects { Ok(Self { coverage_objs }) } - async fn fetch_all( - pool: &PgPool, - data_sets: &HexBoostData, - ) -> anyhow::Result { - Self::from_stream( - sqlx::query_as("SELECT uuid, hex, signal_level, signal_power FROM hexes").fetch(pool), - data_sets, - ) - .await - } - - async fn fetch_unassigned( - pool: &PgPool, - data_sets: &HexBoostData, - ) -> anyhow::Result { - Self::from_stream( - sqlx::query_as( - "SELECT - uuid, hex, signal_level, signal_power - FROM - hexes - WHERE - urbanized IS NULL - OR footfall IS NULL - OR landtype IS NULL", - ) - .fetch(pool), - data_sets, - ) - .await - } - async fn write(&self, boosting_reports: &FileSinkClient) -> file_store::Result { let timestamp = Utc::now().encode_timestamp(); for (uuid, hexes) in self.coverage_objs.iter() { @@ -635,7 +622,7 @@ impl AssignedCoverageObjects { Ok(()) } - async fn save(self, pool: &PgPool) -> anyhow::Result<()> { + pub async fn save(self, pool: &PgPool) -> anyhow::Result<()> { const NUMBER_OF_FIELDS_IN_QUERY: u16 = 7; const ASSIGNMENTS_MAX_BATCH_ENTRIES: usize = (u16::MAX / NUMBER_OF_FIELDS_IN_QUERY) as usize; @@ -702,11 +689,11 @@ impl UnassignedHex { } pub struct AssignedHex { - uuid: uuid::Uuid, - hex: u64, - signal_level: SignalLevel, - signal_power: i32, - assignments: HexAssignments, + pub uuid: uuid::Uuid, + pub hex: u64, + pub signal_level: SignalLevel, + pub signal_power: i32, + pub assignments: HexAssignments, } pub async fn set_all_oracle_boosting_assignments( @@ -714,7 +701,8 @@ pub async fn set_all_oracle_boosting_assignments( data_sets: &HexBoostData, file_sink: &FileSinkClient, ) -> anyhow::Result<()> { - let assigned_coverage_objs = AssignedCoverageObjects::fetch_all(pool, data_sets).await?; + let assigned_coverage_objs = + AssignedCoverageObjects::assign_hex_stream(db::fetch_all_hexes(pool), data_sets).await?; assigned_coverage_objs.write(file_sink).await?; assigned_coverage_objs.save(pool).await?; Ok(()) @@ -725,7 +713,11 @@ pub async fn set_unassigned_oracle_boosting_assignments( data_sets: &HexBoostData, file_sink: &FileSinkClient, ) -> anyhow::Result<()> { - let assigned_coverage_objs = AssignedCoverageObjects::fetch_unassigned(pool, data_sets).await?; + let assigned_coverage_objs = AssignedCoverageObjects::assign_hex_stream( + db::fetch_hexes_with_null_assignments(pool), + data_sets, + ) + .await?; assigned_coverage_objs.write(file_sink).await?; assigned_coverage_objs.save(pool).await?; Ok(()) diff --git a/mobile_verifier/src/boosting_oracles/footfall.rs b/mobile_verifier/src/boosting_oracles/footfall.rs index 1b134bca7..6f9db1e51 100644 --- a/mobile_verifier/src/boosting_oracles/footfall.rs +++ b/mobile_verifier/src/boosting_oracles/footfall.rs @@ -3,14 +3,14 @@ use std::path::Path; use chrono::{DateTime, Utc}; use hextree::disktree::DiskTreeMap; -use super::{Assignment, DataSet, DataSetType, DiskTreeLike, HexAssignment}; +use super::{Assignment, DataSet, DataSetType, HexAssignment}; -pub struct Footfall { - footfall: Option, +pub struct Footfall { + footfall: Option, timestamp: Option>, } -impl Footfall { +impl Footfall { pub fn new() -> Self { Self { footfall: None, @@ -18,7 +18,7 @@ impl Footfall { } } - pub fn new_mock(footfall: F) -> Self { + pub fn new_mock(footfall: DiskTreeMap) -> Self { Self { footfall: Some(footfall), timestamp: None, @@ -26,14 +26,14 @@ impl Footfall { } } -impl Default for Footfall { +impl Default for Footfall { fn default() -> Self { Self::new() } } #[async_trait::async_trait] -impl DataSet for Footfall { +impl DataSet for Footfall { const TYPE: DataSetType = DataSetType::Footfall; fn timestamp(&self) -> Option> { @@ -51,10 +51,7 @@ impl DataSet for Footfall { } } -impl HexAssignment for Footfall -where - Foot: DiskTreeLike + Send + Sync + 'static, -{ +impl HexAssignment for Footfall { fn assignment(&self, cell: hextree::Cell) -> anyhow::Result { let Some(ref footfall) = self.footfall else { anyhow::bail!("No footfall data set has been loaded"); diff --git a/mobile_verifier/src/boosting_oracles/landtype.rs b/mobile_verifier/src/boosting_oracles/landtype.rs index 6493bde3c..b6cda7aef 100644 --- a/mobile_verifier/src/boosting_oracles/landtype.rs +++ b/mobile_verifier/src/boosting_oracles/landtype.rs @@ -3,14 +3,14 @@ use std::path::Path; use chrono::{DateTime, Utc}; use hextree::disktree::DiskTreeMap; -use super::{Assignment, DataSet, DataSetType, DiskTreeLike, HexAssignment}; +use super::{Assignment, DataSet, DataSetType, HexAssignment}; -pub struct Landtype { - landtype: Option, +pub struct Landtype { + landtype: Option, timestamp: Option>, } -impl Landtype { +impl Landtype { pub fn new() -> Self { Self { landtype: None, @@ -18,7 +18,7 @@ impl Landtype { } } - pub fn new_mock(landtype: L) -> Self { + pub fn new_mock(landtype: DiskTreeMap) -> Self { Self { landtype: Some(landtype), timestamp: None, @@ -26,14 +26,14 @@ impl Landtype { } } -impl Default for Landtype { +impl Default for Landtype { fn default() -> Self { Self::new() } } #[async_trait::async_trait] -impl DataSet for Landtype { +impl DataSet for Landtype { const TYPE: DataSetType = DataSetType::Landtype; fn timestamp(&self) -> Option> { @@ -131,10 +131,7 @@ impl From for Assignment { } } -impl HexAssignment for Landtype -where - Land: DiskTreeLike + Send + Sync + 'static, -{ +impl HexAssignment for Landtype { fn assignment(&self, cell: hextree::Cell) -> anyhow::Result { let Some(ref landtype) = self.landtype else { anyhow::bail!("No landtype data set has been loaded"); diff --git a/mobile_verifier/src/boosting_oracles/mod.rs b/mobile_verifier/src/boosting_oracles/mod.rs index 773c86cf7..0f63845cb 100644 --- a/mobile_verifier/src/boosting_oracles/mod.rs +++ b/mobile_verifier/src/boosting_oracles/mod.rs @@ -10,9 +10,6 @@ use crate::boosting_oracles::assignment::HexAssignments; pub use assignment::Assignment; pub use data_sets::*; -use hextree::disktree::DiskTreeMap; -pub use urbanization::Urbanization; - pub trait HexAssignment: Send + Sync + 'static { fn assignment(&self, cell: hextree::Cell) -> anyhow::Result; } @@ -68,38 +65,14 @@ where } } -trait DiskTreeLike: Send + Sync { - fn get(&self, cell: hextree::Cell) -> hextree::Result>; -} - -impl DiskTreeLike for DiskTreeMap { - fn get(&self, cell: hextree::Cell) -> hextree::Result> { - self.get(cell) - } -} - -impl DiskTreeLike for std::collections::HashSet { - fn get(&self, cell: hextree::Cell) -> hextree::Result> { - Ok(self.contains(&cell).then_some((cell, &[]))) - } -} - -pub struct MockDiskTree; - -impl DiskTreeLike for MockDiskTree { - fn get(&self, cell: hextree::Cell) -> hextree::Result> { - Ok(Some((cell, &[]))) - } -} - #[cfg(test)] mod tests { use std::io::Cursor; - use hextree::HexTreeMap; + use hextree::{disktree::DiskTreeMap, HexTreeMap}; - use self::{footfall::Footfall, landtype::Landtype}; + use self::{footfall::Footfall, landtype::Landtype, urbanization::Urbanization}; use super::*; diff --git a/mobile_verifier/src/boosting_oracles/urbanization.rs b/mobile_verifier/src/boosting_oracles/urbanization.rs index 506598943..8070c445c 100644 --- a/mobile_verifier/src/boosting_oracles/urbanization.rs +++ b/mobile_verifier/src/boosting_oracles/urbanization.rs @@ -3,14 +3,14 @@ use std::path::Path; use chrono::{DateTime, Utc}; use hextree::disktree::DiskTreeMap; -use super::{Assignment, DataSet, DataSetType, DiskTreeLike, HexAssignment}; +use super::{Assignment, DataSet, DataSetType, HexAssignment}; -pub struct Urbanization
{ - urbanized: Option
, +pub struct Urbanization { + urbanized: Option, timestamp: Option>, } -impl
Urbanization
{ +impl Urbanization { pub fn new() -> Self { Self { urbanized: None, @@ -18,7 +18,7 @@ impl
Urbanization
{ } } - pub fn new_mock(urbanized: DT) -> Self { + pub fn new_mock(urbanized: DiskTreeMap) -> Self { Self { urbanized: Some(urbanized), timestamp: None, @@ -26,14 +26,14 @@ impl
Urbanization
{ } } -impl
Default for Urbanization
{ +impl Default for Urbanization { fn default() -> Self { Self::new() } } #[async_trait::async_trait] -impl DataSet for Urbanization { +impl DataSet for Urbanization { const TYPE: DataSetType = DataSetType::Urbanization; fn timestamp(&self) -> Option> { @@ -51,10 +51,7 @@ impl DataSet for Urbanization { } } -impl HexAssignment for Urbanization -where - Urban: DiskTreeLike + Send + Sync + 'static, -{ +impl HexAssignment for Urbanization { fn assignment(&self, cell: hextree::Cell) -> anyhow::Result { let Some(ref urbanized) = self.urbanized else { anyhow::bail!("No urbanization data set has been loaded"); diff --git a/mobile_verifier/tests/integrations/boosting_oracles.rs b/mobile_verifier/tests/integrations/boosting_oracles.rs index bc0126fa6..880407c37 100644 --- a/mobile_verifier/tests/integrations/boosting_oracles.rs +++ b/mobile_verifier/tests/integrations/boosting_oracles.rs @@ -13,7 +13,7 @@ use helium_proto::services::poc_mobile::{ }; use mobile_config::boosted_hex_info::BoostedHexes; use mobile_verifier::{ - boosting_oracles::{set_oracle_boosting_assignments, Assignment, HexBoostData, UnassignedHex}, + boosting_oracles::{Assignment, HexBoostData}, coverage::{CoverageClaimTimeCache, CoverageObject, CoverageObjectCache, Seniority}, geofence::GeofenceValidator, heartbeats::{Heartbeat, HeartbeatReward, LocationCache, SeniorityUpdate, ValidatedHeartbeat}, @@ -29,6 +29,8 @@ use sqlx::PgPool; use std::{collections::HashMap, pin::pin}; use uuid::Uuid; +use crate::common; + #[derive(Clone)] struct MockGeofence; @@ -209,15 +211,12 @@ async fn test_footfall_and_urbanization_report(pool: PgPool) -> anyhow::Result<( .await?; transaction.commit().await?; - let unassigned_hexes = UnassignedHex::fetch_unassigned(&pool); let hex_boost_data = HexBoostData::builder() .footfall(footfall) .landtype(landtype) .urbanization(urbanized) .build()?; - let oba = set_oracle_boosting_assignments(unassigned_hexes, &hex_boost_data, &pool) - .await? - .collect::>(); + let oba = common::set_unassigned_oracle_boosting_assignments(&pool, &hex_boost_data).await?; assert_eq!(oba.len(), 1); assert_eq!(oba[0].assignments, hexes); @@ -340,13 +339,12 @@ async fn test_footfall_and_urbanization_and_landtype(pool: PgPool) -> anyhow::Re .await?; transaction.commit().await?; - let unassigned_hexes = UnassignedHex::fetch_unassigned(&pool); let hex_boost_data = HexBoostData::builder() .footfall(footfall) .landtype(landtype) .urbanization(urbanized) .build()?; - let _ = set_oracle_boosting_assignments(unassigned_hexes, &hex_boost_data, &pool).await?; + let _ = common::set_unassigned_oracle_boosting_assignments(&pool, &hex_boost_data).await?; let heartbeats = heartbeats(12, start, &owner, &cbsd_id, 0.0, 0.0, uuid); diff --git a/mobile_verifier/tests/integrations/common/mod.rs b/mobile_verifier/tests/integrations/common/mod.rs index 1ec3a0a94..5722f6a7a 100644 --- a/mobile_verifier/tests/integrations/common/mod.rs +++ b/mobile_verifier/tests/integrations/common/mod.rs @@ -1,10 +1,14 @@ use chrono::{DateTime, Utc}; -use file_store::file_sink::{FileSinkClient, Message as SinkMessage}; +use file_store::{ + file_sink::{FileSinkClient, Message as SinkMessage}, + traits::TimestampEncode, +}; use futures::{stream, StreamExt}; use helium_proto::{ services::poc_mobile::{ - mobile_reward_share::Reward as MobileReward, GatewayReward, MobileRewardShare, RadioReward, - ServiceProviderReward, SpeedtestAvg, SubscriberReward, UnallocatedReward, + mobile_reward_share::Reward as MobileReward, GatewayReward, MobileRewardShare, + OracleBoostingHexAssignment, OracleBoostingReportV1, RadioReward, ServiceProviderReward, + SpeedtestAvg, SubscriberReward, UnallocatedReward, }, Message, }; @@ -12,7 +16,12 @@ use mobile_config::{ boosted_hex_info::{BoostedHexInfo, BoostedHexInfoStream}, client::{hex_boosting_client::HexBoostingInfoResolver, ClientError}, }; -use mobile_verifier::boosting_oracles::{Assignment, HexBoostData}; +use mobile_verifier::boosting_oracles::{ + AssignedCoverageObjects, Assignment, HexAssignment, HexBoostData, +}; +use rust_decimal::prelude::ToPrimitive; +use rust_decimal_macros::dec; +use sqlx::PgPool; use std::collections::HashMap; use tokio::{sync::mpsc::error::TryRecvError, time::timeout}; @@ -204,3 +213,41 @@ pub fn mock_hex_boost_data( .build() .unwrap() } + +pub async fn set_unassigned_oracle_boosting_assignments( + pool: &PgPool, + data_sets: &HexBoostData, +) -> anyhow::Result> { + let assigned_coverage_objs = AssignedCoverageObjects::assign_hex_stream( + mobile_verifier::boosting_oracles::data_sets::db::fetch_hexes_with_null_assignments(pool), + data_sets, + ) + .await?; + let timestamp = Utc::now().encode_timestamp(); + let mut output = Vec::new(); + for (uuid, hexes) in assigned_coverage_objs.coverage_objs.iter() { + let assignments: Vec<_> = hexes + .iter() + .map(|hex| { + let location = format!("{:x}", hex.hex); + let assignment_multiplier = (hex.assignments.boosting_multiplier() * dec!(1000)) + .to_u32() + .unwrap_or(0); + OracleBoostingHexAssignment { + location, + urbanized: hex.assignments.urbanized.into(), + footfall: hex.assignments.footfall.into(), + landtype: hex.assignments.landtype.into(), + assignment_multiplier, + } + }) + .collect(); + output.push(OracleBoostingReportV1 { + coverage_object: Vec::from(uuid.into_bytes()), + assignments, + timestamp, + }); + } + assigned_coverage_objs.save(pool).await?; + Ok(output) +} diff --git a/mobile_verifier/tests/integrations/hex_boosting.rs b/mobile_verifier/tests/integrations/hex_boosting.rs index 8dcbb823d..b438cd19c 100644 --- a/mobile_verifier/tests/integrations/hex_boosting.rs +++ b/mobile_verifier/tests/integrations/hex_boosting.rs @@ -13,7 +13,6 @@ use helium_proto::services::poc_mobile::{ use hextree::Cell; use mobile_config::boosted_hex_info::BoostedHexInfo; use mobile_verifier::{ - boosting_oracles::{set_oracle_boosting_assignments, UnassignedHex}, cell_type::CellType, coverage::CoverageObject, heartbeats::{HbType, Heartbeat, ValidatedHeartbeat}, @@ -35,11 +34,9 @@ const BOOST_HEX_PUBKEY: &str = "J9JiLTpjaShxL8eMvUs8txVw6TZ36E38SiJ89NxnMbLU"; const BOOST_CONFIG_PUBKEY: &str = "BZM1QTud72B2cpTW7PhEnFmRX7ZWzvY7DpPpNJJuDrWG"; async fn update_assignments(pool: &PgPool) -> anyhow::Result<()> { - let unassigned_hexes = UnassignedHex::fetch_unassigned(pool); - let _ = set_oracle_boosting_assignments( - unassigned_hexes, - &common::mock_hex_boost_data_default(), + let _ = common::set_unassigned_oracle_boosting_assignments( pool, + &common::mock_hex_boost_data_default(), ) .await?; Ok(()) diff --git a/mobile_verifier/tests/integrations/modeled_coverage.rs b/mobile_verifier/tests/integrations/modeled_coverage.rs index 392801766..fee26b159 100644 --- a/mobile_verifier/tests/integrations/modeled_coverage.rs +++ b/mobile_verifier/tests/integrations/modeled_coverage.rs @@ -1,4 +1,3 @@ -use crate::common; use chrono::{DateTime, Duration, Utc}; use file_store::{ coverage::{CoverageObjectIngestReport, RadioHexSignalLevel}, @@ -16,7 +15,6 @@ use hextree::Cell; use mobile_config::boosted_hex_info::{BoostedHexInfo, BoostedHexes}; use mobile_verifier::{ - boosting_oracles::{set_oracle_boosting_assignments, UnassignedHex}, coverage::{CoverageClaimTimeCache, CoverageObject, CoverageObjectCache, Seniority}, geofence::GeofenceValidator, heartbeats::{ @@ -34,6 +32,8 @@ use sqlx::PgPool; use std::{collections::HashMap, num::NonZeroU32, ops::Range, pin::pin, str::FromStr}; use uuid::Uuid; +use crate::common; + #[derive(Clone)] struct MockGeofence; @@ -405,11 +405,9 @@ async fn process_input( } transaction.commit().await?; - let unassigned_hexes = UnassignedHex::fetch_unassigned(pool); - let _ = set_oracle_boosting_assignments( - unassigned_hexes, - &common::mock_hex_boost_data_default(), + let _ = common::set_unassigned_oracle_boosting_assignments( pool, + &common::mock_hex_boost_data_default(), ) .await?; diff --git a/mobile_verifier/tests/integrations/rewarder_poc_dc.rs b/mobile_verifier/tests/integrations/rewarder_poc_dc.rs index 2812a06be..f9287ed5e 100644 --- a/mobile_verifier/tests/integrations/rewarder_poc_dc.rs +++ b/mobile_verifier/tests/integrations/rewarder_poc_dc.rs @@ -10,7 +10,6 @@ use helium_proto::services::poc_mobile::{ SignalLevel, UnallocatedReward, UnallocatedRewardType, }; use mobile_verifier::{ - boosting_oracles::{set_oracle_boosting_assignments, UnassignedHex}, cell_type::CellType, coverage::CoverageObject, data_session, @@ -266,11 +265,9 @@ async fn seed_heartbeats( } async fn update_assignments(pool: &PgPool) -> anyhow::Result<()> { - let unassigned_hexes = UnassignedHex::fetch_unassigned(pool); - let _ = set_oracle_boosting_assignments( - unassigned_hexes, - &common::mock_hex_boost_data_default(), + let _ = common::set_unassigned_oracle_boosting_assignments( pool, + &common::mock_hex_boost_data_default(), ) .await?; Ok(()) From a922849571faf73e825ce8caaec6f51b6278f373 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Fri, 17 May 2024 10:14:45 -0400 Subject: [PATCH 44/53] use sleep_until --- mobile_verifier/src/boosting_oracles/data_sets.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index 7ef92a371..afc919929 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -1,7 +1,5 @@ use std::{ - collections::HashMap, - path::{Path, PathBuf}, - pin::pin, + collections::HashMap, path::{Path, PathBuf}, pin::pin, }; use chrono::{DateTime, Duration, Utc}; @@ -19,7 +17,7 @@ use rust_decimal::prelude::ToPrimitive; use rust_decimal_macros::dec; use sqlx::{FromRow, PgPool, QueryBuilder}; use task_manager::{ManagedTask, TaskManager}; -use tokio::{fs::File, io::AsyncWriteExt, sync::mpsc::Receiver}; +use tokio::{fs::File, io::AsyncWriteExt, sync::mpsc::Receiver, time::Instant}; use crate::{boosting_oracles::assignment::HexAssignments, coverage::SignalLevel, Settings}; @@ -329,6 +327,8 @@ where } loop { + let wakeup = Instant::now() + poll_duration.to_std()?; + #[rustfmt::skip] tokio::select! { _ = self.new_coverage_object_signal.recv() => { @@ -342,7 +342,7 @@ where ).await?; } }, - _ = tokio::time::sleep(poll_duration.to_std()?) => { + _ = tokio::time::sleep_until(wakeup) => { self.check_for_new_data_sets().await?; } } From 902e62f656f6b03dbccd4172ddba607fb1b650c7 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Fri, 17 May 2024 10:28:19 -0400 Subject: [PATCH 45/53] Wrap coverage obj notification in struct --- .../src/boosting_oracles/data_sets.rs | 24 +++++++----- mobile_verifier/src/cli/server.rs | 10 ++--- mobile_verifier/src/coverage.rs | 39 +++++++++++++++---- 3 files changed, 52 insertions(+), 21 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index afc919929..0b8b548cd 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -1,5 +1,7 @@ use std::{ - collections::HashMap, path::{Path, PathBuf}, pin::pin, + collections::HashMap, + path::{Path, PathBuf}, + pin::pin, }; use chrono::{DateTime, Duration, Utc}; @@ -17,9 +19,13 @@ use rust_decimal::prelude::ToPrimitive; use rust_decimal_macros::dec; use sqlx::{FromRow, PgPool, QueryBuilder}; use task_manager::{ManagedTask, TaskManager}; -use tokio::{fs::File, io::AsyncWriteExt, sync::mpsc::Receiver, time::Instant}; +use tokio::{fs::File, io::AsyncWriteExt, time::Instant}; -use crate::{boosting_oracles::assignment::HexAssignments, coverage::SignalLevel, Settings}; +use crate::{ + boosting_oracles::assignment::HexAssignments, + coverage::{NewCoverageObjectNotification, SignalLevel}, + Settings, +}; use super::{ footfall::Footfall, landtype::Landtype, urbanization::Urbanization, HexAssignment, HexBoostData, @@ -105,7 +111,7 @@ pub struct DataSetDownloaderDaemon { store: FileStore, oracle_boosting_sink: FileSinkClient, data_set_directory: PathBuf, - new_coverage_object_signal: Receiver<()>, + new_coverage_object_notification: NewCoverageObjectNotification, } #[derive(FromRow)] @@ -173,7 +179,7 @@ impl DataSetDownloaderDaemon { pool: PgPool, settings: &Settings, file_upload: FileUpload, - new_coverage_object_signal: Receiver<()>, + new_coverage_object_notification: NewCoverageObjectNotification, ) -> anyhow::Result { let (oracle_boosting_reports, oracle_boosting_reports_server) = file_sink::FileSinkBuilder::new( @@ -202,7 +208,7 @@ impl DataSetDownloaderDaemon { FileStore::from_settings(&settings.data_sets).await?, oracle_boosting_reports, settings.data_sets_directory.clone(), - new_coverage_object_signal, + new_coverage_object_notification, ); Ok(TaskManager::builder() @@ -224,7 +230,7 @@ where store: FileStore, oracle_boosting_sink: FileSinkClient, data_set_directory: PathBuf, - new_coverage_object_signal: Receiver<()>, + new_coverage_object_notification: NewCoverageObjectNotification, ) -> Self { Self { pool, @@ -232,7 +238,7 @@ where store, oracle_boosting_sink, data_set_directory, - new_coverage_object_signal, + new_coverage_object_notification, } } @@ -331,7 +337,7 @@ where #[rustfmt::skip] tokio::select! { - _ = self.new_coverage_object_signal.recv() => { + _ = self.new_coverage_object_notification.await_new_coverage_object() => { // If we see a new coverage object, we want to assign only those hexes // that don't have an assignment if self.data_sets.is_ready() { diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index 83fa3a35e..0e5c04cb3 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -1,6 +1,6 @@ use crate::{ boosting_oracles::DataSetDownloaderDaemon, - coverage::CoverageDaemon, + coverage::{new_coverage_object_notification_channel, CoverageDaemon}, data_session::DataSessionIngestor, geofence::Geofence, heartbeats::{cbrs::CbrsHeartbeatDaemon, wifi::WifiHeartbeatDaemon}, @@ -22,7 +22,6 @@ use mobile_config::client::{ CarrierServiceClient, GatewayClient, }; use task_manager::TaskManager; -use tokio::sync::mpsc::channel; #[derive(Debug, clap::Args)] pub struct Cmd {} @@ -101,7 +100,8 @@ impl Cmd { settings.usa_and_mexico_fencing_resolution()?, )?; - let (new_coverage_obj_signal_tx, new_coverage_obj_signal_rx) = channel(1); + let (new_coverage_obj_notifier, new_coverage_obj_notification) = + new_coverage_object_notification_channel(); TaskManager::builder() .add_task(file_upload_server) @@ -150,7 +150,7 @@ impl Cmd { file_upload.clone(), report_ingest.clone(), auth_client.clone(), - new_coverage_obj_signal_tx, + new_coverage_obj_notifier, ) .await?, ) @@ -159,7 +159,7 @@ impl Cmd { pool.clone(), settings, file_upload.clone(), - new_coverage_obj_signal_rx, + new_coverage_obj_notification, ) .await?, ) diff --git a/mobile_verifier/src/coverage.rs b/mobile_verifier/src/coverage.rs index eed1463c1..11dfe7d88 100644 --- a/mobile_verifier/src/coverage.rs +++ b/mobile_verifier/src/coverage.rs @@ -41,7 +41,7 @@ use std::{ time::Instant, }; use task_manager::{ManagedTask, TaskManager}; -use tokio::sync::mpsc::{Receiver, Sender}; +use tokio::sync::mpsc::{channel, Receiver, Sender}; use uuid::Uuid; #[derive(Copy, Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Type)] @@ -70,7 +70,7 @@ pub struct CoverageDaemon { auth_client: AuthorizationClient, coverage_objs: Receiver>, coverage_obj_sink: FileSinkClient, - new_coverage_object_signal: Sender<()>, + new_coverage_object_notifier: NewCoverageObjectNotifier, } impl CoverageDaemon { @@ -80,7 +80,7 @@ impl CoverageDaemon { file_upload: FileUpload, file_store: FileStore, auth_client: AuthorizationClient, - new_coverage_object_signal: Sender<()>, + new_coverage_object_notifier: NewCoverageObjectNotifier, ) -> anyhow::Result { let (valid_coverage_objs, valid_coverage_objs_server) = file_sink::FileSinkBuilder::new( FileType::CoverageObject, @@ -108,7 +108,7 @@ impl CoverageDaemon { auth_client, coverage_objs, valid_coverage_objs, - new_coverage_object_signal, + new_coverage_object_notifier, ); Ok(TaskManager::builder() @@ -123,14 +123,14 @@ impl CoverageDaemon { auth_client: AuthorizationClient, coverage_objs: Receiver>, coverage_obj_sink: FileSinkClient, - new_coverage_object_signal: Sender<()>, + new_coverage_object_notifier: NewCoverageObjectNotifier, ) -> Self { Self { pool, auth_client, coverage_objs, coverage_obj_sink, - new_coverage_object_signal, + new_coverage_object_notifier, } } @@ -178,7 +178,7 @@ impl CoverageDaemon { transaction.commit().await?; // Tell the data set manager to update the assignments. - let _ = self.new_coverage_object_signal.try_send(()); + self.new_coverage_object_notifier.notify(); Ok(()) } @@ -198,6 +198,31 @@ impl ManagedTask for CoverageDaemon { } } +pub struct NewCoverageObjectNotifier(Sender<()>); + +impl NewCoverageObjectNotifier { + fn notify(&self) { + let _ = self.0.try_send(()); + } +} + +pub struct NewCoverageObjectNotification(Receiver<()>); + +impl NewCoverageObjectNotification { + pub async fn await_new_coverage_object(&mut self) { + let _ = self.0.recv().await; + } +} + +pub fn new_coverage_object_notification_channel( +) -> (NewCoverageObjectNotifier, NewCoverageObjectNotification) { + let (tx, rx) = channel(1); + ( + NewCoverageObjectNotifier(tx), + NewCoverageObjectNotification(rx), + ) +} + pub struct CoverageObject { pub coverage_object: file_store::coverage::CoverageObject, pub validity: CoverageObjectValidity, From 4b45c1cd6ead8ef8ed95e036e85c8a65b00be952 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Fri, 17 May 2024 10:51:22 -0400 Subject: [PATCH 46/53] Ahhh... that's better --- mobile_verifier/src/cli/server.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index 0e5c04cb3..c1ded037c 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -12,11 +12,7 @@ use crate::{ }; use anyhow::Result; use chrono::Duration; -use file_store::{ - file_sink, - file_upload::{self}, - FileStore, FileType, -}; +use file_store::{file_sink, file_upload, FileStore, FileType}; use mobile_config::client::{ entity_client::EntityClient, hex_boosting_client::HexBoostingClient, AuthorizationClient, CarrierServiceClient, GatewayClient, From 8ebb8c1f5b60f8036020a77e6d5d5dd7f7e26d62 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Fri, 17 May 2024 11:16:06 -0400 Subject: [PATCH 47/53] Fix --- mobile_verifier/src/boosting_oracles/data_sets.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index 0b8b548cd..77693d7f1 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -332,9 +332,8 @@ where .await?; } + let mut wakeup = Instant::now() + poll_duration.to_std()?; loop { - let wakeup = Instant::now() + poll_duration.to_std()?; - #[rustfmt::skip] tokio::select! { _ = self.new_coverage_object_notification.await_new_coverage_object() => { @@ -350,6 +349,7 @@ where }, _ = tokio::time::sleep_until(wakeup) => { self.check_for_new_data_sets().await?; + wakeup = Instant::now() + poll_duration.to_std()?; } } } From c1a3f9657462dabeb1775050464d7fd7284f0431 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Mon, 20 May 2024 10:33:18 -0400 Subject: [PATCH 48/53] rename data_sets table to hex_assignments_data_set_status --- mobile_verifier/migrations/33_data_sets.sql | 2 +- mobile_verifier/src/boosting_oracles/data_sets.rs | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mobile_verifier/migrations/33_data_sets.sql b/mobile_verifier/migrations/33_data_sets.sql index d76d99a37..dba5aea9f 100644 --- a/mobile_verifier/migrations/33_data_sets.sql +++ b/mobile_verifier/migrations/33_data_sets.sql @@ -12,7 +12,7 @@ CREATE TYPE data_set_type AS enum ( 'landtype' ); -CREATE TABLE IF NOT EXISTS data_sets ( +CREATE TABLE IF NOT EXISTS hex_assignment_data_set_status ( filename TEXT PRIMARY KEY, data_set data_set_type NOT NULL, time_to_use TIMESTAMPTZ NOT NULL, diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index 77693d7f1..06e65ae7a 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -444,7 +444,7 @@ pub mod db { pool: &PgPool, data_set_type: DataSetType, ) -> sqlx::Result>> { - sqlx::query_scalar("SELECT time_to_use FROM data_sets WHERE data_set = $1 ORDER BY time_to_use DESC LIMIT 1") + sqlx::query_scalar("SELECT time_to_use FROM hex_assignment_data_set_status WHERE data_set = $1 ORDER BY time_to_use DESC LIMIT 1") .bind(data_set_type) .fetch_optional(pool) .await @@ -458,7 +458,7 @@ pub mod db { ) -> sqlx::Result<()> { sqlx::query( r#" - INSERT INTO data_sets (filename, data_set, time_to_use, status) + INSERT INTO hex_assignment_data_set_status (filename, data_set, time_to_use, status) VALUES ($1, $2, $3, 'pending') ON CONFLICT DO NOTHING "#, @@ -477,7 +477,7 @@ pub mod db { since: Option>, ) -> sqlx::Result> { sqlx::query_as( - "SELECT filename, time_to_use, status FROM data_sets WHERE status != 'processed' AND data_set = $1 AND COALESCE(time_to_use > $2, TRUE) AND time_to_use <= $3 ORDER BY time_to_use DESC LIMIT 1" + "SELECT filename, time_to_use, status FROM hex_assignment_data_set_status WHERE status != 'processed' AND data_set = $1 AND COALESCE(time_to_use > $2, TRUE) AND time_to_use <= $3 ORDER BY time_to_use DESC LIMIT 1" ) .bind(data_set_type) .bind(since) @@ -491,7 +491,7 @@ pub mod db { data_set_type: DataSetType, ) -> sqlx::Result> { sqlx::query_as( - "SELECT filename, time_to_use, status FROM data_sets WHERE status = 'processed' AND data_set = $1 ORDER BY time_to_use DESC LIMIT 1" + "SELECT filename, time_to_use, status FROM hex_assignment_data_set_status WHERE status = 'processed' AND data_set = $1 ORDER BY time_to_use DESC LIMIT 1" ) .bind(data_set_type) .fetch_optional(pool) @@ -503,7 +503,7 @@ pub mod db { filename: &str, status: DataSetStatus, ) -> sqlx::Result<()> { - sqlx::query("UPDATE data_sets SET status = $1 WHERE filename = $2") + sqlx::query("UPDATE hex_assignment_data_set_status SET status = $1 WHERE filename = $2") .bind(status) .bind(filename) .execute(pool) @@ -516,7 +516,7 @@ pub mod db { data_set_type: DataSetType, ) -> sqlx::Result>> { sqlx::query_scalar( - "SELECT time_to_use FROM data_sets WHERE status = 'processed' AND data_set = $1 ORDER BY time_to_use DESC LIMIT 1" + "SELECT time_to_use FROM hex_assignment_data_set_status WHERE status = 'processed' AND data_set = $1 ORDER BY time_to_use DESC LIMIT 1" ) .bind(data_set_type) .fetch_optional(pool) @@ -529,7 +529,7 @@ pub mod db { period_end: DateTime, ) -> sqlx::Result { Ok(sqlx::query_scalar( - "SELECT COUNT(*) > 0 FROM data_sets WHERE time_to_use <= $1 AND status != 'processed'", + "SELECT COUNT(*) > 0 FROM hex_assignment_data_set_status WHERE time_to_use <= $1 AND status != 'processed'", ) .bind(period_end) .fetch_one(pool) From dc8a57d58eb1b64a52b67fdabd17bbbdc5f8d45c Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Mon, 20 May 2024 10:34:51 -0400 Subject: [PATCH 49/53] Add footfall comment --- mobile_verifier/src/boosting_oracles/footfall.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mobile_verifier/src/boosting_oracles/footfall.rs b/mobile_verifier/src/boosting_oracles/footfall.rs index 6f9db1e51..4af2ebaf5 100644 --- a/mobile_verifier/src/boosting_oracles/footfall.rs +++ b/mobile_verifier/src/boosting_oracles/footfall.rs @@ -57,6 +57,9 @@ impl HexAssignment for Footfall { anyhow::bail!("No footfall data set has been loaded"); }; + // The footfall disktree maps hexes to a single byte, a value of one indicating + // assignment A and a value of zero indicating assignment B. If no value is present, + // assignment C is given. match footfall.get(cell)? { Some((_, &[x])) if x >= 1 => Ok(Assignment::A), Some((_, &[0])) => Ok(Assignment::B), From 440776c216d0c5eb447c7be62f7b8d4b8cdd9c52 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Mon, 20 May 2024 10:35:35 -0400 Subject: [PATCH 50/53] Remove allow dead code from tests per moogey --- mobile_verifier/tests/integrations/common/mod.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/mobile_verifier/tests/integrations/common/mod.rs b/mobile_verifier/tests/integrations/common/mod.rs index 5722f6a7a..c9b38bcf9 100644 --- a/mobile_verifier/tests/integrations/common/mod.rs +++ b/mobile_verifier/tests/integrations/common/mod.rs @@ -26,7 +26,6 @@ use std::collections::HashMap; use tokio::{sync::mpsc::error::TryRecvError, time::timeout}; #[derive(Debug, Clone)] -#[allow(dead_code)] pub struct MockHexBoostingClient { boosted_hexes: Vec, } @@ -200,7 +199,6 @@ pub fn mock_hex_boost_data_default() -> HexBoostData; -#[allow(dead_code)] pub fn mock_hex_boost_data( footfall: MockAssignmentMap, urbanized: MockAssignmentMap, From 6c65b8bbc1efef878db4aaf4c3eccf83558b3060 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Mon, 20 May 2024 10:40:37 -0400 Subject: [PATCH 51/53] Nevermind --- mobile_verifier/tests/integrations/common/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mobile_verifier/tests/integrations/common/mod.rs b/mobile_verifier/tests/integrations/common/mod.rs index c9b38bcf9..5722f6a7a 100644 --- a/mobile_verifier/tests/integrations/common/mod.rs +++ b/mobile_verifier/tests/integrations/common/mod.rs @@ -26,6 +26,7 @@ use std::collections::HashMap; use tokio::{sync::mpsc::error::TryRecvError, time::timeout}; #[derive(Debug, Clone)] +#[allow(dead_code)] pub struct MockHexBoostingClient { boosted_hexes: Vec, } @@ -199,6 +200,7 @@ pub fn mock_hex_boost_data_default() -> HexBoostData; +#[allow(dead_code)] pub fn mock_hex_boost_data( footfall: MockAssignmentMap, urbanized: MockAssignmentMap, From 01b787c6a95be5fea670fec35da481b6b81f0e30 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Mon, 20 May 2024 16:33:17 -0400 Subject: [PATCH 52/53] Make data set downloader poll duration a setting --- mobile_verifier/src/boosting_oracles/data_sets.rs | 15 +++++++++------ mobile_verifier/src/settings.rs | 7 +++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/mobile_verifier/src/boosting_oracles/data_sets.rs b/mobile_verifier/src/boosting_oracles/data_sets.rs index 06e65ae7a..76eaec1a4 100644 --- a/mobile_verifier/src/boosting_oracles/data_sets.rs +++ b/mobile_verifier/src/boosting_oracles/data_sets.rs @@ -2,9 +2,10 @@ use std::{ collections::HashMap, path::{Path, PathBuf}, pin::pin, + time::Duration, }; -use chrono::{DateTime, Duration, Utc}; +use chrono::{DateTime, Utc}; use file_store::{ file_sink::{self, FileSinkClient}, file_upload::FileUpload, @@ -112,6 +113,7 @@ pub struct DataSetDownloaderDaemon { oracle_boosting_sink: FileSinkClient, data_set_directory: PathBuf, new_coverage_object_notification: NewCoverageObjectNotification, + poll_duration: Duration, } #[derive(FromRow)] @@ -189,7 +191,7 @@ impl DataSetDownloaderDaemon { concat!(env!("CARGO_PKG_NAME"), "_oracle_boosting_report"), ) .auto_commit(true) - .roll_time(Duration::minutes(15)) + .roll_time(chrono::Duration::minutes(15)) .create() .await?; @@ -209,6 +211,7 @@ impl DataSetDownloaderDaemon { oracle_boosting_reports, settings.data_sets_directory.clone(), new_coverage_object_notification, + settings.data_sets_poll_duration, ); Ok(TaskManager::builder() @@ -231,6 +234,7 @@ where oracle_boosting_sink: FileSinkClient, data_set_directory: PathBuf, new_coverage_object_notification: NewCoverageObjectNotification, + poll_duration: Duration, ) -> Self { Self { pool, @@ -239,6 +243,7 @@ where oracle_boosting_sink, data_set_directory, new_coverage_object_notification, + poll_duration, } } @@ -306,8 +311,6 @@ where } pub async fn run(mut self) -> anyhow::Result<()> { - let poll_duration = Duration::minutes(1); - self.data_sets .urbanization .fetch_first_data_set(&self.pool, &self.data_set_directory) @@ -332,7 +335,7 @@ where .await?; } - let mut wakeup = Instant::now() + poll_duration.to_std()?; + let mut wakeup = Instant::now() + self.poll_duration; loop { #[rustfmt::skip] tokio::select! { @@ -349,7 +352,7 @@ where }, _ = tokio::time::sleep_until(wakeup) => { self.check_for_new_data_sets().await?; - wakeup = Instant::now() + poll_duration.to_std()?; + wakeup = Instant::now() + self.poll_duration; } } } diff --git a/mobile_verifier/src/settings.rs b/mobile_verifier/src/settings.rs index 744ca7fbb..4a5c681b5 100644 --- a/mobile_verifier/src/settings.rs +++ b/mobile_verifier/src/settings.rs @@ -46,6 +46,9 @@ pub struct Settings { pub max_asserted_distance_deviation: u32, /// Directory in which new oracle boosting data sets are downloaded into pub data_sets_directory: PathBuf, + /// Poll duration for new data sets + #[serde(with = "humantime_serde", default = "default_data_sets_poll_duration")] + pub data_sets_poll_duration: Duration, // Geofencing settings pub usa_and_mexico_geofence_regions: String, #[serde(default = "default_fencing_resolution")] @@ -84,6 +87,10 @@ fn default_reward_period_offset() -> Duration { humantime::parse_duration("30 minutes").unwrap() } +fn default_data_sets_poll_duration() -> Duration { + humantime::parse_duration("30 minutes").unwrap() +} + impl Settings { /// Load Settings from a given path. Settings are loaded from a given /// optional path and can be overriden with environment variables. From 26d7c25a59eab758a937c7c0cba1cd2562b21642 Mon Sep 17 00:00:00 2001 From: Matthew Plant Date: Mon, 20 May 2024 17:10:19 -0400 Subject: [PATCH 53/53] Change migration --- mobile_verifier/migrations/33_data_sets.sql | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mobile_verifier/migrations/33_data_sets.sql b/mobile_verifier/migrations/33_data_sets.sql index dba5aea9f..9c726bf9b 100644 --- a/mobile_verifier/migrations/33_data_sets.sql +++ b/mobile_verifier/migrations/33_data_sets.sql @@ -1,16 +1,22 @@ -DROP TYPE IF EXISTS data_set_status; +DO $$ BEGIN CREATE TYPE data_set_status AS enum ( 'pending', 'downloaded', 'processed' ); +EXCEPTION + WHEN duplicate_object THEN null; +END $$; -DROP TYPE IF EXISTS data_set_type; +DO $$ BEGIN CREATE TYPE data_set_type AS enum ( 'urbanization', 'footfall', 'landtype' ); +EXCEPTION + WHEN duplicate_object THEN null; +END $$; CREATE TABLE IF NOT EXISTS hex_assignment_data_set_status ( filename TEXT PRIMARY KEY,