Automatically download data sets

file_store/src/file_info.rs

@@ -154,6 +154,9 @@ pub const COVERAGE_OBJECT_INGEST_REPORT: &str = "coverage_object_ingest_report";
 pub const SENIORITY_UPDATE: &str = "seniority_update";
 pub const BOOSTED_HEX_UPDATE: &str = "boosted_hex_update";
 pub const ORACLE_BOOSTING_REPORT: &str = "oracle_boosting_report";
+pub const URBANIZATION_DATA_SET: &str = "urbanization";
+pub const FOOTFALL_DATA_SET: &str = "footfall";
+pub const LANDTYPE_DATA_SET: &str = "landtype";
 
 #[derive(Debug, PartialEq, Eq, Clone, Serialize, Copy, strum::EnumCount)]
 #[serde(rename_all = "snake_case")]
@@ -200,6 +203,9 @@ pub enum FileType {
     RadioThresholdReq,
     RadioThresholdIngestReport,
     VerifiedRadioThresholdIngestReport,
+    UrbanizationDataSet,
+    FootfallDataSet,
+    LandtypeDataSet,
     InvalidatedRadioThresholdReq,
     InvalidatedRadioThresholdIngestReport,
     VerifiedInvalidatedRadioThresholdIngestReport,
@@ -261,6 +267,9 @@ impl fmt::Display for FileType {
             Self::SeniorityUpdate => SENIORITY_UPDATE,
             Self::BoostedHexUpdate => BOOSTED_HEX_UPDATE,
             Self::OracleBoostingReport => ORACLE_BOOSTING_REPORT,
+            Self::UrbanizationDataSet => URBANIZATION_DATA_SET,
+            Self::FootfallDataSet => FOOTFALL_DATA_SET,
+            Self::LandtypeDataSet => LANDTYPE_DATA_SET,
         };
         f.write_str(s)
     }
@@ -322,6 +331,9 @@ impl FileType {
             Self::SeniorityUpdate => SENIORITY_UPDATE,
             Self::BoostedHexUpdate => BOOSTED_HEX_UPDATE,
             Self::OracleBoostingReport => ORACLE_BOOSTING_REPORT,
+            Self::UrbanizationDataSet => URBANIZATION_DATA_SET,
+            Self::FootfallDataSet => FOOTFALL_DATA_SET,
+            Self::LandtypeDataSet => LANDTYPE_DATA_SET,
         }
     }
 }
@@ -383,6 +395,9 @@ impl FromStr for FileType {
             SENIORITY_UPDATE => Self::SeniorityUpdate,
             BOOSTED_HEX_UPDATE => Self::BoostedHexUpdate,
             ORACLE_BOOSTING_REPORT => Self::OracleBoostingReport,
+            URBANIZATION_DATA_SET => Self::UrbanizationDataSet,
+            FOOTFALL_DATA_SET => Self::FootfallDataSet,
+            LANDTYPE_DATA_SET => Self::LandtypeDataSet,
             _ => return Err(Error::from(io::Error::from(io::ErrorKind::InvalidInput))),
         };
         Ok(result)

mobile_verifier/Cargo.toml

@@ -8,6 +8,7 @@ authors.workspace = true
 
 [dependencies]
 anyhow = {workspace = true}
+async-compression = {version = "0", features = ["tokio", "gzip"]}
 config = {workspace = true}
 thiserror = {workspace = true}
 serde =  {workspace = true}
@@ -36,7 +37,8 @@ humantime = {workspace = true}
 rust_decimal = {workspace = true}
 rust_decimal_macros = {workspace = true}
 tonic = {workspace = true}
-tokio-stream = { workspace = true }
+tokio-stream = {workspace = true}
+tokio-util = {workspace = true}
 metrics = {workspace = true}
 metrics-exporter-prometheus = {workspace = true}
 mobile-config = {path = "../mobile_config"}

mobile_verifier/src/boosting_oracles/assignment.rs

@@ -3,6 +3,8 @@ use rust_decimal::Decimal;
 use rust_decimal_macros::dec;
 use std::fmt;
 
+use super::HexAssignment;
+
 #[derive(Debug, Clone, PartialEq, Eq, sqlx::FromRow)]
 pub struct HexAssignments {
     pub footfall: Assignment,
@@ -58,6 +60,19 @@ impl fmt::Display for Assignment {
 }
 
 impl HexAssignments {
+    pub fn from_data_sets(
+        cell: hextree::Cell,
+        footfall: &impl HexAssignment,
+        landtype: &impl HexAssignment,
+        urbanized: &impl HexAssignment,
+    ) -> anyhow::Result<Self> {
+        Ok(Self {
+            footfall: footfall.assignment(cell)?,
+            landtype: landtype.assignment(cell)?,
+            urbanized: urbanized.assignment(cell)?,
+        })
+    }
+
     pub fn boosting_multiplier(&self) -> Decimal {
         let HexAssignments {
             footfall,

mobile_verifier/src/boosting_oracles/footfall.rs

@@ -0,0 +1,69 @@
+use std::path::Path;
+
+use chrono::{DateTime, Utc};
+use hextree::disktree::DiskTreeMap;
+
+use super::{Assignment, DataSet, DataSetType, DiskTreeLike, HexAssignment};
+
+pub struct Footfall<Foot> {
+    footfall: Option<Foot>,
+    timestamp: Option<DateTime<Utc>>,
+}
+
+impl<F> Footfall<F> {
+    pub fn new() -> Self {
+        Self {
+            footfall: None,
+            timestamp: None,
+        }
+    }
+
+    pub fn new_mock(footfall: F) -> Self {
+        Self {
+            footfall: Some(footfall),
+            timestamp: None,
+        }
+    }
+}
+
+impl<F> Default for Footfall<F> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl DataSet for Footfall<DiskTreeMap> {
+    const TYPE: DataSetType = DataSetType::Footfall;
+
+    fn timestamp(&self) -> Option<DateTime<Utc>> {
+        self.timestamp
+    }
+
+    fn update(&mut self, path: &Path, time_to_use: DateTime<Utc>) -> anyhow::Result<()> {
+        self.footfall = Some(DiskTreeMap::open(path)?);
+        self.timestamp = Some(time_to_use);
+        Ok(())
+    }
+
+    fn is_ready(&self) -> bool {
+        self.footfall.is_some()
+    }
+}
+
+impl<Foot> HexAssignment for Footfall<Foot>
+where
+    Foot: DiskTreeLike + Send + Sync + 'static,
+{
+    fn assignment(&self, cell: hextree::Cell) -> anyhow::Result<Assignment> {
+        let Some(ref footfall) = self.footfall else {
+            anyhow::bail!("No footfall data set has been loaded");
+        };
+
+        match footfall.get(cell)? {
+            Some((_, &[x])) if x >= 1 => Ok(Assignment::A),
+            Some((_, &[0])) => Ok(Assignment::B),
+            None => Ok(Assignment::C),
+            Some((_, other)) => anyhow::bail!("Unexpected disktree data: {cell:?} {other:?}"),
+        }
+    }
+}

mobile_verifier/src/boosting_oracles/landtype.rs

@@ -0,0 +1,154 @@
+use std::path::Path;
+
+use chrono::{DateTime, Utc};
+use hextree::disktree::DiskTreeMap;
+
+use super::{Assignment, DataSet, DataSetType, DiskTreeLike, HexAssignment};
+
+pub struct Landtype<Land> {
+    landtype: Option<Land>,
+    timestamp: Option<DateTime<Utc>>,
+}
+
+impl<L> Landtype<L> {
+    pub fn new() -> Self {
+        Self {
+            landtype: None,
+            timestamp: None,
+        }
+    }
+
+    pub fn new_mock(landtype: L) -> Self {
+        Self {
+            landtype: Some(landtype),
+            timestamp: None,
+        }
+    }
+}
+
+impl<L> Default for Landtype<L> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl DataSet for Landtype<DiskTreeMap> {
+    const TYPE: DataSetType = DataSetType::Landtype;
+
+    fn timestamp(&self) -> Option<DateTime<Utc>> {
+        self.timestamp
+    }
+
+    fn update(&mut self, path: &Path, time_to_use: DateTime<Utc>) -> anyhow::Result<()> {
+        self.landtype = Some(DiskTreeMap::open(path)?);
+        self.timestamp = Some(time_to_use);
+        Ok(())
+    }
+
+    fn is_ready(&self) -> bool {
+        self.landtype.is_some()
+    }
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum LandtypeValue {
+    Tree = 10,
+    Shrub = 20,
+    Grass = 30,
+    Crop = 40,
+    Built = 50,
+    Bare = 60,
+    Frozen = 70,
+    Water = 80,
+    Wet = 90,
+    Mangrove = 95,
+    Moss = 100,
+}
+
+impl std::fmt::Display for LandtypeValue {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(self.to_str())
+    }
+}
+
+impl LandtypeValue {
+    pub(crate) fn to_str(self) -> &'static str {
+        match self {
+            LandtypeValue::Tree => "TreeCover",
+            LandtypeValue::Shrub => "Shrubland",
+            LandtypeValue::Grass => "Grassland",
+            LandtypeValue::Crop => "Cropland",
+            LandtypeValue::Built => "BuiltUp",
+            LandtypeValue::Bare => "BareOrSparseVeg",
+            LandtypeValue::Frozen => "SnowAndIce",
+            LandtypeValue::Water => "Water",
+            LandtypeValue::Wet => "HerbaceousWetland",
+            LandtypeValue::Mangrove => "Mangroves",
+            LandtypeValue::Moss => "MossAndLichen",
+        }
+    }
+}
+
+impl TryFrom<u8> for LandtypeValue {
+    type Error = anyhow::Error;
+    fn try_from(other: u8) -> anyhow::Result<LandtypeValue, Self::Error> {
+        let val = match other {
+            10 => LandtypeValue::Tree,
+            20 => LandtypeValue::Shrub,
+            30 => LandtypeValue::Grass,
+            40 => LandtypeValue::Crop,
+            50 => LandtypeValue::Built,
+            60 => LandtypeValue::Bare,
+            70 => LandtypeValue::Frozen,
+            80 => LandtypeValue::Water,
+            90 => LandtypeValue::Wet,
+            95 => LandtypeValue::Mangrove,
+            100 => LandtypeValue::Moss,
+            other => anyhow::bail!("unexpected landtype disktree value: {other:?}"),
+        };
+        Ok(val)
+    }
+}
+
+impl From<LandtypeValue> for Assignment {
+    fn from(value: LandtypeValue) -> Self {
+        match value {
+            LandtypeValue::Built => Assignment::A,
+            //
+            LandtypeValue::Tree => Assignment::B,
+            LandtypeValue::Shrub => Assignment::B,
+            LandtypeValue::Grass => Assignment::B,
+            //
+            LandtypeValue::Bare => Assignment::C,
+            LandtypeValue::Crop => Assignment::C,
+            LandtypeValue::Frozen => Assignment::C,
+            LandtypeValue::Water => Assignment::C,
+            LandtypeValue::Wet => Assignment::C,
+            LandtypeValue::Mangrove => Assignment::C,
+            LandtypeValue::Moss => Assignment::C,
+        }
+    }
+}
+
+impl<Land> HexAssignment for Landtype<Land>
+where
+    Land: DiskTreeLike + Send + Sync + 'static,
+{
+    fn assignment(&self, cell: hextree::Cell) -> anyhow::Result<Assignment> {
+        let Some(ref landtype) = self.landtype else {
+            anyhow::bail!("No landtype data set has been loaded");
+        };
+
+        let Some((_, vals)) = landtype.get(cell)? else {
+            return Ok(Assignment::C);
+        };
+
+        anyhow::ensure!(
+            vals.len() == 1,
+            "unexpected landtype disktree data: {cell:?} {vals:?}"
+        );
+
+        let cover = LandtypeValue::try_from(vals[0])?;
+        Ok(cover.into())
+    }
+}