From e6d99c72685af257864283467cfcde85967ae1cb Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Thu, 19 Sep 2024 14:28:03 -0700 Subject: [PATCH 01/31] Add promotion funds workspace This workspace is all about dealing with Service Provider Promotion Fund allocation. HIP-114 https://github.com/helium/HIP/blob/main/0114-incentive-escrow-fund-for-subscriber-referrals.md Service Provider Promotions are stored in CarrierV0 on Solana. To keep the mobile-verifier from talking to a chain, this service will periodically check Solana and compare Service Providers allocations to what is stored in S3. If the values have changed, a new file will be output to a bucket for the mobile-verifier rewarder to read from. NOTE: Allocation Values are stored in Bps (Basis Points) https://www.investopedia.com/terms/b/basispoint.asp ** Commands *** ./promotion_fund write-solana Fetch Allocation values from Solana and write them to S3. This command _always_ writes an S3 file. *** ./promotion_fund print-s3 Using the lookback time in the provided settings file, show the Allocation values this service would start up with. *** ./promotion_fund server Start a server that reads from S3, then checks with Solana periodically for updated Allocatino values. Writing new files when needed. --- Cargo.lock | 208 +++++++---------------------------------------------- 1 file changed, 27 insertions(+), 181 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2fa28164d..3cad6df35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1617,7 +1617,7 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "beacon" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=master#197ff9c6cde7dc0d8334d6b4e27c58779e6a7ce0" +source = "git+https://github.com/helium/proto?branch=master#376765fe006051d6dcccf709def58e7ed291b845" dependencies = [ "base64 0.21.7", "byteorder", @@ -2117,16 +2117,7 @@ dependencies = [ [[package]] name = "circuit-breaker" version = "0.1.0" -source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" -dependencies = [ - "anchor-gen", - "anchor-lang 0.30.1", -] - -[[package]] -name = "circuit-breaker" -version = "0.1.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -2767,17 +2758,8 @@ dependencies = [ [[package]] name = "data-credits" -version = "0.2.2" -source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" -dependencies = [ - "anchor-gen", - "anchor-lang 0.30.1", -] - -[[package]] -name = "data-credits" -version = "0.2.2" -source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +version = "0.2.1" +source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -3155,16 +3137,7 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "fanout" version = "0.1.0" -source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" -dependencies = [ - "anchor-gen", - "anchor-lang 0.30.1", -] - -[[package]] -name = "fanout" -version = "0.1.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -3242,8 +3215,8 @@ dependencies = [ "serde_json", "sha2 0.10.8", "sqlx", - "strum 0.24.1", - "strum_macros 0.24.3", + "strum", + "strum_macros", "task-manager", "tempfile", "thiserror", @@ -3731,16 +3704,10 @@ dependencies = [ "unicode-segmentation", ] -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - [[package]] name = "helium-anchor-gen" version = "0.1.0" -source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -3808,17 +3775,8 @@ dependencies = [ [[package]] name = "helium-entity-manager" -version = "0.2.11" -source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" -dependencies = [ - "anchor-gen", - "anchor-lang 0.30.1", -] - -[[package]] -name = "helium-entity-manager" -version = "0.2.11" -source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +version = "0.3.1" +source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -3827,7 +3785,7 @@ dependencies = [ [[package]] name = "helium-lib" version = "0.0.0" -source = "git+https://github.com/helium/helium-wallet-rs.git?branch=master#4acf688beac3c507c33843a745516839e1f814b2" +source = "git+https://github.com/helium/helium-wallet-rs.git?branch=master#a4db666b45a531d690e561c225ca23c503a08bd1" dependencies = [ "anchor-client", "anchor-spl", @@ -3842,7 +3800,6 @@ dependencies = [ "helium-crypto", "helium-proto", "hex", - "hex-literal", "itertools", "jsonrpc_client", "lazy_static", @@ -3859,7 +3816,6 @@ dependencies = [ "spl-account-compression", "spl-associated-token-account 3.0.2", "thiserror", - "tonic", "tracing", "url", ] @@ -3867,23 +3823,7 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=master#197ff9c6cde7dc0d8334d6b4e27c58779e6a7ce0" -dependencies = [ - "bytes", - "prost", - "prost-build", - "serde", - "serde_json", - "strum 0.26.3", - "strum_macros 0.26.4", - "tonic", - "tonic-build", -] - -[[package]] -name = "helium-sub-daos" -version = "0.1.8" -source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +source = "git+https://github.com/helium/proto?branch=master#376765fe006051d6dcccf709def58e7ed291b845" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -3891,8 +3831,8 @@ dependencies = [ [[package]] name = "helium-sub-daos" -version = "0.1.8" -source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +version = "0.1.5" +source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -3945,17 +3885,8 @@ checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" [[package]] name = "hexboosting" -version = "0.1.0" -source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" -dependencies = [ - "anchor-gen", - "anchor-lang 0.30.1", -] - -[[package]] -name = "hexboosting" -version = "0.1.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +version = "0.0.5" +source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -4765,26 +4696,8 @@ dependencies = [ [[package]] name = "lazy-distributor" -version = "0.2.0" -source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" -dependencies = [ - "anchor-gen", - "anchor-lang 0.30.1", -] - -[[package]] -name = "lazy-distributor" -version = "0.2.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" -dependencies = [ - "anchor-gen", - "anchor-lang 0.30.1", -] - -[[package]] -name = "lazy-transactions" -version = "0.2.0" -source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +version = "0.1.0" +source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -4793,7 +4706,7 @@ dependencies = [ [[package]] name = "lazy-transactions" version = "0.2.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -5168,17 +5081,8 @@ dependencies = [ [[package]] name = "mobile-entity-manager" -version = "0.1.3" -source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" -dependencies = [ - "anchor-gen", - "anchor-lang 0.30.1", -] - -[[package]] -name = "mobile-entity-manager" -version = "0.1.3" -source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +version = "0.1.2" +source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -6033,16 +5937,7 @@ dependencies = [ [[package]] name = "price-oracle" version = "0.2.1" -source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" -dependencies = [ - "anchor-gen", - "anchor-lang 0.30.1", -] - -[[package]] -name = "price-oracle" -version = "0.2.1" -source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -6685,16 +6580,7 @@ dependencies = [ [[package]] name = "rewards-oracle" version = "0.2.0" -source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" -dependencies = [ - "anchor-gen", - "anchor-lang 0.30.1", -] - -[[package]] -name = "rewards-oracle" -version = "0.2.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -8760,16 +8646,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" dependencies = [ - "strum_macros 0.24.3", -] - -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" -dependencies = [ - "strum_macros 0.26.4", + "strum_macros", ] [[package]] @@ -8785,19 +8662,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.58", -] - [[package]] name = "subtle" version = "2.4.1" @@ -9329,16 +9193,7 @@ dependencies = [ [[package]] name = "treasury-management" version = "0.2.0" -source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" -dependencies = [ - "anchor-gen", - "anchor-lang 0.30.1", -] - -[[package]] -name = "treasury-management" -version = "0.2.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -9579,17 +9434,8 @@ dependencies = [ [[package]] name = "voter-stake-registry" -version = "0.3.3" -source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" -dependencies = [ - "anchor-gen", - "anchor-lang 0.30.1", -] - -[[package]] -name = "voter-stake-registry" -version = "0.3.3" -source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +version = "0.3.1" +source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", From 19db98ffd56bfbd7279f05ac63ce65bea5a3ab75 Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Thu, 19 Sep 2024 16:48:56 -0700 Subject: [PATCH 02/31] Supporting material for promotion_fund workspace - ingest promotion rewards, nothing will be done with them until the processor is added into mobile-verifier. - dump reward files - add sp_allocations dummy field to rewarder output - reward indexer mobile promotion type added --- Cargo.lock | 132 ++++++++++++------ .../11_add_mobile_promotion_reward_type.sql | 1 + reward_index/src/indexer.rs | 1 + 3 files changed, 91 insertions(+), 43 deletions(-) create mode 100644 reward_index/migrations/11_add_mobile_promotion_reward_type.sql diff --git a/Cargo.lock b/Cargo.lock index 3cad6df35..5fc339410 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1617,11 +1617,11 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "beacon" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=master#376765fe006051d6dcccf709def58e7ed291b845" +source = "git+https://github.com/helium/proto?branch=map/subscriber-referral#734bd1ef05e50f1a047a1dc28e5d78b24e7deccd" dependencies = [ "base64 0.21.7", "byteorder", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", "prost", "rand 0.8.5", "rand_chacha 0.3.0", @@ -1776,7 +1776,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", "http 0.2.11", "http-serde", "humantime-serde", @@ -2117,7 +2117,7 @@ dependencies = [ [[package]] name = "circuit-breaker" version = "0.1.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" +source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -2618,7 +2618,7 @@ dependencies = [ "axum 0.7.4", "bs58 0.4.0", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", "http 0.2.11", "notify", "serde", @@ -2758,8 +2758,8 @@ dependencies = [ [[package]] name = "data-credits" -version = "0.2.1" -source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" +version = "0.2.2" +source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -3137,7 +3137,7 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "fanout" version = "0.1.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" +source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -3200,7 +3200,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", "hex-literal", "http 0.2.11", "lazy_static", @@ -3215,8 +3215,8 @@ dependencies = [ "serde_json", "sha2 0.10.8", "sqlx", - "strum", - "strum_macros", + "strum 0.24.1", + "strum_macros 0.24.3", "task-manager", "tempfile", "thiserror", @@ -3704,10 +3704,16 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "helium-anchor-gen" version = "0.1.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" +source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -3776,7 +3782,7 @@ dependencies = [ [[package]] name = "helium-entity-manager" version = "0.3.1" -source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" +source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -3785,7 +3791,7 @@ dependencies = [ [[package]] name = "helium-lib" version = "0.0.0" -source = "git+https://github.com/helium/helium-wallet-rs.git?branch=master#a4db666b45a531d690e561c225ca23c503a08bd1" +source = "git+https://github.com/helium/helium-wallet-rs.git?branch=master#b54819ac4c4bd73be37d25d7d6d48842bbc95ea9" dependencies = [ "anchor-client", "anchor-spl", @@ -3798,8 +3804,9 @@ dependencies = [ "h3o", "helium-anchor-gen 0.1.0 (git+https://github.com/helium/helium-anchor-gen.git?branch=main)", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=master)", "hex", + "hex-literal", "itertools", "jsonrpc_client", "lazy_static", @@ -3816,10 +3823,27 @@ dependencies = [ "spl-account-compression", "spl-associated-token-account 3.0.2", "thiserror", + "tonic", "tracing", "url", ] +[[package]] +name = "helium-proto" +version = "0.1.0" +source = "git+https://github.com/helium/proto?branch=map/subscriber-referral#734bd1ef05e50f1a047a1dc28e5d78b24e7deccd" +dependencies = [ + "bytes", + "prost", + "prost-build", + "serde", + "serde_json", + "strum 0.26.3", + "strum_macros 0.26.4", + "tonic", + "tonic-build", +] + [[package]] name = "helium-proto" version = "0.1.0" @@ -3831,8 +3855,8 @@ dependencies = [ [[package]] name = "helium-sub-daos" -version = "0.1.5" -source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" +version = "0.1.8" +source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -3870,7 +3894,7 @@ dependencies = [ "async-trait", "chrono", "derive_builder", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", "hextree", "rust_decimal", "rust_decimal_macros", @@ -3885,8 +3909,8 @@ checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" [[package]] name = "hexboosting" -version = "0.0.5" -source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" +version = "0.1.0" +source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -4286,7 +4310,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", "http 0.2.11", "humantime-serde", "metrics", @@ -4355,7 +4379,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", "hextree", "http 0.2.11", "http-serde", @@ -4397,7 +4421,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", "http 0.2.11", "http-serde", "humantime-serde", @@ -4439,7 +4463,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", "http-serde", "humantime-serde", "iot-config", @@ -4696,8 +4720,8 @@ dependencies = [ [[package]] name = "lazy-distributor" -version = "0.1.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" +version = "0.2.0" +source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -4706,7 +4730,7 @@ dependencies = [ [[package]] name = "lazy-transactions" version = "0.2.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" +source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -5027,7 +5051,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", "hextree", "http 0.2.11", "http-serde", @@ -5067,7 +5091,7 @@ dependencies = [ "futures", "h3o", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", "mobile-config", "prost", "rand 0.8.5", @@ -5081,8 +5105,8 @@ dependencies = [ [[package]] name = "mobile-entity-manager" -version = "0.1.2" -source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" +version = "0.1.3" +source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -5103,7 +5127,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", "http 0.2.11", "http-serde", "humantime-serde", @@ -5147,7 +5171,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", "hex-assignments", "hextree", "http-serde", @@ -5830,7 +5854,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", "http 0.2.11", "hyper 0.14.28", "jsonrpsee", @@ -5913,7 +5937,7 @@ dependencies = [ "futures-util", "helium-anchor-gen 0.1.0 (git+https://github.com/helium/helium-anchor-gen.git)", "helium-lib", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", "humantime-serde", "metrics", "metrics-exporter-prometheus", @@ -5937,7 +5961,7 @@ dependencies = [ [[package]] name = "price-oracle" version = "0.2.1" -source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" +source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -6029,7 +6053,7 @@ dependencies = [ "custom-tracing", "file-store", "futures", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", "humantime-serde", "metrics", "metrics-exporter-prometheus", @@ -6546,7 +6570,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", "humantime-serde", "lazy_static", "metrics", @@ -6580,7 +6604,7 @@ dependencies = [ [[package]] name = "rewards-oracle" version = "0.2.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" +source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -8646,7 +8670,16 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" dependencies = [ - "strum_macros", + "strum_macros 0.24.3", +] + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros 0.26.4", ] [[package]] @@ -8662,6 +8695,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.58", +] + [[package]] name = "subtle" version = "2.4.1" @@ -9193,7 +9239,7 @@ dependencies = [ [[package]] name = "treasury-management" version = "0.2.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" +source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -9434,8 +9480,8 @@ dependencies = [ [[package]] name = "voter-stake-registry" -version = "0.3.1" -source = "git+https://github.com/helium/helium-anchor-gen.git#fe60ed1d49e9255bd779a99bdd7928f278c07256" +version = "0.3.3" +source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", diff --git a/reward_index/migrations/11_add_mobile_promotion_reward_type.sql b/reward_index/migrations/11_add_mobile_promotion_reward_type.sql new file mode 100644 index 000000000..c728749de --- /dev/null +++ b/reward_index/migrations/11_add_mobile_promotion_reward_type.sql @@ -0,0 +1 @@ +ALTER TYPE reward_type ADD VALUE 'mobile_promotion'; diff --git a/reward_index/src/indexer.rs b/reward_index/src/indexer.rs index 8cd612206..56c89bc04 100644 --- a/reward_index/src/indexer.rs +++ b/reward_index/src/indexer.rs @@ -37,6 +37,7 @@ pub enum RewardType { IotOperational, MobileSubscriber, MobileServiceProvider, + MobilePromotion, MobileUnallocated, IotUnallocated, } From ff393359a0aaaade3c50afb83144e85aa4d88b02 Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Mon, 30 Sep 2024 11:55:14 -0700 Subject: [PATCH 03/31] use existing reward_types for reward_indexer Otherwise inserting a new reward would match on the address and continually change the reward_type column for no reason. --- reward_index/migrations/11_add_mobile_promotion_reward_type.sql | 1 - reward_index/src/indexer.rs | 1 - 2 files changed, 2 deletions(-) delete mode 100644 reward_index/migrations/11_add_mobile_promotion_reward_type.sql diff --git a/reward_index/migrations/11_add_mobile_promotion_reward_type.sql b/reward_index/migrations/11_add_mobile_promotion_reward_type.sql deleted file mode 100644 index c728749de..000000000 --- a/reward_index/migrations/11_add_mobile_promotion_reward_type.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TYPE reward_type ADD VALUE 'mobile_promotion'; diff --git a/reward_index/src/indexer.rs b/reward_index/src/indexer.rs index 56c89bc04..8cd612206 100644 --- a/reward_index/src/indexer.rs +++ b/reward_index/src/indexer.rs @@ -37,7 +37,6 @@ pub enum RewardType { IotOperational, MobileSubscriber, MobileServiceProvider, - MobilePromotion, MobileUnallocated, IotUnallocated, } From 97b0c57654fc8b93967432d4715f6cb25c9ed660 Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Mon, 30 Sep 2024 12:54:19 -0700 Subject: [PATCH 04/31] update proto --- Cargo.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5fc339410..92819b246 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1617,7 +1617,7 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "beacon" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=map/subscriber-referral#734bd1ef05e50f1a047a1dc28e5d78b24e7deccd" +source = "git+https://github.com/helium/proto?branch=map/subscriber-referral#effd56a31ab83a1079c7819cd1f783f4141c9c9b" dependencies = [ "base64 0.21.7", "byteorder", @@ -1627,7 +1627,7 @@ dependencies = [ "rand_chacha 0.3.0", "rust_decimal", "serde", - "sha2 0.10.8", + "sha2 0.9.9", "thiserror", ] @@ -3764,7 +3764,7 @@ dependencies = [ "bs58 0.5.0", "byteorder", "ed25519-compact", - "getrandom 0.2.10", + "getrandom 0.1.16", "k256", "lazy_static", "multihash", @@ -3831,7 +3831,7 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=map/subscriber-referral#734bd1ef05e50f1a047a1dc28e5d78b24e7deccd" +source = "git+https://github.com/helium/proto?branch=map/subscriber-referral#effd56a31ab83a1079c7819cd1f783f4141c9c9b" dependencies = [ "bytes", "prost", @@ -5538,7 +5538,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" dependencies = [ - "proc-macro-crate 3.1.0", + "proc-macro-crate 1.1.3", "proc-macro2", "quote", "syn 2.0.58", @@ -6084,7 +6084,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80b776a1b2dc779f5ee0641f8ade0125bc1298dd41a9a0c16d8bd57b42d222b1" dependencies = [ "bytes", - "heck 0.5.0", + "heck 0.4.0", "itertools", "log", "multimap", @@ -9285,7 +9285,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", - "rand 0.8.5", + "rand 0.7.3", "static_assertions", ] @@ -9940,7 +9940,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "sha2 0.10.8", + "sha2 0.9.9", "thiserror", "twox-hash", "xorf", From 882aadbdfbb29df4fc45c9cd6fa5263563957e1b Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Thu, 19 Sep 2024 19:52:02 -0700 Subject: [PATCH 05/31] Add Continuous struct helper for file_source Default trait impls cannot be added to functions, but they can be added to structs. This struct keeps us from needing to do the gross blank generic filling in when we want a continuous file source with a decode other than MsgDecodeFileInfoPollerParser. --- file_store/src/file_source.rs | 36 +++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/file_store/src/file_source.rs b/file_store/src/file_source.rs index a028d2d8b..511a46ea1 100644 --- a/file_store/src/file_source.rs +++ b/file_store/src/file_source.rs @@ -1,5 +1,7 @@ use crate::{ - file_info_poller::{FileInfoPollerConfigBuilder, MsgDecodeFileInfoPollerParser}, + file_info_poller::{ + FileInfoPollerConfigBuilder, MsgDecodeFileInfoPollerParser, ProstFileInfoPollerParser, + }, file_sink, BytesMutStream, Error, FileStore, }; use async_compression::tokio::bufread::GzipDecoder; @@ -7,17 +9,43 @@ use futures::{ stream::{self}, StreamExt, TryFutureExt, TryStreamExt, }; -use std::path::{Path, PathBuf}; +use std::{ + marker::PhantomData, + path::{Path, PathBuf}, +}; use tokio::{fs::File, io::BufReader}; use tokio_util::codec::{length_delimited::LengthDelimitedCodec, FramedRead}; +pub struct Continuous(PhantomData<(Store, Parser)>); + +impl Continuous { + pub fn msg_source( + ) -> FileInfoPollerConfigBuilder + where + Msg: Clone, + { + FileInfoPollerConfigBuilder::::default() + .parser(MsgDecodeFileInfoPollerParser) + } +} + +impl Continuous { + pub fn prost_source( + ) -> FileInfoPollerConfigBuilder + where + Msg: Clone, + { + FileInfoPollerConfigBuilder::::default() + .parser(ProstFileInfoPollerParser) + } +} + pub fn continuous_source( ) -> FileInfoPollerConfigBuilder where T: Clone, { - FileInfoPollerConfigBuilder::::default() - .parser(MsgDecodeFileInfoPollerParser) + Continuous::msg_source::() } pub fn source(paths: I) -> BytesMutStream From eb97f571ae3ebd7b07ae5c90e72be52ad9355288 Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Thu, 19 Sep 2024 19:53:29 -0700 Subject: [PATCH 06/31] Add the Service Provider Promotions Daemon --- .../migrations/37_sp_promotions.sql | 21 ++ mobile_verifier/src/cli/server.rs | 15 +- mobile_verifier/src/lib.rs | 7 + mobile_verifier/src/sp_promotions/daemon.rs | 260 ++++++++++++++++++ mobile_verifier/src/sp_promotions/funds.rs | 92 +++++++ mobile_verifier/src/sp_promotions/mod.rs | 11 + mobile_verifier/src/sp_promotions/rewards.rs | 139 ++++++++++ 7 files changed, 544 insertions(+), 1 deletion(-) create mode 100644 mobile_verifier/migrations/37_sp_promotions.sql create mode 100644 mobile_verifier/src/sp_promotions/daemon.rs create mode 100644 mobile_verifier/src/sp_promotions/funds.rs create mode 100644 mobile_verifier/src/sp_promotions/mod.rs create mode 100644 mobile_verifier/src/sp_promotions/rewards.rs diff --git a/mobile_verifier/migrations/37_sp_promotions.sql b/mobile_verifier/migrations/37_sp_promotions.sql new file mode 100644 index 000000000..1b55e8cc4 --- /dev/null +++ b/mobile_verifier/migrations/37_sp_promotions.sql @@ -0,0 +1,21 @@ +CREATE TABLE IF NOT EXISTS subscriber_promotion_rewards ( + time_of_reward TIMESTAMPTZ NOT NULL, + subscriber_id BYTEA NOT NULL, + carrier_key TEXT NOT NULL, + shares BIGINT NOT NULL, + PRIMARY KEY (time_of_reward, subscriber_id, carrier_key) +); + +CREATE TABLE IF NOT EXISTS gateway_promotion_rewards ( + time_of_reward TIMESTAMPTZ NOT NULL, + gateway_key TEXT NOT NULL, + carrier_key TEXT NOT NULL, + shares BIGINT NOT NULL, + PRIMARY KEY (time_of_reward, gateway_key, carrier_key) +); + +CREATE TABLE IF NOT EXISTS service_provider_promotion_funds ( + service_provider BIGINT NOT NULL PRIMARY KEY, + basis_points BIGINT NOT NULL, + inserted_at TIMESTAMPTZ NOT NULL +); diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index 193455c5c..d89765343 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -9,6 +9,7 @@ use crate::{ radio_threshold::RadioThresholdIngestor, rewarder::Rewarder, sp_boosted_rewards_bans::ServiceProviderBoostedRewardsBanIngestor, + sp_promotions, speedtests::SpeedtestDaemon, subscriber_location::SubscriberLocationIngestor, subscriber_verified_mapping_event::SubscriberVerifiedMappingEventDaemon, @@ -137,7 +138,7 @@ impl Cmd { file_upload.clone(), report_ingest.clone(), speedtests_avg.clone(), - gateway_client, + gateway_client.clone(), ) .await?, ) @@ -193,6 +194,18 @@ impl Cmd { ) .await?, ) + .add_task( + sp_promotions::SpPromotionDaemon::create_managed_task( + pool.clone(), + settings, + file_upload.clone(), + report_ingest.clone(), + gateway_client.clone(), + auth_client.clone(), + entity_client.clone(), + ) + .await?, + ) .add_task(DataSessionIngestor::create_managed_task(pool.clone(), settings).await?) .add_task( ServiceProviderBoostedRewardsBanIngestor::create_managed_task( diff --git a/mobile_verifier/src/lib.rs b/mobile_verifier/src/lib.rs index 61cb44a9f..4e131926c 100644 --- a/mobile_verifier/src/lib.rs +++ b/mobile_verifier/src/lib.rs @@ -11,6 +11,7 @@ pub mod rewarder; pub mod seniority; mod settings; pub mod sp_boosted_rewards_bans; +pub mod sp_promotions; pub mod speedtests; pub mod speedtests_average; pub mod subscriber_location; @@ -29,6 +30,12 @@ pub enum GatewayResolution { DataOnly, } +impl GatewayResolution { + pub fn is_not_found(&self) -> bool { + matches!(self, GatewayResolution::GatewayNotFound) + } +} + #[async_trait::async_trait] pub trait GatewayResolver: Clone + Send + Sync + 'static { type Error: Error + Send + Sync + 'static; diff --git a/mobile_verifier/src/sp_promotions/daemon.rs b/mobile_verifier/src/sp_promotions/daemon.rs new file mode 100644 index 000000000..ec74b44c1 --- /dev/null +++ b/mobile_verifier/src/sp_promotions/daemon.rs @@ -0,0 +1,260 @@ +use std::time::{Duration, Instant}; + +use chrono::Utc; +use file_store::{ + file_info_poller::{FileInfoStream, LookbackBehavior}, + file_sink::FileSinkClient, + file_source, + file_upload::FileUpload, + promotion_reward::{Entity, PromotionReward}, + traits::{FileSinkCommitStrategy, FileSinkRollTime, FileSinkWriteExt, TimestampEncode}, + FileType, +}; +use futures::{StreamExt, TryFutureExt}; +use helium_proto::{ + services::{ + mobile_config::NetworkKeyRole, + poc_mobile::{ + PromotionRewardIngestReportV1, PromotionRewardStatus, VerifiedPromotionRewardV1, + }, + }, + ServiceProviderPromotionFundV1, +}; +use mobile_config::{ + client::{ + authorization_client::AuthorizationVerifier, entity_client::EntityVerifier, + AuthorizationClient, EntityClient, + }, + GatewayClient, +}; +use sqlx::PgPool; +use task_manager::{ManagedTask, TaskManager}; +use tokio::sync::mpsc::Receiver; + +use crate::{ + sp_promotions::{funds, rewards}, + GatewayResolver, Settings, +}; + +pub struct SpPromotionDaemon { + pool: PgPool, + gateway_info_resolver: GatewayClient, + authorization_verifier: AuthorizationClient, + entity_verifier: EntityClient, + promotion_funds: Receiver>, + promotion_rewards: Receiver>, + promotion_rewards_sink: FileSinkClient, +} + +impl ManagedTask for SpPromotionDaemon { + fn start_task( + self: Box, + shutdown: triggered::Listener, + ) -> futures::prelude::future::LocalBoxFuture<'static, anyhow::Result<()>> { + let handle = tokio::spawn(self.run(shutdown)); + Box::pin( + handle + .map_err(anyhow::Error::from) + .and_then(|result| async move { result.map_err(anyhow::Error::from) }), + ) + } +} + +impl SpPromotionDaemon { + pub async fn create_managed_task( + pool: PgPool, + settings: &Settings, + file_upload: FileUpload, + file_store: file_store::FileStore, + gateway_info_resolver: GatewayClient, + authorization_verifier: AuthorizationClient, + entity_verifier: EntityClient, + ) -> anyhow::Result { + let (promotion_rewards_sink, valid_promotion_rewards_server) = + VerifiedPromotionRewardV1::file_sink( + settings.store_base_path(), + file_upload.clone(), + FileSinkCommitStrategy::Automatic, + FileSinkRollTime::Duration(Duration::from_secs(15 * 60)), + env!("CARGO_PKG_NAME"), + ) + .await?; + + let (promotion_rewards, promotion_rewards_server) = + file_source::Continuous::msg_source::() + .state(pool.clone()) + .store(file_store.clone()) + .lookback(LookbackBehavior::StartAfter(settings.start_after)) + .prefix(FileType::PromotionRewardIngestReport.to_string()) + .create() + .await?; + + let (promotion_funds, promotion_funds_server) = + file_source::Continuous::prost_source::() + .state(pool.clone()) + .store(file_store) + .lookback(LookbackBehavior::StartAfter(settings.start_after)) + .prefix(FileType::ServiceProviderPromotionFund.to_string()) + .create() + .await?; + + let promotion_reward_daemon = Self { + pool, + gateway_info_resolver, + authorization_verifier, + entity_verifier, + promotion_funds, + promotion_rewards, + promotion_rewards_sink, + }; + + Ok(TaskManager::builder() + .add_task(valid_promotion_rewards_server) + .add_task(promotion_funds_server) + .add_task(promotion_rewards_server) + .add_task(promotion_reward_daemon) + .build()) + } + + async fn run(mut self, shutdown: triggered::Listener) -> anyhow::Result<()> { + loop { + tokio::select! { + _ = shutdown.clone() => { + tracing::info!("PromotionRewardDaemon shutting down"); + break; + } + Some(file) = self.promotion_rewards.recv() => { + let start = Instant::now(); + self.process_rewards_file(file).await?; + metrics::histogram!("promotion_reward_processing_time").record(start.elapsed()); + } + Some(file) = self.promotion_funds.recv() => { + let start = Instant::now(); + self.process_funds_file(file).await?; + metrics::histogram!("promotion_funds_processing_time").record(start.elapsed()); + } + } + } + + Ok(()) + } + + async fn process_rewards_file( + &self, + file: FileInfoStream, + ) -> anyhow::Result<()> { + tracing::info!(key = file.file_info.key, "Processing promotion reward file"); + + let mut transaction = self.pool.begin().await?; + let mut promotion_rewards = file.into_stream(&mut transaction).await?; + + while let Some(promotion_reward) = promotion_rewards.next().await { + let promotion_reward_status = validate_promotion_reward( + &promotion_reward, + &self.authorization_verifier, + &self.gateway_info_resolver, + &self.entity_verifier, + ) + .await?; + + if promotion_reward_status == PromotionRewardStatus::Valid { + rewards::save_promotion_reward(&mut transaction, &promotion_reward).await?; + } + + write_promotion_reward( + &self.promotion_rewards_sink, + &promotion_reward, + promotion_reward_status, + ) + .await?; + } + + self.promotion_rewards_sink.commit().await?; + transaction.commit().await?; + + Ok(()) + } + + async fn process_funds_file( + &self, + file: FileInfoStream, + ) -> anyhow::Result<()> { + tracing::info!(key = file.file_info.key, "Processing promotion funds file"); + + let mut txn = self.pool.begin().await?; + + let mut promotion_funds = file.into_stream(&mut txn).await?; + while let Some(promotion_fund) = promotion_funds.next().await { + funds::save_promotion_fund( + &mut txn, + promotion_fund.service_provider, + promotion_fund.bps as u16, + ) + .await?; + } + + txn.commit().await?; + + Ok(()) + } +} + +async fn validate_promotion_reward( + promotion_reward: &PromotionReward, + authorization_verifier: &impl AuthorizationVerifier, + gateway_info_resolver: &impl GatewayResolver, + entity_verifier: &impl EntityVerifier, +) -> anyhow::Result { + if authorization_verifier + .verify_authorized_key( + &promotion_reward.carrier_pub_key, + NetworkKeyRole::MobileCarrier, + ) + .await + .is_err() + { + return Ok(PromotionRewardStatus::InvalidCarrierKey); + } + match &promotion_reward.entity { + Entity::SubscriberId(ref subscriber_id) + if entity_verifier + .verify_rewardable_entity(subscriber_id) + .await + .is_err() => + { + Ok(PromotionRewardStatus::InvalidSubscriberId) + } + Entity::GatewayKey(ref gateway_key) + if gateway_info_resolver + .resolve_gateway(gateway_key) + .await? + .is_not_found() => + { + Ok(PromotionRewardStatus::InvalidGatewayKey) + } + _ => Ok(PromotionRewardStatus::Valid), + } +} + +async fn write_promotion_reward( + file_sink: &FileSinkClient, + promotion_reward: &PromotionReward, + status: PromotionRewardStatus, +) -> anyhow::Result<()> { + file_sink + .write( + VerifiedPromotionRewardV1 { + report: Some(PromotionRewardIngestReportV1 { + received_timestamp: promotion_reward + .received_timestamp + .encode_timestamp_millis(), + report: Some(promotion_reward.clone().into()), + }), + status: status as i32, + timestamp: Utc::now().encode_timestamp_millis(), + }, + &[("validity", status.as_str_name())], + ) + .await?; + Ok(()) +} diff --git a/mobile_verifier/src/sp_promotions/funds.rs b/mobile_verifier/src/sp_promotions/funds.rs new file mode 100644 index 000000000..c9359f6b9 --- /dev/null +++ b/mobile_verifier/src/sp_promotions/funds.rs @@ -0,0 +1,92 @@ +use std::collections::HashMap; + +use chrono::Utc; +use rust_decimal::Decimal; +use rust_decimal_macros::dec; +use sqlx::{PgPool, Postgres, Transaction}; + +use super::ServiceProviderId; + +#[derive(Debug, Default)] +pub struct ServiceProviderFunds(HashMap); + +impl ServiceProviderFunds { + pub fn fetch_incentive_escrow_fund_percent( + &self, + service_provider_id: ServiceProviderId, + ) -> Decimal { + let bps = self + .0 + .get(&service_provider_id) + .cloned() + .unwrap_or_default(); + Decimal::from(bps) / dec!(10_000) + } +} + +pub async fn get_promotion_funds(pool: &PgPool) -> anyhow::Result { + #[derive(Debug, sqlx::FromRow)] + struct PromotionFund { + #[sqlx(try_from = "i64")] + pub service_provider: ServiceProviderId, + #[sqlx(try_from = "i64")] + pub basis_points: u16, + } + + let funds = sqlx::query_as::<_, PromotionFund>( + r#" + SELECT + service_provider, basis_points + FROM + service_provider_promotion_funds + "#, + ) + .fetch_all(pool) + .await?; + + let funds = funds + .into_iter() + .map(|fund| (fund.service_provider, fund.basis_points)) + .collect(); + + Ok(ServiceProviderFunds(funds)) +} + +pub async fn save_promotion_fund( + transaction: &mut Transaction<'_, Postgres>, + service_provider_id: ServiceProviderId, + basis_points: u16, +) -> anyhow::Result<()> { + sqlx::query( + r#" + INSERT INTO service_provider_promotion_funds + (service_provider, basis_points, inserted_at) + VALUES + ($1, $2, $3) + "#, + ) + .bind(service_provider_id) + .bind(basis_points as i64) + .bind(Utc::now()) + .execute(transaction) + .await?; + + Ok(()) +} + +pub async fn delete_promotion_fund( + pool: &PgPool, + service_provider_id: ServiceProviderId, +) -> anyhow::Result<()> { + sqlx::query( + r#" + DELETE FROM service_provider_promotion_funds + WHERE service_provider = $1 + "#, + ) + .bind(service_provider_id) + .execute(pool) + .await?; + + Ok(()) +} diff --git a/mobile_verifier/src/sp_promotions/mod.rs b/mobile_verifier/src/sp_promotions/mod.rs new file mode 100644 index 000000000..e98464a09 --- /dev/null +++ b/mobile_verifier/src/sp_promotions/mod.rs @@ -0,0 +1,11 @@ +pub use daemon::SpPromotionDaemon; +pub use rewards::clear_promotion_rewards; + +pub mod daemon; +pub mod funds; +pub mod rewards; + +// This type is used in lieu of the helium_proto::ServiceProvider enum so we can +// handle more than a single value without adding a hard deploy dependency to +// mobile-verifier when a new carrier is added.. +pub type ServiceProviderId = i32; diff --git a/mobile_verifier/src/sp_promotions/rewards.rs b/mobile_verifier/src/sp_promotions/rewards.rs new file mode 100644 index 000000000..6eb1a3b60 --- /dev/null +++ b/mobile_verifier/src/sp_promotions/rewards.rs @@ -0,0 +1,139 @@ +use std::ops::Range; + +use chrono::{DateTime, Utc}; +use file_store::promotion_reward::{Entity, PromotionReward}; +use futures::TryStreamExt; +use helium_crypto::PublicKeyBinary; +use mobile_config::client::{carrier_service_client::CarrierServiceVerifier, ClientError}; +use sqlx::{postgres::PgRow, PgPool, Postgres, Row, Transaction}; + +use super::ServiceProviderId; + +pub async fn save_promotion_reward( + transaction: &mut Transaction<'_, Postgres>, + promotion_reward: &PromotionReward, +) -> anyhow::Result<()> { + match &promotion_reward.entity { + Entity::SubscriberId(subscriber_id) => { + sqlx::query( + r#" + INSERT INTO subscriber_promotion_rewards (time_of_reward, subscriber_id, carrier_key, shares) + VALUES ($1, $2, $3, $4) + ON CONFLICT DO NOTHING + "# + ) + .bind(promotion_reward.timestamp) + .bind(subscriber_id) + .bind(&promotion_reward.carrier_pub_key) + .bind(promotion_reward.shares as i64) + .execute(&mut *transaction) + .await?; + } + Entity::GatewayKey(gateway_key) => { + sqlx::query( + r#" + INSERT INTO gateway_promotion_rewards (time_of_reward, gateway_key, carrier_key, shares) + VALUES ($1, $2, $3, $4) + ON CONFLICT DO NOTHING + "# + ) + .bind(promotion_reward.timestamp) + .bind(gateway_key) + .bind(&promotion_reward.carrier_pub_key) + .bind(promotion_reward.shares as i64) + .execute(&mut *transaction) + .await?; + } + } + Ok(()) +} + +pub async fn get_promotion_rewards( + pool: &PgPool, + carrier: &impl CarrierServiceVerifier, + epoch: &Range>, +) -> anyhow::Result> { + let rewards = sqlx::query_as( + r#" + SELECT + subscriber_id, NULL as gateway_key, SUM(shares)::bigint as shares, carrier_key + FROM + subscriber_promotion_rewards + WHERE + time_of_reward >= $1 AND time_of_reward < $2 + GROUP BY + subscriber_id, carrier_key + UNION + SELECT + NULL as subscriber_id, gateway_key, SUM(shares)::bigint as shares, carrier_key + FROM + gateway_promotion_rewards + WHERE + time_of_reward >= $1 AND time_of_reward < $2 + GROUP + BY gateway_key, carrier_key + "#, + ) + .bind(epoch.start) + .bind(epoch.end) + .fetch(pool) + .map_err(anyhow::Error::from) + .and_then(|x: PromotionRewardShares| async move { + let service_provider_id = carrier + .payer_key_to_service_provider(&x.carrier_key.to_string()) + .await?; + Ok(ServiceProviderPromotionRewardShares { + service_provider_id: service_provider_id as ServiceProviderId, + rewardable_entity: x.rewardable_entity, + shares: x.shares, + }) + }) + .try_collect() + .await?; + + Ok(rewards) +} + +pub async fn clear_promotion_rewards( + tx: &mut Transaction<'_, Postgres>, + timestamp: &DateTime, +) -> Result<(), sqlx::Error> { + sqlx::query("DELETE FROM subscriber_promotion_rewards WHERE time_of_reward < $1") + .bind(timestamp) + .execute(&mut *tx) + .await?; + sqlx::query("DELETE FROM gateway_promotion_rewards WHERE time_of_reward < $1") + .bind(timestamp) + .execute(&mut *tx) + .await?; + Ok(()) +} + +struct PromotionRewardShares { + pub carrier_key: PublicKeyBinary, + pub rewardable_entity: Entity, + pub shares: u64, +} + +impl sqlx::FromRow<'_, PgRow> for PromotionRewardShares { + fn from_row(row: &PgRow) -> sqlx::Result { + let subscriber_id: Option> = row.try_get("subscriber_id")?; + let shares: i64 = row.try_get("shares")?; + Ok(Self { + rewardable_entity: if let Some(subscriber_id) = subscriber_id { + Entity::SubscriberId(subscriber_id) + } else { + Entity::GatewayKey(row.try_get("gateway_key")?) + }, + shares: shares as u64, + carrier_key: row.try_get("carrier_key")?, + }) + } +} + +#[derive(Debug)] +pub struct ServiceProviderPromotionRewardShares { + pub service_provider_id: ServiceProviderId, + pub rewardable_entity: Entity, + pub shares: u64, +} From 754a380fab787b3ca78203e57a6ab99455b96168 Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Thu, 19 Sep 2024 21:46:04 -0700 Subject: [PATCH 07/31] Swap over Service Provider rewards to use percents Rather than always dealing with conrete numbers, now that Service Providers can allocate a percentage of their rewards to promotions for subscribers or gateways, it becomes easier to speak of everything in percentages. This is a quick overview of what a more detailed overview you will find in the PR. For every Service Provider (SP) we figure out how much of the total rewards allocation they are being awarded for data transfer. We get the percent they have allocated for promotions (essentially from solana, but really s3), and we determine which percentage of the _total rewards allocation_ that is. If a SP is getting 50% of the total rewards, and they allocate 20% of those rewards to promotions, then the SP will receive 40% of the total, and promotions from them will represent 10% of the total. We do that for all SPs. The unallocated percentage is then distributed to the promotions of each SP. If there is more than enough unallocated left over, each SP get's a matched percentage of the whole to what they set aside. When there is not enough, they get a percentage equal to their initial rewards percentage. This is to keep a service provider from getting the bulk of extra rewards by settings aside a large amount for promotions, and receiving little in rewards for data transfer, but getting more for matching. A SP may never receive more in matched rewards than they have allocated themselves. --- mobile_verifier/src/cli/server.rs | 4 +- mobile_verifier/src/lib.rs | 2 +- mobile_verifier/src/reward_shares.rs | 12 +- mobile_verifier/src/rewarder.rs | 49 +- .../src/service_provider/dc_sessions.rs | 89 ++++ mobile_verifier/src/service_provider/mod.rs | 25 ++ .../promotions}/daemon.rs | 8 +- .../promotions}/funds.rs | 11 +- .../src/service_provider/promotions/mod.rs | 3 + .../promotions}/rewards.rs | 28 +- .../src/service_provider/reward.rs | 422 ++++++++++++++++++ mobile_verifier/src/sp_promotions/mod.rs | 11 - 12 files changed, 594 insertions(+), 70 deletions(-) create mode 100644 mobile_verifier/src/service_provider/dc_sessions.rs create mode 100644 mobile_verifier/src/service_provider/mod.rs rename mobile_verifier/src/{sp_promotions => service_provider/promotions}/daemon.rs (98%) rename mobile_verifier/src/{sp_promotions => service_provider/promotions}/funds.rs (85%) create mode 100644 mobile_verifier/src/service_provider/promotions/mod.rs rename mobile_verifier/src/{sp_promotions => service_provider/promotions}/rewards.rs (91%) create mode 100644 mobile_verifier/src/service_provider/reward.rs delete mode 100644 mobile_verifier/src/sp_promotions/mod.rs diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index d89765343..d325581c7 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -8,8 +8,8 @@ use crate::{ heartbeats::{cbrs::CbrsHeartbeatDaemon, wifi::WifiHeartbeatDaemon}, radio_threshold::RadioThresholdIngestor, rewarder::Rewarder, + service_provider, sp_boosted_rewards_bans::ServiceProviderBoostedRewardsBanIngestor, - sp_promotions, speedtests::SpeedtestDaemon, subscriber_location::SubscriberLocationIngestor, subscriber_verified_mapping_event::SubscriberVerifiedMappingEventDaemon, @@ -195,7 +195,7 @@ impl Cmd { .await?, ) .add_task( - sp_promotions::SpPromotionDaemon::create_managed_task( + service_provider::PromotionDaemon::create_managed_task( pool.clone(), settings, file_upload.clone(), diff --git a/mobile_verifier/src/lib.rs b/mobile_verifier/src/lib.rs index 4e131926c..9fc3757c0 100644 --- a/mobile_verifier/src/lib.rs +++ b/mobile_verifier/src/lib.rs @@ -9,9 +9,9 @@ pub mod radio_threshold; pub mod reward_shares; pub mod rewarder; pub mod seniority; +pub mod service_provider; mod settings; pub mod sp_boosted_rewards_bans; -pub mod sp_promotions; pub mod speedtests; pub mod speedtests_average; pub mod subscriber_location; diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index e5f2d2d00..ee2cd2e98 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -1,11 +1,7 @@ use crate::{ - coverage::CoveredHexStream, - data_session::{HotspotMap, ServiceProviderDataSession}, - heartbeats::HeartbeatReward, - rewarder::boosted_hex_eligibility::BoostedHexEligibility, - seniority::Seniority, - sp_boosted_rewards_bans::BannedRadios, - speedtests_average::SpeedtestAverages, + coverage::CoveredHexStream, data_session::HotspotMap, heartbeats::HeartbeatReward, + rewarder::boosted_hex_eligibility::BoostedHexEligibility, seniority::Seniority, + sp_boosted_rewards_bans::BannedRadios, speedtests_average::SpeedtestAverages, subscriber_location::SubscriberValidatedLocations, subscriber_verified_mapping_event::VerifiedSubscriberVerifiedMappingEventShares, }; @@ -52,7 +48,7 @@ const BOOSTED_POC_REWARDS_PERCENT: Decimal = dec!(0.1); const DC_USD_PRICE: Decimal = dec!(0.00001); /// Default precision used for rounding -const DEFAULT_PREC: u32 = 15; +pub const DEFAULT_PREC: u32 = 15; /// Percent of total emissions allocated for mapper rewards const MAPPERS_REWARDS_PERCENT: Decimal = dec!(0.2); diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index 055b4a399..4e7b9026f 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -5,9 +5,9 @@ use crate::{ radio_threshold, reward_shares::{ self, CalculatedPocRewardShares, CoverageShares, DataTransferAndPocAllocatedRewardBuckets, - MapperShares, ServiceProviderShares, TransferRewards, + MapperShares, TransferRewards, }, - sp_boosted_rewards_bans, speedtests, + service_provider, sp_boosted_rewards_bans, speedtests, speedtests_average::SpeedtestAverages, subscriber_location, subscriber_verified_mapping_event, telemetry, Settings, }; @@ -596,33 +596,36 @@ pub async fn reward_service_providers( reward_period: &Range>, mobile_bone_price: Decimal, ) -> anyhow::Result<()> { - let payer_dc_sessions = - data_session::sum_data_sessions_to_dc_by_payer(pool, reward_period).await?; - let sp_shares = - ServiceProviderShares::from_payers_dc(payer_dc_sessions, carrier_client).await?; - let total_sp_rewards = reward_shares::get_scheduled_tokens_for_service_providers( - reward_period.end - reward_period.start, + use service_provider::db; + let dc_sessions = db::fetch_dc_sessions(pool, carrier_client, reward_period).await?; + let promo_funds = db::fetch_promotion_funds(pool).await?; + let promo_rewards = db::fetch_promotion_rewards(pool, carrier_client, reward_period).await?; + + let total_sp_rewards = service_provider::get_scheduled_tokens(reward_period); + + let sps = service_provider::RewardInfoColl::new( + dc_sessions, + promo_funds, + promo_rewards, + total_sp_rewards, + mobile_bone_price, ); - let rewards_per_share = sp_shares.rewards_per_share(total_sp_rewards, mobile_bone_price)?; - // translate service provider shares into service provider rewards - // track the amount of allocated reward value as we go - let mut allocated_sp_rewards = 0_u64; - for (amount, sp_share) in - sp_shares.into_service_provider_rewards(reward_period, rewards_per_share) - { - allocated_sp_rewards += amount; - mobile_rewards.write(sp_share.clone(), []).await?.await??; - } - // write out any unallocated service provider reward - let unallocated_sp_reward_amount = total_sp_rewards + + let mut unallocated_sp_rewards = total_sp_rewards .round_dp_with_strategy(0, RoundingStrategy::ToZero) .to_u64() - .unwrap_or(0) - - allocated_sp_rewards; + .unwrap_or(0); + + for (amount, reward) in sps.iter_rewards(reward_period) { + unallocated_sp_rewards -= amount; + mobile_rewards.write(reward, []).await?.await??; + } + + // write out any unallocated service provider reward write_unallocated_reward( mobile_rewards, UnallocatedRewardType::ServiceProvider, - unallocated_sp_reward_amount, + unallocated_sp_rewards, reward_period, ) .await?; diff --git a/mobile_verifier/src/service_provider/dc_sessions.rs b/mobile_verifier/src/service_provider/dc_sessions.rs new file mode 100644 index 000000000..37dd6e45d --- /dev/null +++ b/mobile_verifier/src/service_provider/dc_sessions.rs @@ -0,0 +1,89 @@ +use std::{collections::HashMap, ops::Range}; + +use chrono::{DateTime, Utc}; +use mobile_config::client::{carrier_service_client::CarrierServiceVerifier, ClientError}; +use rust_decimal::{Decimal, RoundingStrategy}; +use sqlx::PgPool; + +use crate::{ + data_session, + reward_shares::{dc_to_mobile_bones, DEFAULT_PREC}, +}; + +use super::ServiceProviderId; + +pub async fn fetch_dc_sessions( + pool: &PgPool, + carrier_client: &impl CarrierServiceVerifier, + reward_period: &Range>, +) -> anyhow::Result { + let payer_dc_sessions = + data_session::sum_data_sessions_to_dc_by_payer(pool, reward_period).await?; + + let mut dc_sessions = ServiceProviderDCSessions::default(); + for (payer_key, dc_amount) in payer_dc_sessions { + let service_provider = carrier_client + .payer_key_to_service_provider(&payer_key) + .await?; + dc_sessions.insert( + service_provider as ServiceProviderId, + Decimal::from(dc_amount), + ); + } + + Ok(dc_sessions) +} + +#[derive(Debug, Default)] +pub struct ServiceProviderDCSessions(pub(crate) HashMap); + +impl ServiceProviderDCSessions { + pub fn insert(&mut self, service_provider: ServiceProviderId, dc: Decimal) { + self.0.insert(service_provider, dc); + } + + pub fn all_transfer(&self) -> Decimal { + self.0.values().sum() + } + + pub fn iter(&self) -> impl Iterator + '_ { + self.0.iter().map(|(k, v)| (*k, *v)) + } + + pub fn rewards_per_share( + &self, + total_sp_rewards: Decimal, + mobile_bone_price: Decimal, + ) -> anyhow::Result { + // the total amount of DC spent across all service providers + let total_sp_dc = self.all_transfer(); + // the total amount of service provider rewards in bones based on the spent DC + let total_sp_rewards_used = dc_to_mobile_bones(total_sp_dc, mobile_bone_price); + // cap the service provider rewards if used > pool total + let capped_sp_rewards_used = + Self::maybe_cap_service_provider_rewards(total_sp_rewards_used, total_sp_rewards); + Ok(Self::calc_rewards_per_share( + capped_sp_rewards_used, + total_sp_dc, + )) + } + + fn maybe_cap_service_provider_rewards( + total_sp_rewards_used: Decimal, + total_sp_rewards: Decimal, + ) -> Decimal { + match total_sp_rewards_used <= total_sp_rewards { + true => total_sp_rewards_used, + false => total_sp_rewards, + } + } + + fn calc_rewards_per_share(total_rewards: Decimal, total_shares: Decimal) -> Decimal { + if total_shares > Decimal::ZERO { + (total_rewards / total_shares) + .round_dp_with_strategy(DEFAULT_PREC, RoundingStrategy::MidpointNearestEven) + } else { + Decimal::ZERO + } + } +} diff --git a/mobile_verifier/src/service_provider/mod.rs b/mobile_verifier/src/service_provider/mod.rs new file mode 100644 index 000000000..592d2a415 --- /dev/null +++ b/mobile_verifier/src/service_provider/mod.rs @@ -0,0 +1,25 @@ +use std::ops::Range; + +use chrono::{DateTime, Utc}; +pub use promotions::daemon::PromotionDaemon; +pub use reward::RewardInfoColl; + +pub mod dc_sessions; +pub mod promotions; +pub mod reward; + +pub mod db { + pub use super::dc_sessions::fetch_dc_sessions; + pub use super::promotions::funds::fetch_promotion_funds; + pub use super::promotions::rewards::fetch_promotion_rewards; +} + +// This type is used in lieu of the helium_proto::ServiceProvider enum so we can +// handle more than a single value without adding a hard deploy dependency to +// mobile-verifier when a new carrier is added.. +pub type ServiceProviderId = i32; + +pub fn get_scheduled_tokens(reward_period: &Range>) -> rust_decimal::Decimal { + let duration = reward_period.end - reward_period.start; + crate::reward_shares::get_scheduled_tokens_for_service_providers(duration) +} diff --git a/mobile_verifier/src/sp_promotions/daemon.rs b/mobile_verifier/src/service_provider/promotions/daemon.rs similarity index 98% rename from mobile_verifier/src/sp_promotions/daemon.rs rename to mobile_verifier/src/service_provider/promotions/daemon.rs index ec74b44c1..aea62dc0f 100644 --- a/mobile_verifier/src/sp_promotions/daemon.rs +++ b/mobile_verifier/src/service_provider/promotions/daemon.rs @@ -32,11 +32,11 @@ use task_manager::{ManagedTask, TaskManager}; use tokio::sync::mpsc::Receiver; use crate::{ - sp_promotions::{funds, rewards}, + service_provider::promotions::{funds, rewards}, GatewayResolver, Settings, }; -pub struct SpPromotionDaemon { +pub struct PromotionDaemon { pool: PgPool, gateway_info_resolver: GatewayClient, authorization_verifier: AuthorizationClient, @@ -46,7 +46,7 @@ pub struct SpPromotionDaemon { promotion_rewards_sink: FileSinkClient, } -impl ManagedTask for SpPromotionDaemon { +impl ManagedTask for PromotionDaemon { fn start_task( self: Box, shutdown: triggered::Listener, @@ -60,7 +60,7 @@ impl ManagedTask for SpPromotionDaemon { } } -impl SpPromotionDaemon { +impl PromotionDaemon { pub async fn create_managed_task( pool: PgPool, settings: &Settings, diff --git a/mobile_verifier/src/sp_promotions/funds.rs b/mobile_verifier/src/service_provider/promotions/funds.rs similarity index 85% rename from mobile_verifier/src/sp_promotions/funds.rs rename to mobile_verifier/src/service_provider/promotions/funds.rs index c9359f6b9..ab7a49671 100644 --- a/mobile_verifier/src/sp_promotions/funds.rs +++ b/mobile_verifier/src/service_provider/promotions/funds.rs @@ -5,16 +5,13 @@ use rust_decimal::Decimal; use rust_decimal_macros::dec; use sqlx::{PgPool, Postgres, Transaction}; -use super::ServiceProviderId; +use crate::service_provider::ServiceProviderId; #[derive(Debug, Default)] -pub struct ServiceProviderFunds(HashMap); +pub struct ServiceProviderFunds(pub(crate) HashMap); impl ServiceProviderFunds { - pub fn fetch_incentive_escrow_fund_percent( - &self, - service_provider_id: ServiceProviderId, - ) -> Decimal { + pub fn get_fund_percent(&self, service_provider_id: ServiceProviderId) -> Decimal { let bps = self .0 .get(&service_provider_id) @@ -24,7 +21,7 @@ impl ServiceProviderFunds { } } -pub async fn get_promotion_funds(pool: &PgPool) -> anyhow::Result { +pub async fn fetch_promotion_funds(pool: &PgPool) -> anyhow::Result { #[derive(Debug, sqlx::FromRow)] struct PromotionFund { #[sqlx(try_from = "i64")] diff --git a/mobile_verifier/src/service_provider/promotions/mod.rs b/mobile_verifier/src/service_provider/promotions/mod.rs new file mode 100644 index 000000000..766b23940 --- /dev/null +++ b/mobile_verifier/src/service_provider/promotions/mod.rs @@ -0,0 +1,3 @@ +pub mod daemon; +pub mod funds; +pub mod rewards; diff --git a/mobile_verifier/src/sp_promotions/rewards.rs b/mobile_verifier/src/service_provider/promotions/rewards.rs similarity index 91% rename from mobile_verifier/src/sp_promotions/rewards.rs rename to mobile_verifier/src/service_provider/promotions/rewards.rs index 6eb1a3b60..d8d584086 100644 --- a/mobile_verifier/src/sp_promotions/rewards.rs +++ b/mobile_verifier/src/service_provider/promotions/rewards.rs @@ -7,7 +7,14 @@ use helium_crypto::PublicKeyBinary; use mobile_config::client::{carrier_service_client::CarrierServiceVerifier, ClientError}; use sqlx::{postgres::PgRow, PgPool, Postgres, Row, Transaction}; -use super::ServiceProviderId; +use crate::service_provider::ServiceProviderId; + +#[derive(Debug, Clone, PartialEq)] +pub struct PromotionRewardShares { + pub service_provider_id: ServiceProviderId, + pub rewardable_entity: Entity, + pub shares: u64, +} pub async fn save_promotion_reward( transaction: &mut Transaction<'_, Postgres>, @@ -48,11 +55,11 @@ pub async fn save_promotion_reward( Ok(()) } -pub async fn get_promotion_rewards( +pub async fn fetch_promotion_rewards( pool: &PgPool, carrier: &impl CarrierServiceVerifier, epoch: &Range>, -) -> anyhow::Result> { +) -> anyhow::Result> { let rewards = sqlx::query_as( r#" SELECT @@ -78,11 +85,11 @@ pub async fn get_promotion_rewards( .bind(epoch.end) .fetch(pool) .map_err(anyhow::Error::from) - .and_then(|x: PromotionRewardShares| async move { + .and_then(|x: DbPromotionRewardShares| async move { let service_provider_id = carrier .payer_key_to_service_provider(&x.carrier_key.to_string()) .await?; - Ok(ServiceProviderPromotionRewardShares { + Ok(PromotionRewardShares { service_provider_id: service_provider_id as ServiceProviderId, rewardable_entity: x.rewardable_entity, shares: x.shares, @@ -109,13 +116,13 @@ pub async fn clear_promotion_rewards( Ok(()) } -struct PromotionRewardShares { +struct DbPromotionRewardShares { pub carrier_key: PublicKeyBinary, pub rewardable_entity: Entity, pub shares: u64, } -impl sqlx::FromRow<'_, PgRow> for PromotionRewardShares { +impl sqlx::FromRow<'_, PgRow> for DbPromotionRewardShares { fn from_row(row: &PgRow) -> sqlx::Result { let subscriber_id: Option> = row.try_get("subscriber_id")?; let shares: i64 = row.try_get("shares")?; @@ -130,10 +137,3 @@ impl sqlx::FromRow<'_, PgRow> for PromotionRewardShares { }) } } - -#[derive(Debug)] -pub struct ServiceProviderPromotionRewardShares { - pub service_provider_id: ServiceProviderId, - pub rewardable_entity: Entity, - pub shares: u64, -} diff --git a/mobile_verifier/src/service_provider/reward.rs b/mobile_verifier/src/service_provider/reward.rs new file mode 100644 index 000000000..4ef33952d --- /dev/null +++ b/mobile_verifier/src/service_provider/reward.rs @@ -0,0 +1,422 @@ +use std::ops::Range; + +use chrono::{DateTime, Utc}; + +use file_store::traits::TimestampEncode; +use rust_decimal::{prelude::ToPrimitive, Decimal, RoundingStrategy}; +use rust_decimal_macros::dec; + +use crate::reward_shares::{dc_to_mobile_bones, DEFAULT_PREC}; + +use super::{ + dc_sessions::ServiceProviderDCSessions, + promotions::{funds::ServiceProviderFunds, rewards::PromotionRewardShares}, +}; + +mod proto { + + pub use helium_proto::services::poc_mobile::{ + mobile_reward_share::Reward, MobileRewardShare, PromotionReward, ServiceProviderReward, + }; +} + +/// Container for ['ServiceProvideRewardInfo'] +#[derive(Debug)] +pub struct RewardInfoColl { + coll: Vec, + total_sp_allocation: Decimal, + all_transfer: Decimal, + mobile_bone_price: Decimal, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct RewardInfo { + // proto::ServiceProvider enum repr + sp_id: i32, + + // How much DC in sessions + dc: Decimal, + // % allocated from DC to promo rewards (found in db from file from on chain) + allocated_promo_perc: Decimal, + + // % of total allocated rewards for all service providers + dc_perc: Decimal, + // % of total allocated rewards going towards rewards + realized_promo_perc: Decimal, + // % usable dc percentage of total rewards + realized_dc_perc: Decimal, + // % matched promotions from unallocated + matched_promo_perc: Decimal, + + // Rewards for the epoch + rewards: Vec, +} + +impl RewardInfoColl { + pub fn new( + dc_sessions: ServiceProviderDCSessions, + promo_funds: ServiceProviderFunds, + rewards: Vec, + total_sp_allocation: Decimal, + mobile_bone_price: Decimal, + ) -> Self { + let all_transfer = dc_sessions.all_transfer(); + + let mut me = Self { + coll: vec![], + all_transfer, + total_sp_allocation, + mobile_bone_price, + }; + + let used_allocation = total_sp_allocation.max(all_transfer); + for (dc_session, dc_transfer) in dc_sessions.iter() { + let promo_fund_perc = promo_funds.get_fund_percent(dc_session); + me.coll.push(RewardInfo::new( + dc_session, + dc_transfer, + promo_fund_perc, + used_allocation, + rewards + .iter() + .filter(|r| r.service_provider_id == dc_session) + .cloned() + .collect(), + )); + } + + distribute_unallocated(&mut me.coll); + + me + } + + pub fn iter_rewards( + &self, + reward_epoch: &Range>, + ) -> Vec<(u64, proto::MobileRewardShare)> { + let rewards_per_share = rewards_per_share( + self.all_transfer, + self.total_sp_allocation, + self.mobile_bone_price, + ); + let sp_rewards = self.total_sp_allocation * rewards_per_share; + + let mut rewards = vec![]; + for sp in self.coll.iter() { + rewards.extend(sp.promo_rewards(sp_rewards, reward_epoch)); + rewards.push(sp.carrier_reward(sp_rewards, reward_epoch)); + } + rewards + } +} + +impl RewardInfo { + fn new( + sp_id: i32, + dc_transfer: Decimal, + promo_fund_perc: Decimal, + used_allocation: Decimal, + rewards: Vec, + ) -> Self { + let dc_perc = dc_transfer / used_allocation; + let realized_promo_perc = if rewards.is_empty() { + dec!(0) + } else { + dc_perc * promo_fund_perc + }; + let realized_dc_perc = dc_perc - realized_promo_perc; + + Self { + sp_id, + dc: dc_transfer, + allocated_promo_perc: promo_fund_perc, + + dc_perc, + realized_promo_perc, + realized_dc_perc, + matched_promo_perc: dec!(0), + + rewards, + } + } + + pub fn carrier_reward( + &self, + total_allocation: Decimal, + reward_period: &Range>, + ) -> (u64, proto::MobileRewardShare) { + let amount = total_allocation * self.realized_dc_perc; + + ( + amount + .round_dp_with_strategy(0, RoundingStrategy::MidpointNearestEven) + .to_u64() + .unwrap_or(0), + proto::MobileRewardShare { + start_period: reward_period.start.encode_timestamp(), + end_period: reward_period.end.encode_timestamp(), + reward: Some(proto::Reward::ServiceProviderReward( + proto::ServiceProviderReward { + service_provider_id: self.sp_id, + amount: amount + .round_dp_with_strategy(0, RoundingStrategy::MidpointNearestEven) + .to_u64() + .unwrap_or(0), + }, + )), + }, + ) + } + + pub fn promo_rewards( + &self, + total_allocation: Decimal, + reward_period: &Range>, + ) -> Vec<(u64, proto::MobileRewardShare)> { + if self.rewards.is_empty() { + return vec![]; + } + + let mut rewards = vec![]; + + let sp_amount = total_allocation * self.realized_promo_perc; + let matched_amount = total_allocation * self.matched_promo_perc; + + let total_shares = self + .rewards + .iter() + .map(|r| Decimal::from(r.shares)) + .sum::(); + let sp_amount_per_share = sp_amount / total_shares; + let matched_amount_per_share = matched_amount / total_shares; + + for r in self.rewards.iter() { + let shares = Decimal::from(r.shares); + + let service_provider_amount = sp_amount_per_share * shares; + let matched_amount = matched_amount_per_share * shares; + + let total_amount = (service_provider_amount + matched_amount) + .round_dp_with_strategy(0, RoundingStrategy::MidpointNearestEven) + .to_u64() + .unwrap_or(0); + + rewards.push(( + total_amount, + proto::MobileRewardShare { + start_period: reward_period.start.encode_timestamp(), + end_period: reward_period.end.encode_timestamp(), + reward: Some(proto::Reward::PromotionReward(proto::PromotionReward { + service_provider_amount: service_provider_amount + .round_dp_with_strategy(0, RoundingStrategy::MidpointNearestEven) + .to_u64() + .unwrap_or(0), + matched_amount: matched_amount + .round_dp_with_strategy(0, RoundingStrategy::MidpointNearestEven) + .to_u64() + .unwrap_or(0), + entity: Some(r.rewardable_entity.clone().into()), + })), + }, + )) + } + + rewards + } +} + +pub fn rewards_per_share( + total_sp_dc: Decimal, + total_sp_rewards: Decimal, + mobile_bone_price: Decimal, +) -> Decimal { + let total_sp_rewards_used = dc_to_mobile_bones(total_sp_dc, mobile_bone_price); + let capped_sp_rewards_used = total_sp_rewards_used.min(total_sp_rewards); + + if capped_sp_rewards_used > Decimal::ZERO { + (capped_sp_rewards_used / total_sp_dc) + .round_dp_with_strategy(DEFAULT_PREC, RoundingStrategy::MidpointNearestEven) + } else { + Decimal::ZERO + } +} + +fn distribute_unallocated(coll: &mut [RewardInfo]) { + let allocated_perc = coll.iter().map(|x| x.dc_perc).sum::(); + let unallocated_perc = dec!(1) - allocated_perc; + + let maybe_matching_perc = coll + .iter() + .filter(|x| !x.rewards.is_empty()) + .map(|x| x.realized_promo_perc) + .sum::(); + + if maybe_matching_perc > unallocated_perc { + distribute_unalloc_over_limit(coll, unallocated_perc); + } else { + distribute_unalloc_under_limit(coll); + } +} + +fn distribute_unalloc_over_limit(coll: &mut [RewardInfo], unallocated_perc: Decimal) { + // NOTE: This can also allocate based off the dc_perc of each carrier. + let total = coll.iter().map(|x| x.realized_promo_perc).sum::() * dec!(100); + + for sp in coll.iter_mut() { + if sp.rewards.is_empty() { + continue; + } + let shares = sp.realized_promo_perc * dec!(100); + sp.matched_promo_perc = ((shares / total) * unallocated_perc).round_dp(5); + } +} + +fn distribute_unalloc_under_limit(coll: &mut [RewardInfo]) { + for sp in coll.iter_mut() { + if sp.rewards.is_empty() { + continue; + } + sp.matched_promo_perc = sp.realized_promo_perc + } +} + +#[cfg(test)] +mod tests { + use super::*; + use file_store::promotion_reward::Entity; + use std::collections::HashMap; + use uuid::Uuid; + + fn get_unallocated_percent(sps: &RewardInfoColl) -> Decimal { + dec!(1.0) - sps.coll.iter().map(|x| x.dc_perc).sum::() + } + + #[test] + fn test_multiple_with_reward() { + let dc_sessions = HashMap::from_iter([(0, dec!(600)), (1, dec!(600))]); + let promo_funds = HashMap::from_iter([(0, 2000), (1, 4000)]); // bps values + let total_allocation = dec!(1000); + + let sps = RewardInfoColl::new( + ServiceProviderDCSessions(dc_sessions), + ServiceProviderFunds(promo_funds), + vec![ + PromotionRewardShares { + service_provider_id: 0, + rewardable_entity: Entity::SubscriberId(Uuid::new_v4().into()), + shares: 1, + }, + PromotionRewardShares { + service_provider_id: 0, + rewardable_entity: Entity::SubscriberId(Uuid::new_v4().into()), + shares: 2, + }, + PromotionRewardShares { + service_provider_id: 0, + rewardable_entity: Entity::SubscriberId(Uuid::new_v4().into()), + shares: 3, + }, + ], + total_allocation, + dec!(0.0001), + ); + let x = get_unallocated_percent(&sps); + + println!("leftover: {x:?}"); + + let reward_period = DateTime::::MIN_UTC..DateTime::::MAX_UTC; + for sp in sps.coll { + println!("{sp:?}"); + for reward in sp.promo_rewards(total_allocation, &reward_period) { + println!("--> {reward:?}"); + } + } + } + + #[test] + fn test_multiple_no_reward() { + let dc_sessions = HashMap::from_iter([(0, dec!(600)), (1, dec!(600))]); + let promo_funds = HashMap::from_iter([(0, 2000), (1, 4000)]); // bps values + let total_allocation = dec!(1200); + + let mut sps = RewardInfoColl::new( + ServiceProviderDCSessions(dc_sessions), + ServiceProviderFunds(promo_funds), + vec![], + total_allocation, + dec!(0.0001), + ); + + distribute_unallocated(&mut sps.coll); + + println!("{sps:#?}"); + } + + #[test] + fn over_limit_with_rewards() { + let promo_0 = PromotionRewardShares { + service_provider_id: 0, + rewardable_entity: file_store::promotion_reward::Entity::SubscriberId( + uuid::Uuid::new_v4().into(), + ), + shares: 1, + }; + let promo_1 = PromotionRewardShares { + service_provider_id: 0, + rewardable_entity: file_store::promotion_reward::Entity::SubscriberId( + uuid::Uuid::new_v4().into(), + ), + shares: 1, + }; + let expected_one = RewardInfo { + sp_id: 0, + dc: dec!(200), + allocated_promo_perc: dec!(0.2), + dc_perc: dec!(0.2), + realized_promo_perc: dec!(0.04), + realized_dc_perc: dec!(0.16), + matched_promo_perc: dec!(0.02667), + + rewards: vec![promo_0.clone()], + }; + + let one = RewardInfo::new(0, dec!(200), dec!(0.2), dec!(1000), vec![promo_0]); + let two = RewardInfo::new(1, dec!(200), dec!(0.4), dec!(1000), vec![promo_1]); + + let unallocated_perc = dec!(0.08); + let mut x = vec![one, two]; + distribute_unalloc_over_limit(&mut x, unallocated_perc); + + println!("{:#?}", x[0]); + println!("{:#?}", x[1]); + + assert_eq!(expected_one, x[0]); + } + + #[test] + fn over_limit_no_rewards() { + let expected_one = RewardInfo { + sp_id: 0, + dc: dec!(200), + allocated_promo_perc: dec!(0.20), + dc_perc: dec!(0.20), + realized_promo_perc: dec!(0.00), + realized_dc_perc: dec!(0.20), + matched_promo_perc: dec!(0.0), + + rewards: vec![], + }; + + let one = RewardInfo::new(0, dec!(200), dec!(0.2), dec!(1000), vec![]); + let two = RewardInfo::new(1, dec!(200), dec!(0.4), dec!(1000), vec![]); + + let unallocated_perc = dec!(0.08); + let mut x = vec![one, two]; + distribute_unalloc_over_limit(&mut x, unallocated_perc); + + println!("{:#?}", x[0]); + println!("{:#?}", x[1]); + + assert_eq!(expected_one, x[0]); + } +} diff --git a/mobile_verifier/src/sp_promotions/mod.rs b/mobile_verifier/src/sp_promotions/mod.rs deleted file mode 100644 index e98464a09..000000000 --- a/mobile_verifier/src/sp_promotions/mod.rs +++ /dev/null @@ -1,11 +0,0 @@ -pub use daemon::SpPromotionDaemon; -pub use rewards::clear_promotion_rewards; - -pub mod daemon; -pub mod funds; -pub mod rewards; - -// This type is used in lieu of the helium_proto::ServiceProvider enum so we can -// handle more than a single value without adding a hard deploy dependency to -// mobile-verifier when a new carrier is added.. -pub type ServiceProviderId = i32; From d3a424112e6d62a89d8f67a5725d62ea9bfd2b56 Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Fri, 20 Sep 2024 08:56:48 -0700 Subject: [PATCH 08/31] Wrap everything in a service provider type This makes invocations of rewarding look very consistent --- .../src/service_provider/dc_sessions.rs | 10 ++++ .../src/service_provider/promotions/funds.rs | 10 ++++ .../service_provider/promotions/rewards.rs | 48 +++++++++++++++++-- 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/mobile_verifier/src/service_provider/dc_sessions.rs b/mobile_verifier/src/service_provider/dc_sessions.rs index 37dd6e45d..296fabef6 100644 --- a/mobile_verifier/src/service_provider/dc_sessions.rs +++ b/mobile_verifier/src/service_provider/dc_sessions.rs @@ -87,3 +87,13 @@ impl ServiceProviderDCSessions { } } } + +impl From for ServiceProviderDCSessions +where + F: IntoIterator, + I: Into, +{ + fn from(iter: F) -> Self { + Self(iter.into_iter().map(|(k, v)| (k.into(), v)).collect()) + } +} diff --git a/mobile_verifier/src/service_provider/promotions/funds.rs b/mobile_verifier/src/service_provider/promotions/funds.rs index ab7a49671..70a0aa28e 100644 --- a/mobile_verifier/src/service_provider/promotions/funds.rs +++ b/mobile_verifier/src/service_provider/promotions/funds.rs @@ -21,6 +21,16 @@ impl ServiceProviderFunds { } } +impl From for ServiceProviderFunds +where + F: IntoIterator, + I: Into, +{ + fn from(funds: F) -> Self { + Self(funds.into_iter().map(|(k, v)| (k.into(), v)).collect()) + } +} + pub async fn fetch_promotion_funds(pool: &PgPool) -> anyhow::Result { #[derive(Debug, sqlx::FromRow)] struct PromotionFund { diff --git a/mobile_verifier/src/service_provider/promotions/rewards.rs b/mobile_verifier/src/service_provider/promotions/rewards.rs index d8d584086..70c91ad2a 100644 --- a/mobile_verifier/src/service_provider/promotions/rewards.rs +++ b/mobile_verifier/src/service_provider/promotions/rewards.rs @@ -5,12 +5,52 @@ use file_store::promotion_reward::{Entity, PromotionReward}; use futures::TryStreamExt; use helium_crypto::PublicKeyBinary; use mobile_config::client::{carrier_service_client::CarrierServiceVerifier, ClientError}; +use rust_decimal::Decimal; use sqlx::{postgres::PgRow, PgPool, Postgres, Row, Transaction}; use crate::service_provider::ServiceProviderId; +#[derive(Debug, Default, Clone, PartialEq)] +pub struct ServiceProviderPromotions(Vec); + +impl ServiceProviderPromotions { + pub fn for_service_provider( + &self, + service_provider_id: ServiceProviderId, + ) -> ServiceProviderPromotions { + let promotions = self + .0 + .iter() + .filter(|x| x.service_provider_id == service_provider_id) + .cloned() + .collect(); + Self(promotions) + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn total_shares(&self) -> Decimal { + self.0.iter().map(|x| Decimal::from(x.shares)).sum() + } + + pub fn iter(&self) -> impl Iterator { + self.0.iter() + } +} + +impl From for ServiceProviderPromotions +where + F: IntoIterator, +{ + fn from(promotions: F) -> Self { + Self(promotions.into_iter().collect()) + } +} + #[derive(Debug, Clone, PartialEq)] -pub struct PromotionRewardShares { +pub struct PromotionRewardShare { pub service_provider_id: ServiceProviderId, pub rewardable_entity: Entity, pub shares: u64, @@ -59,7 +99,7 @@ pub async fn fetch_promotion_rewards( pool: &PgPool, carrier: &impl CarrierServiceVerifier, epoch: &Range>, -) -> anyhow::Result> { +) -> anyhow::Result { let rewards = sqlx::query_as( r#" SELECT @@ -89,7 +129,7 @@ pub async fn fetch_promotion_rewards( let service_provider_id = carrier .payer_key_to_service_provider(&x.carrier_key.to_string()) .await?; - Ok(PromotionRewardShares { + Ok(PromotionRewardShare { service_provider_id: service_provider_id as ServiceProviderId, rewardable_entity: x.rewardable_entity, shares: x.shares, @@ -98,7 +138,7 @@ pub async fn fetch_promotion_rewards( .try_collect() .await?; - Ok(rewards) + Ok(ServiceProviderPromotions(rewards)) } pub async fn clear_promotion_rewards( From 1dfe1da2e68cdb1979cf6227de0f451022c50ef1 Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Fri, 20 Sep 2024 09:15:30 -0700 Subject: [PATCH 09/31] Update tests to use new service_provider code --- mobile_verifier/src/reward_shares.rs | 115 +++++++++++++-------------- 1 file changed, 55 insertions(+), 60 deletions(-) diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index ee2cd2e98..ca426fa0e 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -901,6 +901,11 @@ mod test { data_session::{self, HotspotDataSession, HotspotReward}, heartbeats::{HeartbeatReward, KeyType, OwnedKeyType}, reward_shares, + service_provider::{ + self, + dc_sessions::ServiceProviderDCSessions, + promotions::{funds::ServiceProviderFunds, rewards::ServiceProviderPromotions}, + }, speedtests::Speedtest, speedtests_average::SpeedtestAverage, subscriber_location::SubscriberValidatedLocations, @@ -2344,8 +2349,8 @@ mod test { .is_none()); } - #[tokio::test] - async fn service_provider_reward_amounts() { + #[test] + fn service_provider_reward_amounts() { let mobile_bone_price = dec!(0.00001); let sp1 = ServiceProvider::HeliumMobile; @@ -2353,22 +2358,20 @@ mod test { let now = Utc::now(); let epoch = (now - Duration::hours(1))..now; - let service_provider_sessions = vec![ServiceProviderDataSession { - service_provider: sp1, - total_dcs: dec!(1000), - }]; - let sp_shares = ServiceProviderShares::new(service_provider_sessions); - let total_sp_rewards = get_scheduled_tokens_for_service_providers(epoch.end - epoch.start); - let rewards_per_share = sp_shares - .rewards_per_share(total_sp_rewards, mobile_bone_price) - .unwrap(); + let total_sp_rewards = service_provider::get_scheduled_tokens(&epoch); + let sp_reward_infos = service_provider::RewardInfoColl::new( + ServiceProviderDCSessions::from([(sp1, dec!(1000))]), + ServiceProviderFunds::default(), + ServiceProviderPromotions::default(), + total_sp_rewards, + mobile_bone_price, + ); let mut sp_rewards = HashMap::::new(); let mut allocated_sp_rewards = 0_u64; - for (reward_amount, sp_reward) in - sp_shares.into_service_provider_rewards(&epoch, rewards_per_share) - { - if let Some(MobileReward::ServiceProviderReward(r)) = sp_reward.reward { + + for (reward_amount, reward) in sp_reward_infos.iter_rewards(&epoch) { + if let Some(MobileReward::ServiceProviderReward(r)) = reward.reward { sp_rewards.insert(r.service_provider_id, r.amount); assert_eq!(reward_amount, r.amount); allocated_sp_rewards += reward_amount; @@ -2388,8 +2391,8 @@ mod test { assert_eq!(unallocated_sp_reward_amount, 342_465_752_424); } - #[tokio::test] - async fn service_provider_reward_amounts_capped() { + #[test] + fn service_provider_reward_amounts_capped() { let mobile_bone_price = dec!(1.0); let sp1 = ServiceProvider::HeliumMobile; @@ -2400,28 +2403,26 @@ mod test { let total_rewards_value_in_dc = mobile_bones_to_dc(total_sp_rewards_in_bones, mobile_bone_price); - let service_provider_sessions = vec![ServiceProviderDataSession { - service_provider: ServiceProvider::HeliumMobile, + let sp_reward_infos = service_provider::RewardInfoColl::new( // force the service provider to have spend more DC than total rewardable - total_dcs: total_rewards_value_in_dc * dec!(2.0), - }]; - - let sp_shares = ServiceProviderShares::new(service_provider_sessions); - let rewards_per_share = sp_shares - .rewards_per_share(total_sp_rewards_in_bones, mobile_bone_price) - .unwrap(); + ServiceProviderDCSessions::from([(sp1, total_rewards_value_in_dc * dec!(2.0))]), + ServiceProviderFunds::default(), + ServiceProviderPromotions::default(), + total_rewards_value_in_dc, + mobile_bone_price, + ); let mut sp_rewards = HashMap::new(); let mut allocated_sp_rewards = 0_u64; - for (reward_amount, sp_reward) in - sp_shares.into_service_provider_rewards(&epoch, rewards_per_share) - { - if let Some(MobileReward::ServiceProviderReward(r)) = sp_reward.reward { + + for (reward_amount, reward) in sp_reward_infos.iter_rewards(&epoch) { + if let Some(MobileReward::ServiceProviderReward(r)) = reward.reward { sp_rewards.insert(r.service_provider_id, r.amount); assert_eq!(reward_amount, r.amount); allocated_sp_rewards += reward_amount; } } + let sp1_reward_amount = *sp_rewards .get(&(sp1 as i32)) .expect("Could not fetch sp1 shares"); @@ -2438,8 +2439,8 @@ mod test { assert_eq!(unallocated_sp_reward_amount, 0); } - #[tokio::test] - async fn service_provider_reward_hip87_ex1() { + #[test] + fn service_provider_reward_hip87_ex1() { // mobile price from hip example and converted to bones let mobile_bone_price = dec!(0.0001) / dec!(1_000_000); let sp1 = ServiceProvider::HeliumMobile; @@ -2448,22 +2449,19 @@ mod test { let epoch = (now - Duration::hours(1))..now; let total_sp_rewards_in_bones = dec!(500_000_000) * dec!(1_000_000); - let service_provider_sessions = vec![ServiceProviderDataSession { - service_provider: sp1, - total_dcs: dec!(100_000_000), - }]; - - let sp_shares = ServiceProviderShares::new(service_provider_sessions); - let rewards_per_share = sp_shares - .rewards_per_share(total_sp_rewards_in_bones, mobile_bone_price) - .unwrap(); + let sp_reward_infos = service_provider::RewardInfoColl::new( + ServiceProviderDCSessions::from([(sp1, dec!(100_000_000))]), + ServiceProviderFunds::default(), + ServiceProviderPromotions::default(), + total_sp_rewards_in_bones, + mobile_bone_price, + ); let mut sp_rewards = HashMap::new(); let mut allocated_sp_rewards = 0_u64; - for (reward_amount, sp_reward) in - sp_shares.into_service_provider_rewards(&epoch, rewards_per_share) - { - if let Some(MobileReward::ServiceProviderReward(r)) = sp_reward.reward { + + for (reward_amount, reward) in sp_reward_infos.iter_rewards(&epoch) { + if let Some(MobileReward::ServiceProviderReward(r)) = reward.reward { sp_rewards.insert(r.service_provider_id, r.amount); assert_eq!(reward_amount, r.amount); allocated_sp_rewards += reward_amount; @@ -2486,8 +2484,8 @@ mod test { assert_eq!(unallocated_sp_reward_amount, 490_000_000_000_000); } - #[tokio::test] - async fn service_provider_reward_hip87_ex2() { + #[test] + fn service_provider_reward_hip87_ex2() { // mobile price from hip example and converted to bones let mobile_bone_price = dec!(0.0001) / dec!(1_000_000); let sp1 = ServiceProvider::HeliumMobile; @@ -2496,22 +2494,19 @@ mod test { let epoch = (now - Duration::hours(24))..now; let total_sp_rewards_in_bones = dec!(500_000_000) * dec!(1_000_000); - let service_provider_sessions = vec![ServiceProviderDataSession { - service_provider: sp1, - total_dcs: dec!(100_000_000_000), - }]; - - let sp_shares = ServiceProviderShares::new(service_provider_sessions); - let rewards_per_share = sp_shares - .rewards_per_share(total_sp_rewards_in_bones, mobile_bone_price) - .unwrap(); + let sp_reward_infos = service_provider::RewardInfoColl::new( + ServiceProviderDCSessions::from([(sp1, dec!(100_000_000_000))]), + ServiceProviderFunds::default(), + ServiceProviderPromotions::default(), + total_sp_rewards_in_bones, + mobile_bone_price, + ); let mut sp_rewards = HashMap::new(); let mut allocated_sp_rewards = 0_u64; - for (reward_amount, sp_reward) in - sp_shares.into_service_provider_rewards(&epoch, rewards_per_share) - { - if let Some(MobileReward::ServiceProviderReward(r)) = sp_reward.reward { + + for (reward_amount, reward) in sp_reward_infos.iter_rewards(&epoch) { + if let Some(MobileReward::ServiceProviderReward(r)) = reward.reward { sp_rewards.insert(r.service_provider_id, r.amount); assert_eq!(reward_amount, r.amount); allocated_sp_rewards += reward_amount; From be4e610b457cc68756600cc11b9f174f7f6d9e90 Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Fri, 20 Sep 2024 09:23:02 -0700 Subject: [PATCH 10/31] remove unused code --- mobile_verifier/src/data_session.rs | 8 -- mobile_verifier/src/reward_shares.rs | 136 +-------------------------- 2 files changed, 3 insertions(+), 141 deletions(-) diff --git a/mobile_verifier/src/data_session.rs b/mobile_verifier/src/data_session.rs index b519fed64..7f1940777 100644 --- a/mobile_verifier/src/data_session.rs +++ b/mobile_verifier/src/data_session.rs @@ -10,8 +10,6 @@ use futures::{ TryFutureExt, }; use helium_crypto::PublicKeyBinary; -use helium_proto::ServiceProvider; -use rust_decimal::Decimal; use sqlx::{PgPool, Pool, Postgres, Row, Transaction}; use std::{collections::HashMap, ops::Range, time::Instant}; use task_manager::{ManagedTask, TaskManager}; @@ -30,12 +28,6 @@ pub struct HotspotReward { pub rewardable_dc: u64, } -#[derive(Clone, Debug)] -pub struct ServiceProviderDataSession { - pub service_provider: ServiceProvider, - pub total_dcs: Decimal, -} - pub type HotspotMap = HashMap; impl DataSessionIngestor { diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index ca426fa0e..8ae4fdc85 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -10,19 +10,10 @@ use coverage_point_calculator::{OracleBoostingStatus, SPBoostedRewardEligibility use file_store::traits::TimestampEncode; use futures::{Stream, StreamExt}; use helium_crypto::PublicKeyBinary; -use helium_proto::{ - services::{ - poc_mobile as proto, - poc_mobile::{ - mobile_reward_share::Reward as ProtoReward, UnallocatedReward, UnallocatedRewardType, - }, - }, - ServiceProvider, -}; -use mobile_config::{ - boosted_hex_info::BoostedHexes, - client::{carrier_service_client::CarrierServiceVerifier, ClientError}, +use helium_proto::services::{ + poc_mobile as proto, poc_mobile::mobile_reward_share::Reward as ProtoReward, }; +use mobile_config::boosted_hex_info::BoostedHexes; use radio_reward_v2::{RadioRewardV2Ext, ToProtoDecimal}; use rust_decimal::prelude::*; use rust_decimal_macros::dec; @@ -295,127 +286,6 @@ impl MapperShares { } } -#[derive(Default)] -pub struct ServiceProviderShares { - pub shares: Vec, -} - -impl ServiceProviderShares { - pub fn new(shares: Vec) -> Self { - Self { shares } - } - - pub async fn from_payers_dc( - payer_shares: HashMap, - client: &impl CarrierServiceVerifier, - ) -> anyhow::Result { - let mut sp_shares = ServiceProviderShares::default(); - for (payer, total_dcs) in payer_shares { - let service_provider = Self::payer_key_to_service_provider(&payer, client).await?; - sp_shares.shares.push(ServiceProviderDataSession { - service_provider, - total_dcs: Decimal::from(total_dcs), - }) - } - Ok(sp_shares) - } - - fn total_dc(&self) -> Decimal { - self.shares.iter().map(|v| v.total_dcs).sum() - } - - pub fn rewards_per_share( - &self, - total_sp_rewards: Decimal, - mobile_bone_price: Decimal, - ) -> anyhow::Result { - // the total amount of DC spent across all service providers - let total_sp_dc = self.total_dc(); - // the total amount of service provider rewards in bones based on the spent DC - let total_sp_rewards_used = dc_to_mobile_bones(total_sp_dc, mobile_bone_price); - // cap the service provider rewards if used > pool total - let capped_sp_rewards_used = - Self::maybe_cap_service_provider_rewards(total_sp_rewards_used, total_sp_rewards); - Ok(Self::calc_rewards_per_share( - capped_sp_rewards_used, - total_sp_dc, - )) - } - - pub fn into_service_provider_rewards( - self, - reward_period: &'_ Range>, - reward_per_share: Decimal, - ) -> impl Iterator + '_ { - self.shares - .into_iter() - .map(move |share| proto::ServiceProviderReward { - service_provider_id: share.service_provider as i32, - amount: (share.total_dcs * reward_per_share) - .round_dp_with_strategy(0, RoundingStrategy::ToZero) - .to_u64() - .unwrap_or(0), - }) - .filter(|service_provider_reward| service_provider_reward.amount > 0) - .map(|service_provider_reward| { - ( - service_provider_reward.amount, - proto::MobileRewardShare { - start_period: reward_period.start.encode_timestamp(), - end_period: reward_period.end.encode_timestamp(), - reward: Some(ProtoReward::ServiceProviderReward(service_provider_reward)), - }, - ) - }) - } - - pub fn into_unallocated_reward( - unallocated_amount: Decimal, - reward_period: &'_ Range>, - ) -> anyhow::Result { - let reward = UnallocatedReward { - reward_type: UnallocatedRewardType::ServiceProvider as i32, - amount: unallocated_amount - .round_dp_with_strategy(0, RoundingStrategy::ToZero) - .to_u64() - .unwrap_or(0), - }; - Ok(proto::MobileRewardShare { - start_period: reward_period.start.encode_timestamp(), - end_period: reward_period.end.encode_timestamp(), - reward: Some(ProtoReward::UnallocatedReward(reward)), - }) - } - - fn maybe_cap_service_provider_rewards( - total_sp_rewards_used: Decimal, - total_sp_rewards: Decimal, - ) -> Decimal { - match total_sp_rewards_used <= total_sp_rewards { - true => total_sp_rewards_used, - false => total_sp_rewards, - } - } - - fn calc_rewards_per_share(total_rewards: Decimal, total_shares: Decimal) -> Decimal { - if total_shares > Decimal::ZERO { - (total_rewards / total_shares) - .round_dp_with_strategy(DEFAULT_PREC, RoundingStrategy::MidpointNearestEven) - } else { - Decimal::ZERO - } - } - - async fn payer_key_to_service_provider( - payer: &str, - client: &impl CarrierServiceVerifier, - ) -> anyhow::Result { - tracing::info!(payer, "getting service provider for payer"); - let sp = client.payer_key_to_service_provider(payer).await?; - Ok(sp) - } -} - /// Returns the equivalent amount of Mobile bones for a specified amount of Data Credits pub fn dc_to_mobile_bones(dc_amount: Decimal, mobile_bone_price: Decimal) -> Decimal { let dc_in_usd = dc_amount * DC_USD_PRICE; From 1f16e7affd89702e9b8d984843af5a5505ed0ab3 Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Fri, 20 Sep 2024 09:23:18 -0700 Subject: [PATCH 11/31] use wrapped types for promotions --- .../src/service_provider/reward.rs | 125 ++++++++++-------- 1 file changed, 71 insertions(+), 54 deletions(-) diff --git a/mobile_verifier/src/service_provider/reward.rs b/mobile_verifier/src/service_provider/reward.rs index 4ef33952d..bc943841f 100644 --- a/mobile_verifier/src/service_provider/reward.rs +++ b/mobile_verifier/src/service_provider/reward.rs @@ -10,16 +10,31 @@ use crate::reward_shares::{dc_to_mobile_bones, DEFAULT_PREC}; use super::{ dc_sessions::ServiceProviderDCSessions, - promotions::{funds::ServiceProviderFunds, rewards::PromotionRewardShares}, + promotions::{funds::ServiceProviderFunds, rewards::ServiceProviderPromotions}, }; mod proto { - pub use helium_proto::services::poc_mobile::{ mobile_reward_share::Reward, MobileRewardShare, PromotionReward, ServiceProviderReward, }; } +pub fn rewards_per_share( + total_sp_dc: Decimal, + total_sp_rewards: Decimal, + mobile_bone_price: Decimal, +) -> Decimal { + let total_sp_rewards_used = dc_to_mobile_bones(total_sp_dc, mobile_bone_price); + let capped_sp_rewards_used = total_sp_rewards_used.min(total_sp_rewards); + + if capped_sp_rewards_used > Decimal::ZERO { + (capped_sp_rewards_used / total_sp_dc) + .round_dp_with_strategy(DEFAULT_PREC, RoundingStrategy::MidpointNearestEven) + } else { + Decimal::ZERO + } +} + /// Container for ['ServiceProvideRewardInfo'] #[derive(Debug)] pub struct RewardInfoColl { @@ -49,14 +64,14 @@ pub struct RewardInfo { matched_promo_perc: Decimal, // Rewards for the epoch - rewards: Vec, + promotion_rewards: ServiceProviderPromotions, } impl RewardInfoColl { pub fn new( dc_sessions: ServiceProviderDCSessions, promo_funds: ServiceProviderFunds, - rewards: Vec, + rewards: ServiceProviderPromotions, total_sp_allocation: Decimal, mobile_bone_price: Decimal, ) -> Self { @@ -77,11 +92,7 @@ impl RewardInfoColl { dc_transfer, promo_fund_perc, used_allocation, - rewards - .iter() - .filter(|r| r.service_provider_id == dc_session) - .cloned() - .collect(), + rewards.for_service_provider(dc_session), )); } @@ -111,14 +122,14 @@ impl RewardInfoColl { } impl RewardInfo { - fn new( + pub fn new( sp_id: i32, dc_transfer: Decimal, promo_fund_perc: Decimal, - used_allocation: Decimal, - rewards: Vec, + total_sp_allocation: Decimal, + rewards: ServiceProviderPromotions, ) -> Self { - let dc_perc = dc_transfer / used_allocation; + let dc_perc = dc_transfer / total_sp_allocation; let realized_promo_perc = if rewards.is_empty() { dec!(0) } else { @@ -136,7 +147,7 @@ impl RewardInfo { realized_dc_perc, matched_promo_perc: dec!(0), - rewards, + promotion_rewards: rewards, } } @@ -173,7 +184,7 @@ impl RewardInfo { total_allocation: Decimal, reward_period: &Range>, ) -> Vec<(u64, proto::MobileRewardShare)> { - if self.rewards.is_empty() { + if self.promotion_rewards.is_empty() { return vec![]; } @@ -182,15 +193,11 @@ impl RewardInfo { let sp_amount = total_allocation * self.realized_promo_perc; let matched_amount = total_allocation * self.matched_promo_perc; - let total_shares = self - .rewards - .iter() - .map(|r| Decimal::from(r.shares)) - .sum::(); + let total_shares = self.promotion_rewards.total_shares(); let sp_amount_per_share = sp_amount / total_shares; let matched_amount_per_share = matched_amount / total_shares; - for r in self.rewards.iter() { + for r in self.promotion_rewards.iter() { let shares = Decimal::from(r.shares); let service_provider_amount = sp_amount_per_share * shares; @@ -225,29 +232,13 @@ impl RewardInfo { } } -pub fn rewards_per_share( - total_sp_dc: Decimal, - total_sp_rewards: Decimal, - mobile_bone_price: Decimal, -) -> Decimal { - let total_sp_rewards_used = dc_to_mobile_bones(total_sp_dc, mobile_bone_price); - let capped_sp_rewards_used = total_sp_rewards_used.min(total_sp_rewards); - - if capped_sp_rewards_used > Decimal::ZERO { - (capped_sp_rewards_used / total_sp_dc) - .round_dp_with_strategy(DEFAULT_PREC, RoundingStrategy::MidpointNearestEven) - } else { - Decimal::ZERO - } -} - fn distribute_unallocated(coll: &mut [RewardInfo]) { let allocated_perc = coll.iter().map(|x| x.dc_perc).sum::(); let unallocated_perc = dec!(1) - allocated_perc; let maybe_matching_perc = coll .iter() - .filter(|x| !x.rewards.is_empty()) + .filter(|x| !x.promotion_rewards.is_empty()) .map(|x| x.realized_promo_perc) .sum::(); @@ -263,7 +254,7 @@ fn distribute_unalloc_over_limit(coll: &mut [RewardInfo], unallocated_perc: Deci let total = coll.iter().map(|x| x.realized_promo_perc).sum::() * dec!(100); for sp in coll.iter_mut() { - if sp.rewards.is_empty() { + if sp.promotion_rewards.is_empty() { continue; } let shares = sp.realized_promo_perc * dec!(100); @@ -273,7 +264,7 @@ fn distribute_unalloc_over_limit(coll: &mut [RewardInfo], unallocated_perc: Deci fn distribute_unalloc_under_limit(coll: &mut [RewardInfo]) { for sp in coll.iter_mut() { - if sp.rewards.is_empty() { + if sp.promotion_rewards.is_empty() { continue; } sp.matched_promo_perc = sp.realized_promo_perc @@ -282,6 +273,8 @@ fn distribute_unalloc_under_limit(coll: &mut [RewardInfo]) { #[cfg(test)] mod tests { + use crate::service_provider::promotions::rewards::PromotionRewardShare; + use super::*; use file_store::promotion_reward::Entity; use std::collections::HashMap; @@ -300,23 +293,23 @@ mod tests { let sps = RewardInfoColl::new( ServiceProviderDCSessions(dc_sessions), ServiceProviderFunds(promo_funds), - vec![ - PromotionRewardShares { + ServiceProviderPromotions::from(vec![ + PromotionRewardShare { service_provider_id: 0, rewardable_entity: Entity::SubscriberId(Uuid::new_v4().into()), shares: 1, }, - PromotionRewardShares { + PromotionRewardShare { service_provider_id: 0, rewardable_entity: Entity::SubscriberId(Uuid::new_v4().into()), shares: 2, }, - PromotionRewardShares { + PromotionRewardShare { service_provider_id: 0, rewardable_entity: Entity::SubscriberId(Uuid::new_v4().into()), shares: 3, }, - ], + ]), total_allocation, dec!(0.0001), ); @@ -342,7 +335,7 @@ mod tests { let mut sps = RewardInfoColl::new( ServiceProviderDCSessions(dc_sessions), ServiceProviderFunds(promo_funds), - vec![], + ServiceProviderPromotions::default(), total_allocation, dec!(0.0001), ); @@ -354,14 +347,14 @@ mod tests { #[test] fn over_limit_with_rewards() { - let promo_0 = PromotionRewardShares { + let promo_0 = PromotionRewardShare { service_provider_id: 0, rewardable_entity: file_store::promotion_reward::Entity::SubscriberId( uuid::Uuid::new_v4().into(), ), shares: 1, }; - let promo_1 = PromotionRewardShares { + let promo_1 = PromotionRewardShare { service_provider_id: 0, rewardable_entity: file_store::promotion_reward::Entity::SubscriberId( uuid::Uuid::new_v4().into(), @@ -377,11 +370,23 @@ mod tests { realized_dc_perc: dec!(0.16), matched_promo_perc: dec!(0.02667), - rewards: vec![promo_0.clone()], + promotion_rewards: ServiceProviderPromotions::from(vec![promo_0.clone()]), }; - let one = RewardInfo::new(0, dec!(200), dec!(0.2), dec!(1000), vec![promo_0]); - let two = RewardInfo::new(1, dec!(200), dec!(0.4), dec!(1000), vec![promo_1]); + let one = RewardInfo::new( + 0, + dec!(200), + dec!(0.2), + dec!(1000), + ServiceProviderPromotions::from(vec![promo_0]), + ); + let two = RewardInfo::new( + 1, + dec!(200), + dec!(0.4), + dec!(1000), + ServiceProviderPromotions::from(vec![promo_1]), + ); let unallocated_perc = dec!(0.08); let mut x = vec![one, two]; @@ -404,11 +409,23 @@ mod tests { realized_dc_perc: dec!(0.20), matched_promo_perc: dec!(0.0), - rewards: vec![], + promotion_rewards: ServiceProviderPromotions::default(), }; - let one = RewardInfo::new(0, dec!(200), dec!(0.2), dec!(1000), vec![]); - let two = RewardInfo::new(1, dec!(200), dec!(0.4), dec!(1000), vec![]); + let one = RewardInfo::new( + 0, + dec!(200), + dec!(0.2), + dec!(1000), + ServiceProviderPromotions::default(), + ); + let two = RewardInfo::new( + 1, + dec!(200), + dec!(0.4), + dec!(1000), + ServiceProviderPromotions::default(), + ); let unallocated_perc = dec!(0.08); let mut x = vec![one, two]; From 7d44da207438f5cd80652944ff217a301b3e0b6a Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Fri, 20 Sep 2024 09:45:06 -0700 Subject: [PATCH 12/31] add back promotion_fund cli --- mobile_verifier/src/cli/mod.rs | 1 + mobile_verifier/src/cli/promotion_funds.rs | 59 ++++++++++++++++++++++ mobile_verifier/src/main.rs | 4 +- 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 mobile_verifier/src/cli/promotion_funds.rs diff --git a/mobile_verifier/src/cli/mod.rs b/mobile_verifier/src/cli/mod.rs index 611db001b..36cc1a1df 100644 --- a/mobile_verifier/src/cli/mod.rs +++ b/mobile_verifier/src/cli/mod.rs @@ -1,3 +1,4 @@ +pub mod promotion_funds; pub mod reward_from_db; pub mod server; pub mod verify_disktree; diff --git a/mobile_verifier/src/cli/promotion_funds.rs b/mobile_verifier/src/cli/promotion_funds.rs new file mode 100644 index 000000000..0801a9074 --- /dev/null +++ b/mobile_verifier/src/cli/promotion_funds.rs @@ -0,0 +1,59 @@ +use crate::{ + service_provider::promotions::funds::{ + delete_promotion_fund, fetch_promotion_funds, save_promotion_fund, + }, + Settings, +}; + +#[derive(Debug, clap::Args)] +pub struct Cmd { + #[clap(subcommand)] + sub_command: SubCommand, +} + +#[derive(Debug, clap::Subcommand)] +enum SubCommand { + /// Print Service Provider promotions in mobile-verifier db + List, + /// Set Service Provider promotion in mobile-verifier db + Set { + service_provider_id: i32, + basis_points: u16, + }, + /// Remove Service Provider promotion allocation from mobile-verifier db + Unset { service_provider_id: i32 }, +} + +impl Cmd { + pub async fn run(&self, settings: &Settings) -> anyhow::Result<()> { + let pool = settings.database.connect(env!("CARGO_PKG_NAME")).await?; + + match self.sub_command { + SubCommand::List => { + let funds = fetch_promotion_funds(&pool).await?; + println!("{funds:?}"); + } + SubCommand::Set { + service_provider_id, + basis_points, + } => { + let mut txn = pool.begin().await?; + save_promotion_fund(&mut txn, service_provider_id, basis_points).await?; + txn.commit().await?; + + let funds = fetch_promotion_funds(&pool).await?; + println!("{funds:?}"); + } + SubCommand::Unset { + service_provider_id, + } => { + delete_promotion_fund(&pool, service_provider_id).await?; + + let funds = fetch_promotion_funds(&pool).await?; + println!("{funds:?}"); + } + } + + Ok(()) + } +} diff --git a/mobile_verifier/src/main.rs b/mobile_verifier/src/main.rs index 3b3c991b7..43d6a3ba2 100644 --- a/mobile_verifier/src/main.rs +++ b/mobile_verifier/src/main.rs @@ -1,7 +1,7 @@ use anyhow::Result; use clap::Parser; use mobile_verifier::{ - cli::{reward_from_db, server, verify_disktree}, + cli::{promotion_funds, reward_from_db, server, verify_disktree}, Settings, }; use std::path; @@ -37,6 +37,7 @@ pub enum Cmd { /// Go through every cell and ensure it's value can be turned into an Assignment. /// NOTE: This can take a very long time. Run with a --release binary. VerifyDisktree(verify_disktree::Cmd), + PromotionFunds(promotion_funds::Cmd), } impl Cmd { @@ -45,6 +46,7 @@ impl Cmd { Self::Server(cmd) => cmd.run(&settings).await, Self::RewardFromDb(cmd) => cmd.run(&settings).await, Self::VerifyDisktree(cmd) => cmd.run(&settings).await, + Self::PromotionFunds(cmd) => cmd.run(&settings).await, } } } From 1632262227dfb3e0fa6ab6396398db2b4d924c7e Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Fri, 20 Sep 2024 11:45:47 -0700 Subject: [PATCH 13/31] port over existing tests --- .../src/service_provider/reward.rs | 334 ++++++++++-------- 1 file changed, 178 insertions(+), 156 deletions(-) diff --git a/mobile_verifier/src/service_provider/reward.rs b/mobile_verifier/src/service_provider/reward.rs index bc943841f..fbabe7181 100644 --- a/mobile_verifier/src/service_provider/reward.rs +++ b/mobile_verifier/src/service_provider/reward.rs @@ -112,12 +112,15 @@ impl RewardInfoColl { ); let sp_rewards = self.total_sp_allocation * rewards_per_share; - let mut rewards = vec![]; - for sp in self.coll.iter() { - rewards.extend(sp.promo_rewards(sp_rewards, reward_epoch)); - rewards.push(sp.carrier_reward(sp_rewards, reward_epoch)); - } - rewards + self.coll + .iter() + .flat_map(|sp| { + let mut rewards = sp.promo_rewards(sp_rewards, reward_epoch); + rewards.push(sp.carrier_reward(sp_rewards, reward_epoch)); + rewards + }) + .filter(|(amount, _r)| *amount > 0) + .collect() } } @@ -156,23 +159,17 @@ impl RewardInfo { total_allocation: Decimal, reward_period: &Range>, ) -> (u64, proto::MobileRewardShare) { - let amount = total_allocation * self.realized_dc_perc; + let amount = (total_allocation * self.realized_dc_perc).to_u64_rounded(); ( - amount - .round_dp_with_strategy(0, RoundingStrategy::MidpointNearestEven) - .to_u64() - .unwrap_or(0), + amount, proto::MobileRewardShare { start_period: reward_period.start.encode_timestamp(), end_period: reward_period.end.encode_timestamp(), reward: Some(proto::Reward::ServiceProviderReward( proto::ServiceProviderReward { service_provider_id: self.sp_id, - amount: amount - .round_dp_with_strategy(0, RoundingStrategy::MidpointNearestEven) - .to_u64() - .unwrap_or(0), + amount, }, )), }, @@ -200,13 +197,10 @@ impl RewardInfo { for r in self.promotion_rewards.iter() { let shares = Decimal::from(r.shares); - let service_provider_amount = sp_amount_per_share * shares; - let matched_amount = matched_amount_per_share * shares; + let service_provider_amount = (sp_amount_per_share * shares).to_u64_rounded(); + let matched_amount = (matched_amount_per_share * shares).to_u64_rounded(); - let total_amount = (service_provider_amount + matched_amount) - .round_dp_with_strategy(0, RoundingStrategy::MidpointNearestEven) - .to_u64() - .unwrap_or(0); + let total_amount = service_provider_amount + matched_amount; rewards.push(( total_amount, @@ -214,14 +208,8 @@ impl RewardInfo { start_period: reward_period.start.encode_timestamp(), end_period: reward_period.end.encode_timestamp(), reward: Some(proto::Reward::PromotionReward(proto::PromotionReward { - service_provider_amount: service_provider_amount - .round_dp_with_strategy(0, RoundingStrategy::MidpointNearestEven) - .to_u64() - .unwrap_or(0), - matched_amount: matched_amount - .round_dp_with_strategy(0, RoundingStrategy::MidpointNearestEven) - .to_u64() - .unwrap_or(0), + service_provider_amount, + matched_amount, entity: Some(r.rewardable_entity.clone().into()), })), }, @@ -271,169 +259,203 @@ fn distribute_unalloc_under_limit(coll: &mut [RewardInfo]) { } } +trait DecimalRoundingExt { + fn to_u64_rounded(&self) -> u64; +} + +impl DecimalRoundingExt for Decimal { + fn to_u64_rounded(&self) -> u64 { + self.round_dp_with_strategy(0, RoundingStrategy::ToZero) + .to_u64() + .unwrap_or(0) + } +} + #[cfg(test)] mod tests { + use chrono::Duration; + use file_store::promotion_reward::Entity; + use helium_proto::services::poc_mobile::{MobileRewardShare, PromotionReward}; + use crate::service_provider::promotions::rewards::PromotionRewardShare; use super::*; - use file_store::promotion_reward::Entity; - use std::collections::HashMap; - use uuid::Uuid; - fn get_unallocated_percent(sps: &RewardInfoColl) -> Decimal { - dec!(1.0) - sps.coll.iter().map(|x| x.dc_perc).sum::() + use super::RewardInfoColl; + + #[test] + fn no_rewards_if_none_allocated() { + let sp_infos = RewardInfoColl::new( + ServiceProviderDCSessions::from([(0, dec!(100))]), + ServiceProviderFunds::from([(0, 5000)]), + ServiceProviderPromotions::from([PromotionRewardShare { + service_provider_id: 0, + rewardable_entity: Entity::SubscriberId(vec![0]), + shares: 1, + }]), + dec!(0), + dec!(0.0001), + ); + + let epoch = epoch(); + assert!(sp_infos.iter_rewards(&epoch).is_empty()); } #[test] - fn test_multiple_with_reward() { - let dc_sessions = HashMap::from_iter([(0, dec!(600)), (1, dec!(600))]); - let promo_funds = HashMap::from_iter([(0, 2000), (1, 4000)]); // bps values - let total_allocation = dec!(1000); - - let sps = RewardInfoColl::new( - ServiceProviderDCSessions(dc_sessions), - ServiceProviderFunds(promo_funds), - ServiceProviderPromotions::from(vec![ + fn no_matched_rewards_if_no_unallocated() { + let total_rewards = dec!(1000); + + let sp_infos = RewardInfoColl::new( + ServiceProviderDCSessions::from([(0, total_rewards)]), + ServiceProviderFunds::from([(0, 5000)]), + ServiceProviderPromotions::from([PromotionRewardShare { + service_provider_id: 0, + rewardable_entity: Entity::SubscriberId(vec![0]), + shares: 1, + }]), + total_rewards, + dec!(0.001), + ); + + let epoch = epoch(); + let promo_rewards = sp_infos.iter_rewards(&epoch).only_promotion_rewards(); + + assert!(!promo_rewards.is_empty()); + for reward in promo_rewards { + assert_eq!(reward.matched_amount, 0); + } + } + + #[test] + fn single_sp_unallocated_less_than_matched_distributed_by_shares() { + // 100 unallocated + let total_rewards = dec!(1100); + let sp_session = dec!(1000); + + let sp_infos = RewardInfoColl::new( + ServiceProviderDCSessions::from([(0, sp_session)]), + ServiceProviderFunds::from([(0, 10000)]), // All rewards allocated to promotions + ServiceProviderPromotions::from([ PromotionRewardShare { service_provider_id: 0, - rewardable_entity: Entity::SubscriberId(Uuid::new_v4().into()), + rewardable_entity: Entity::SubscriberId(vec![0]), shares: 1, }, PromotionRewardShare { service_provider_id: 0, - rewardable_entity: Entity::SubscriberId(Uuid::new_v4().into()), + rewardable_entity: Entity::SubscriberId(vec![1]), shares: 2, }, - PromotionRewardShare { - service_provider_id: 0, - rewardable_entity: Entity::SubscriberId(Uuid::new_v4().into()), - shares: 3, - }, ]), - total_allocation, - dec!(0.0001), + total_rewards, + dec!(0.00001), ); - let x = get_unallocated_percent(&sps); - println!("leftover: {x:?}"); + let epoch = epoch(); + let promo_rewards = sp_infos.iter_rewards(&epoch).only_promotion_rewards(); + assert_eq!(2, promo_rewards.len()); - let reward_period = DateTime::::MIN_UTC..DateTime::::MAX_UTC; - for sp in sps.coll { - println!("{sp:?}"); - for reward in sp.promo_rewards(total_allocation, &reward_period) { - println!("--> {reward:?}"); - } - } + assert_eq!(promo_rewards[0].service_provider_amount, 333); + assert_eq!(promo_rewards[0].matched_amount, 33); + // + assert_eq!(promo_rewards[1].service_provider_amount, 666); + assert_eq!(promo_rewards[1].matched_amount, 66); } #[test] - fn test_multiple_no_reward() { - let dc_sessions = HashMap::from_iter([(0, dec!(600)), (1, dec!(600))]); - let promo_funds = HashMap::from_iter([(0, 2000), (1, 4000)]); // bps values - let total_allocation = dec!(1200); - - let mut sps = RewardInfoColl::new( - ServiceProviderDCSessions(dc_sessions), - ServiceProviderFunds(promo_funds), - ServiceProviderPromotions::default(), - total_allocation, - dec!(0.0001), + fn single_sp_unallocated_more_than_matched_promotion() { + // 1,000 unallocated + let total_rewards = dec!(11_000); + let sp_session = dec!(1000); + + let sp_infos = RewardInfoColl::new( + ServiceProviderDCSessions::from([(0, sp_session)]), + ServiceProviderFunds::from([(0, 10000)]), // All rewards allocated to promotions + ServiceProviderPromotions::from([ + PromotionRewardShare { + service_provider_id: 0, + rewardable_entity: Entity::SubscriberId(vec![0]), + shares: 1, + }, + PromotionRewardShare { + service_provider_id: 0, + rewardable_entity: Entity::SubscriberId(vec![1]), + shares: 2, + }, + ]), + total_rewards, + dec!(0.00001), ); - distribute_unallocated(&mut sps.coll); + let epoch = epoch(); + let promo_rewards = sp_infos.iter_rewards(&epoch).only_promotion_rewards(); + assert_eq!(2, promo_rewards.len()); - println!("{sps:#?}"); + assert_eq!(promo_rewards[0].service_provider_amount, 333); + assert_eq!(promo_rewards[0].matched_amount, 333); + // + assert_eq!(promo_rewards[1].service_provider_amount, 666); + assert_eq!(promo_rewards[1].matched_amount, 666); } #[test] - fn over_limit_with_rewards() { - let promo_0 = PromotionRewardShare { - service_provider_id: 0, - rewardable_entity: file_store::promotion_reward::Entity::SubscriberId( - uuid::Uuid::new_v4().into(), - ), - shares: 1, - }; - let promo_1 = PromotionRewardShare { - service_provider_id: 0, - rewardable_entity: file_store::promotion_reward::Entity::SubscriberId( - uuid::Uuid::new_v4().into(), - ), - shares: 1, - }; - let expected_one = RewardInfo { - sp_id: 0, - dc: dec!(200), - allocated_promo_perc: dec!(0.2), - dc_perc: dec!(0.2), - realized_promo_perc: dec!(0.04), - realized_dc_perc: dec!(0.16), - matched_promo_perc: dec!(0.02667), - - promotion_rewards: ServiceProviderPromotions::from(vec![promo_0.clone()]), - }; - - let one = RewardInfo::new( - 0, - dec!(200), - dec!(0.2), - dec!(1000), - ServiceProviderPromotions::from(vec![promo_0]), - ); - let two = RewardInfo::new( - 1, - dec!(200), - dec!(0.4), - dec!(1000), - ServiceProviderPromotions::from(vec![promo_1]), + fn unallocated_matching_does_not_exceed_promotion() { + // 100 unallocated + let total_rewards = dec!(1100); + let sp_session = dec!(1000); + + let sp_infos = RewardInfoColl::new( + ServiceProviderDCSessions::from([(0, sp_session)]), + ServiceProviderFunds::from([(0, 100)]), // Severely limit promotion rewards + ServiceProviderPromotions::from([ + PromotionRewardShare { + service_provider_id: 0, + rewardable_entity: Entity::SubscriberId(vec![0]), + shares: 1, + }, + PromotionRewardShare { + service_provider_id: 0, + rewardable_entity: Entity::SubscriberId(vec![1]), + shares: 2, + }, + ]), + total_rewards, + dec!(0.00001), ); - let unallocated_perc = dec!(0.08); - let mut x = vec![one, two]; - distribute_unalloc_over_limit(&mut x, unallocated_perc); - - println!("{:#?}", x[0]); - println!("{:#?}", x[1]); + let epoch = epoch(); + let promo_rewards = sp_infos.iter_rewards(&epoch).only_promotion_rewards(); + assert_eq!(2, promo_rewards.len()); - assert_eq!(expected_one, x[0]); + assert_eq!(promo_rewards[0].service_provider_amount, 3); + assert_eq!(promo_rewards[0].matched_amount, 3); + // + assert_eq!(promo_rewards[1].service_provider_amount, 6); + assert_eq!(promo_rewards[1].matched_amount, 6); } - #[test] - fn over_limit_no_rewards() { - let expected_one = RewardInfo { - sp_id: 0, - dc: dec!(200), - allocated_promo_perc: dec!(0.20), - dc_perc: dec!(0.20), - realized_promo_perc: dec!(0.00), - realized_dc_perc: dec!(0.20), - matched_promo_perc: dec!(0.0), - - promotion_rewards: ServiceProviderPromotions::default(), - }; - - let one = RewardInfo::new( - 0, - dec!(200), - dec!(0.2), - dec!(1000), - ServiceProviderPromotions::default(), - ); - let two = RewardInfo::new( - 1, - dec!(200), - dec!(0.4), - dec!(1000), - ServiceProviderPromotions::default(), - ); - - let unallocated_perc = dec!(0.08); - let mut x = vec![one, two]; - distribute_unalloc_over_limit(&mut x, unallocated_perc); + fn epoch() -> Range> { + let now = Utc::now(); + let epoch = now - Duration::hours(24)..now; + epoch + } - println!("{:#?}", x[0]); - println!("{:#?}", x[1]); + trait PromoRewardFiltersExt { + fn only_promotion_rewards(&self) -> Vec; + } - assert_eq!(expected_one, x[0]); + impl PromoRewardFiltersExt for Vec<(u64, MobileRewardShare)> { + fn only_promotion_rewards(&self) -> Vec { + self.clone() + .into_iter() + .filter_map(|(_, r)| { + if let Some(proto::Reward::PromotionReward(reward)) = r.reward { + Some(reward) + } else { + None + } + }) + .collect() + } } } From 9167d53e6eaf10e88768615bb3d1cbde8b9dcdfa Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Fri, 20 Sep 2024 16:57:57 -0700 Subject: [PATCH 14/31] service provider reward info collection can hold onto the reward date --- mobile_verifier/src/rewarder.rs | 3 +- .../src/service_provider/reward.rs | 36 ++++++++++--------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index 4e7b9026f..00d32e33c 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -609,6 +609,7 @@ pub async fn reward_service_providers( promo_rewards, total_sp_rewards, mobile_bone_price, + reward_period.clone(), ); let mut unallocated_sp_rewards = total_sp_rewards @@ -616,7 +617,7 @@ pub async fn reward_service_providers( .to_u64() .unwrap_or(0); - for (amount, reward) in sps.iter_rewards(reward_period) { + for (amount, reward) in sps.iter_rewards() { unallocated_sp_rewards -= amount; mobile_rewards.write(reward, []).await?.await??; } diff --git a/mobile_verifier/src/service_provider/reward.rs b/mobile_verifier/src/service_provider/reward.rs index fbabe7181..db953da8d 100644 --- a/mobile_verifier/src/service_provider/reward.rs +++ b/mobile_verifier/src/service_provider/reward.rs @@ -3,7 +3,7 @@ use std::ops::Range; use chrono::{DateTime, Utc}; use file_store::traits::TimestampEncode; -use rust_decimal::{prelude::ToPrimitive, Decimal, RoundingStrategy}; +use rust_decimal::{Decimal, RoundingStrategy}; use rust_decimal_macros::dec; use crate::reward_shares::{dc_to_mobile_bones, DEFAULT_PREC}; @@ -42,6 +42,7 @@ pub struct RewardInfoColl { total_sp_allocation: Decimal, all_transfer: Decimal, mobile_bone_price: Decimal, + reward_epoch: Range>, } #[derive(Debug, Clone, PartialEq)] @@ -74,6 +75,7 @@ impl RewardInfoColl { rewards: ServiceProviderPromotions, total_sp_allocation: Decimal, mobile_bone_price: Decimal, + reward_epoch: Range>, ) -> Self { let all_transfer = dc_sessions.all_transfer(); @@ -82,6 +84,7 @@ impl RewardInfoColl { all_transfer, total_sp_allocation, mobile_bone_price, + reward_epoch, }; let used_allocation = total_sp_allocation.max(all_transfer); @@ -101,10 +104,7 @@ impl RewardInfoColl { me } - pub fn iter_rewards( - &self, - reward_epoch: &Range>, - ) -> Vec<(u64, proto::MobileRewardShare)> { + pub fn iter_rewards(&self) -> Vec<(u64, proto::MobileRewardShare)> { let rewards_per_share = rewards_per_share( self.all_transfer, self.total_sp_allocation, @@ -115,8 +115,8 @@ impl RewardInfoColl { self.coll .iter() .flat_map(|sp| { - let mut rewards = sp.promo_rewards(sp_rewards, reward_epoch); - rewards.push(sp.carrier_reward(sp_rewards, reward_epoch)); + let mut rewards = sp.promo_rewards(sp_rewards, &self.reward_epoch); + rewards.push(sp.carrier_reward(sp_rewards, &self.reward_epoch)); rewards }) .filter(|(amount, _r)| *amount > 0) @@ -265,6 +265,8 @@ trait DecimalRoundingExt { impl DecimalRoundingExt for Decimal { fn to_u64_rounded(&self) -> u64 { + use rust_decimal::{prelude::ToPrimitive, RoundingStrategy}; + self.round_dp_with_strategy(0, RoundingStrategy::ToZero) .to_u64() .unwrap_or(0) @@ -295,10 +297,10 @@ mod tests { }]), dec!(0), dec!(0.0001), + epoch(), ); - let epoch = epoch(); - assert!(sp_infos.iter_rewards(&epoch).is_empty()); + assert!(sp_infos.iter_rewards().is_empty()); } #[test] @@ -315,10 +317,10 @@ mod tests { }]), total_rewards, dec!(0.001), + epoch(), ); - let epoch = epoch(); - let promo_rewards = sp_infos.iter_rewards(&epoch).only_promotion_rewards(); + let promo_rewards = sp_infos.iter_rewards().only_promotion_rewards(); assert!(!promo_rewards.is_empty()); for reward in promo_rewards { @@ -349,10 +351,10 @@ mod tests { ]), total_rewards, dec!(0.00001), + epoch(), ); - let epoch = epoch(); - let promo_rewards = sp_infos.iter_rewards(&epoch).only_promotion_rewards(); + let promo_rewards = sp_infos.iter_rewards().only_promotion_rewards(); assert_eq!(2, promo_rewards.len()); assert_eq!(promo_rewards[0].service_provider_amount, 333); @@ -385,10 +387,10 @@ mod tests { ]), total_rewards, dec!(0.00001), + epoch(), ); - let epoch = epoch(); - let promo_rewards = sp_infos.iter_rewards(&epoch).only_promotion_rewards(); + let promo_rewards = sp_infos.iter_rewards().only_promotion_rewards(); assert_eq!(2, promo_rewards.len()); assert_eq!(promo_rewards[0].service_provider_amount, 333); @@ -421,10 +423,10 @@ mod tests { ]), total_rewards, dec!(0.00001), + epoch(), ); - let epoch = epoch(); - let promo_rewards = sp_infos.iter_rewards(&epoch).only_promotion_rewards(); + let promo_rewards = sp_infos.iter_rewards().only_promotion_rewards(); assert_eq!(2, promo_rewards.len()); assert_eq!(promo_rewards[0].service_provider_amount, 3); From 72c770624a0fc755ddd1787f3c0b3d6628d052c3 Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Fri, 20 Sep 2024 16:58:23 -0700 Subject: [PATCH 15/31] add test with seeded promotions --- .../tests/integrations/common/mod.rs | 26 ++- .../tests/integrations/rewarder_sp_rewards.rs | 160 +++++++++++++++++- 2 files changed, 175 insertions(+), 11 deletions(-) diff --git a/mobile_verifier/tests/integrations/common/mod.rs b/mobile_verifier/tests/integrations/common/mod.rs index 47fe1cc30..f2e5a75e3 100644 --- a/mobile_verifier/tests/integrations/common/mod.rs +++ b/mobile_verifier/tests/integrations/common/mod.rs @@ -9,8 +9,9 @@ use helium_proto::services::{ mobile_config::NetworkKeyRole, poc_mobile::{ mobile_reward_share::Reward as MobileReward, radio_reward_v2, GatewayReward, - MobileRewardShare, OracleBoostingHexAssignment, OracleBoostingReportV1, RadioReward, - RadioRewardV2, ServiceProviderReward, SpeedtestAvg, SubscriberReward, UnallocatedReward, + MobileRewardShare, OracleBoostingHexAssignment, OracleBoostingReportV1, PromotionReward, + RadioReward, RadioRewardV2, ServiceProviderReward, SpeedtestAvg, SubscriberReward, + UnallocatedReward, }, }; use hex_assignments::{Assignment, HexAssignment, HexBoostData}; @@ -135,7 +136,7 @@ impl MockFileSinkReceiver { pub async fn receive_gateway_reward(&mut self) -> GatewayReward { match self.receive("receive_gateway_reward").await { Some(mobile_reward) => { - println!("mobile_reward: {:?}", mobile_reward); + // println!("mobile_reward: {:?}", mobile_reward); match mobile_reward.reward { Some(MobileReward::GatewayReward(r)) => r, _ => panic!("failed to get gateway reward"), @@ -148,7 +149,7 @@ impl MockFileSinkReceiver { pub async fn receive_service_provider_reward(&mut self) -> ServiceProviderReward { match self.receive("receive_service_provider_reward").await { Some(mobile_reward) => { - println!("mobile_reward: {:?}", mobile_reward); + // println!("mobile_reward: {:?}", mobile_reward); match mobile_reward.reward { Some(MobileReward::ServiceProviderReward(r)) => r, _ => panic!("failed to get service provider reward"), @@ -161,7 +162,7 @@ impl MockFileSinkReceiver { pub async fn receive_subscriber_reward(&mut self) -> SubscriberReward { match self.receive("receive_subscriber_reward").await { Some(mobile_reward) => { - println!("mobile_reward: {:?}", mobile_reward); + // println!("mobile_reward: {:?}", mobile_reward); match mobile_reward.reward { Some(MobileReward::SubscriberReward(r)) => r, _ => panic!("failed to get subscriber reward"), @@ -171,10 +172,23 @@ impl MockFileSinkReceiver { } } + pub async fn receive_promotion_reward(&mut self) -> PromotionReward { + match self.receive("receive_promotion_reward").await { + Some(mobile_reward) => { + // println!("mobile_reward: {:?}", mobile_reward); + match mobile_reward.reward { + Some(MobileReward::PromotionReward(r)) => r, + _ => panic!("failed to get promotion reward"), + } + } + None => panic!("failed to receive promotion reward"), + } + } + pub async fn receive_unallocated_reward(&mut self) -> UnallocatedReward { match self.receive("receive_unallocated_reward").await { Some(mobile_reward) => { - println!("mobile_reward: {:?}", mobile_reward); + // println!("mobile_reward: {:?}", mobile_reward); match mobile_reward.reward { Some(MobileReward::UnallocatedReward(r)) => r, _ => panic!("failed to get unallocated reward"), diff --git a/mobile_verifier/tests/integrations/rewarder_sp_rewards.rs b/mobile_verifier/tests/integrations/rewarder_sp_rewards.rs index 9aa63f3dc..bee29a461 100644 --- a/mobile_verifier/tests/integrations/rewarder_sp_rewards.rs +++ b/mobile_verifier/tests/integrations/rewarder_sp_rewards.rs @@ -3,6 +3,8 @@ use std::string::ToString; use async_trait::async_trait; use chrono::{DateTime, Duration as ChronoDuration, Utc}; +use file_store::promotion_reward::{self, PromotionReward}; +use helium_crypto::PublicKeyBinary; use helium_proto::{ services::poc_mobile::{ MobileRewardShare, ServiceProviderReward, UnallocatedReward, UnallocatedRewardType, @@ -12,10 +14,18 @@ use helium_proto::{ use rust_decimal::prelude::*; use rust_decimal_macros::dec; use sqlx::{PgPool, Postgres, Transaction}; +use uuid::Uuid; use crate::common::{self, MockFileSinkReceiver}; use mobile_config::client::{carrier_service_client::CarrierServiceVerifier, ClientError}; -use mobile_verifier::{data_session, reward_shares, rewarder}; +use mobile_verifier::{ + data_session, reward_shares, rewarder, + service_provider::{ + self, + promotions::{funds::save_promotion_fund, rewards::save_promotion_reward}, + ServiceProviderId, + }, +}; const HOTSPOT_1: &str = "112NqN2WWMwtK29PMzRby62fDydBJfsCLkCAf392stdok48ovNT6"; const HOTSPOT_2: &str = "11eX55faMbqZB7jzN4p67m6w7ScPMH6ubnvCjCPLh72J49PaJEL"; @@ -144,6 +154,101 @@ async fn test_service_provider_rewards_invalid_sp(pool: PgPool) -> anyhow::Resul Ok(()) } +#[sqlx::test] +async fn test_service_provider_promotion_rewards(pool: PgPool) -> anyhow::Result<()> { + // Single SP has allocated shares for a few of their subscribers. + // Rewards are matched by the unallocated SP rewards for the subscribers + + let valid_sps = HashMap::from_iter([(PAYER_1.to_string(), SP_1.to_string())]); + let carrier_client = MockCarrierServiceClient::new(valid_sps); + + let now = Utc::now(); + let epoch = (now - ChronoDuration::hours(24))..now; + let (mobile_rewards_client, mut mobile_rewards) = common::create_file_sink(); + + let mut txn = pool.begin().await?; + seed_hotspot_data(epoch.end, &mut txn).await?; // DC transferred == 6,000 reward amount + seed_sp_promotion_rewards_with_random_subscribers( + PAYER_1.to_string().parse().unwrap(), + &[1, 2, 3], + &mut txn, + ) + .await?; + // promotions allocated 10.00% + seed_sp_promotion_rewards_funds(&[(0, 1500)], &mut txn).await?; + txn.commit().await?; + + let (_, rewards) = tokio::join!( + rewarder::reward_service_providers( + &pool, + &carrier_client, + &mobile_rewards_client, + &epoch, + dec!(0.00001) + ), + async move { + let mut promos = vec![ + mobile_rewards.receive_promotion_reward().await, + mobile_rewards.receive_promotion_reward().await, + mobile_rewards.receive_promotion_reward().await, + ]; + // sort by awarded amount least -> most + promos.sort_by_key(|a| a.service_provider_amount); + + let sp_reward = mobile_rewards.receive_service_provider_reward().await; + let unallocated = mobile_rewards.receive_unallocated_reward().await; + + mobile_rewards.assert_no_messages(); + + (promos, sp_reward, unallocated) + } + ); + + let (promos, sp_reward, unallocated) = rewards; + let promo_reward_1 = promos[0].clone(); + let promo_reward_2 = promos[1].clone(); + let promo_reward_3 = promos[2].clone(); + + // 1 share + assert_eq!(promo_reward_1.service_provider_amount, 1_500); + assert_eq!(promo_reward_1.matched_amount, 1_500); + + // 2 shares + assert_eq!(promo_reward_2.service_provider_amount, 3_000); + assert_eq!(promo_reward_2.matched_amount, 3_000); + + // 3 shares + assert_eq!(promo_reward_3.service_provider_amount, 4_500); + assert_eq!(promo_reward_3.matched_amount, 4_500); + + // dc_percentage * total_sp_allocation rounded down + assert_eq!(sp_reward.amount, 50_999); + + let unallocated_sp_rewards = get_unallocated_sp_rewards(&epoch); + let expected_unallocated = unallocated_sp_rewards + - 50_999 // 85% service provider rewards rounded down + - 9_000 // 15% service provider promotions + - 9_000; // matched promotion + + assert_eq!(unallocated.amount, expected_unallocated); + + // Ensure the cleanup job can run + let mut txn = pool.begin().await?; + + service_provider::promotions::rewards::clear_promotion_rewards(&mut txn, &Utc::now()).await?; + txn.commit().await?; + + let promos = service_provider::promotions::rewards::fetch_promotion_rewards( + &pool, + &carrier_client, + &(epoch.start..Utc::now()), + ) + .await?; + assert!(promos.is_empty()); + + Ok(()) +} + async fn receive_expected_rewards( mobile_rewards: &mut MockFileSinkReceiver, ) -> anyhow::Result<(ServiceProviderReward, UnallocatedReward)> { @@ -171,7 +276,7 @@ async fn seed_hotspot_data( payer: PAYER_1.parse().unwrap(), upload_bytes: 1024 * 1000, download_bytes: 1024 * 10000, - num_dcs: 10000, + num_dcs: 10_000, received_timestamp: ts - ChronoDuration::hours(1), }; @@ -180,7 +285,7 @@ async fn seed_hotspot_data( payer: PAYER_1.parse().unwrap(), upload_bytes: 1024 * 1000, download_bytes: 1024 * 50000, - num_dcs: 50000, + num_dcs: 50_000, received_timestamp: ts - ChronoDuration::hours(2), }; @@ -198,7 +303,7 @@ async fn seed_hotspot_data_invalid_sp( payer: PAYER_1.parse().unwrap(), upload_bytes: 1024 * 1000, download_bytes: 1024 * 10000, - num_dcs: 10000, + num_dcs: 10_000, received_timestamp: ts - ChronoDuration::hours(2), }; @@ -207,7 +312,7 @@ async fn seed_hotspot_data_invalid_sp( payer: PAYER_2.parse().unwrap(), upload_bytes: 1024 * 1000, download_bytes: 1024 * 50000, - num_dcs: 50000, + num_dcs: 50_000, received_timestamp: ts - ChronoDuration::hours(2), }; @@ -215,3 +320,48 @@ async fn seed_hotspot_data_invalid_sp( data_session_2.save(txn).await?; Ok(()) } + +// Service Provider promotion rewards are verified during ingest. When you write +// directly to the database, the assumption is the entity and the payer are +// valid. +async fn seed_sp_promotion_rewards_with_random_subscribers( + payer: PublicKeyBinary, + sub_shares: &[u64], + txn: &mut Transaction<'_, Postgres>, +) -> anyhow::Result<()> { + for &shares in sub_shares { + save_promotion_reward( + txn, + &PromotionReward { + entity: promotion_reward::Entity::SubscriberId(Uuid::new_v4().into()), + shares, + timestamp: Utc::now() - chrono::Duration::hours(2), + received_timestamp: Utc::now(), + carrier_pub_key: payer.clone(), + signature: vec![], + }, + ) + .await?; + } + + Ok(()) +} + +async fn seed_sp_promotion_rewards_funds( + sp_fund_allocations: &[(ServiceProviderId, u16)], + txn: &mut Transaction<'_, Postgres>, +) -> anyhow::Result<()> { + for (sp_id, basis_points) in sp_fund_allocations { + save_promotion_fund(txn, *sp_id, *basis_points).await?; + } + + Ok(()) +} + +// Helper for turning Decimal -> u64 to compare against output rewards +fn get_unallocated_sp_rewards(epoch: &std::ops::Range>) -> u64 { + reward_shares::get_scheduled_tokens_for_service_providers(epoch.end - epoch.start) + .round_dp_with_strategy(0, RoundingStrategy::ToZero) + .to_u64() + .unwrap_or(0) +} From df41d2c8584f8db2da84dd1ff787ee8ab5e3078f Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Fri, 20 Sep 2024 16:59:15 -0700 Subject: [PATCH 16/31] the nature of the math has changed because we're dealing with percentages, and not doing bankers rounding or nearest even, the likelihood that a calculation comes out extremely close to a hole number then gets rounded down is increased. Especially when dealing with percentages to the 7th decimal point and a base number in the trillions. --- mobile_verifier/src/service_provider/reward.rs | 3 +++ mobile_verifier/tests/integrations/rewarder_sp_rewards.rs | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/mobile_verifier/src/service_provider/reward.rs b/mobile_verifier/src/service_provider/reward.rs index db953da8d..b3019d24e 100644 --- a/mobile_verifier/src/service_provider/reward.rs +++ b/mobile_verifier/src/service_provider/reward.rs @@ -111,6 +111,9 @@ impl RewardInfoColl { self.mobile_bone_price, ); let sp_rewards = self.total_sp_allocation * rewards_per_share; + // NOTE(mj): `rewards_per_share * self.dc` vs `sp_rewards * self.dc_perc` + // They're veeeeery close. But the % multiplication produces a floating point number + // that will typically be rounded down. self.coll .iter() diff --git a/mobile_verifier/tests/integrations/rewarder_sp_rewards.rs b/mobile_verifier/tests/integrations/rewarder_sp_rewards.rs index bee29a461..2450bc933 100644 --- a/mobile_verifier/tests/integrations/rewarder_sp_rewards.rs +++ b/mobile_verifier/tests/integrations/rewarder_sp_rewards.rs @@ -94,13 +94,16 @@ async fn test_service_provider_rewards(pool: PgPool) -> anyhow::Result<()> { .unwrap() .to_string() ); - assert_eq!(6000, sp_reward.amount); + assert_eq!(5_999, sp_reward.amount); + // assert_eq!(6_000, sp_reward.amount); assert_eq!( UnallocatedRewardType::ServiceProvider as i32, unallocated_reward.reward_type ); - assert_eq!(8_219_178_076_191, unallocated_reward.amount); + assert_eq!(8_219_178_076_192, unallocated_reward.amount); + // assert_eq!(8_219_178_076_191, unallocated_reward.amount); + // confirm the total rewards allocated matches expectations let expected_sum = reward_shares::get_scheduled_tokens_for_service_providers(epoch.end - epoch.start) From 4c0c05e4fa6782bbfbf3ecb823dcc77a4ea6ac24 Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Fri, 20 Sep 2024 17:08:11 -0700 Subject: [PATCH 17/31] make sure we're cleaning up promotions --- mobile_verifier/src/rewarder.rs | 2 ++ mobile_verifier/src/service_provider/mod.rs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index 00d32e33c..b59e68bda 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -296,6 +296,8 @@ where coverage::clear_coverage_objects(&mut transaction, &reward_period.start).await?; sp_boosted_rewards_bans::clear_bans(&mut transaction, reward_period.start).await?; subscriber_verified_mapping_event::clear(&mut transaction, &reward_period.start).await?; + service_provider::db::clear_promotion_rewards(&mut transaction, &reward_period.start) + .await?; // subscriber_location::clear_location_shares(&mut transaction, &reward_period.end).await?; let next_reward_period = scheduler.next_reward_period(); diff --git a/mobile_verifier/src/service_provider/mod.rs b/mobile_verifier/src/service_provider/mod.rs index 592d2a415..81534661a 100644 --- a/mobile_verifier/src/service_provider/mod.rs +++ b/mobile_verifier/src/service_provider/mod.rs @@ -11,7 +11,7 @@ pub mod reward; pub mod db { pub use super::dc_sessions::fetch_dc_sessions; pub use super::promotions::funds::fetch_promotion_funds; - pub use super::promotions::rewards::fetch_promotion_rewards; + pub use super::promotions::rewards::{clear_promotion_rewards, fetch_promotion_rewards}; } // This type is used in lieu of the helium_proto::ServiceProvider enum so we can From 9c1740474027ba80829d086a3015518fa81298cb Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Fri, 20 Sep 2024 17:08:32 -0700 Subject: [PATCH 18/31] stub for reporting sp allocations I'm hoping there's an easy way to do this someone can point me to --- mobile_verifier/src/rewarder.rs | 3 +-- mobile_verifier/src/service_provider/mod.rs | 6 ++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index b59e68bda..d49c713f5 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -313,8 +313,7 @@ where boosted_poc_bones_per_reward_share: Some(helium_proto::Decimal { value: poc_dc_shares.boost.to_string(), }), - // TODO: Filled in with the next PR - sp_allocations: vec![], + sp_allocations: service_provider::reward_data_sp_allocations(), }; self.reward_manifests .write( diff --git a/mobile_verifier/src/service_provider/mod.rs b/mobile_verifier/src/service_provider/mod.rs index 81534661a..9587af31e 100644 --- a/mobile_verifier/src/service_provider/mod.rs +++ b/mobile_verifier/src/service_provider/mod.rs @@ -1,6 +1,7 @@ use std::ops::Range; use chrono::{DateTime, Utc}; +use helium_proto::ServiceProviderAllocation; pub use promotions::daemon::PromotionDaemon; pub use reward::RewardInfoColl; @@ -23,3 +24,8 @@ pub fn get_scheduled_tokens(reward_period: &Range>) -> rust_decima let duration = reward_period.end - reward_period.start; crate::reward_shares::get_scheduled_tokens_for_service_providers(duration) } + +pub fn reward_data_sp_allocations() -> Vec { + // TODO: How do you go from a service provider id to the payer key? + vec![] +} From 1012a00d9e6b965706cad8510b78d11b00bebcf5 Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Fri, 20 Sep 2024 17:17:54 -0700 Subject: [PATCH 19/31] rename collection class for consistency it get a bit verbose with all the service_provider flying around, but I think it will be rather helpful when you only have the type and it hasn't been aliased. --- mobile_verifier/src/reward_shares.rs | 8 ++--- mobile_verifier/src/rewarder.rs | 4 +-- mobile_verifier/src/service_provider/mod.rs | 2 +- .../src/service_provider/reward.rs | 34 ++++++++++--------- 4 files changed, 25 insertions(+), 23 deletions(-) diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index 8ae4fdc85..3ad7714aa 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -2229,7 +2229,7 @@ mod test { let epoch = (now - Duration::hours(1))..now; let total_sp_rewards = service_provider::get_scheduled_tokens(&epoch); - let sp_reward_infos = service_provider::RewardInfoColl::new( + let sp_reward_infos = service_provider::ServiceProviderRewardInfos::new( ServiceProviderDCSessions::from([(sp1, dec!(1000))]), ServiceProviderFunds::default(), ServiceProviderPromotions::default(), @@ -2273,7 +2273,7 @@ mod test { let total_rewards_value_in_dc = mobile_bones_to_dc(total_sp_rewards_in_bones, mobile_bone_price); - let sp_reward_infos = service_provider::RewardInfoColl::new( + let sp_reward_infos = service_provider::ServiceProviderRewardInfos::new( // force the service provider to have spend more DC than total rewardable ServiceProviderDCSessions::from([(sp1, total_rewards_value_in_dc * dec!(2.0))]), ServiceProviderFunds::default(), @@ -2319,7 +2319,7 @@ mod test { let epoch = (now - Duration::hours(1))..now; let total_sp_rewards_in_bones = dec!(500_000_000) * dec!(1_000_000); - let sp_reward_infos = service_provider::RewardInfoColl::new( + let sp_reward_infos = service_provider::ServiceProviderRewardInfos::new( ServiceProviderDCSessions::from([(sp1, dec!(100_000_000))]), ServiceProviderFunds::default(), ServiceProviderPromotions::default(), @@ -2364,7 +2364,7 @@ mod test { let epoch = (now - Duration::hours(24))..now; let total_sp_rewards_in_bones = dec!(500_000_000) * dec!(1_000_000); - let sp_reward_infos = service_provider::RewardInfoColl::new( + let sp_reward_infos = service_provider::ServiceProviderRewardInfos::new( ServiceProviderDCSessions::from([(sp1, dec!(100_000_000_000))]), ServiceProviderFunds::default(), ServiceProviderPromotions::default(), diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index d49c713f5..59ff9ede7 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -597,14 +597,14 @@ pub async fn reward_service_providers( reward_period: &Range>, mobile_bone_price: Decimal, ) -> anyhow::Result<()> { - use service_provider::db; + use service_provider::{db, ServiceProviderRewardInfos}; let dc_sessions = db::fetch_dc_sessions(pool, carrier_client, reward_period).await?; let promo_funds = db::fetch_promotion_funds(pool).await?; let promo_rewards = db::fetch_promotion_rewards(pool, carrier_client, reward_period).await?; let total_sp_rewards = service_provider::get_scheduled_tokens(reward_period); - let sps = service_provider::RewardInfoColl::new( + let sps = ServiceProviderRewardInfos::new( dc_sessions, promo_funds, promo_rewards, diff --git a/mobile_verifier/src/service_provider/mod.rs b/mobile_verifier/src/service_provider/mod.rs index 9587af31e..d45aa4138 100644 --- a/mobile_verifier/src/service_provider/mod.rs +++ b/mobile_verifier/src/service_provider/mod.rs @@ -3,7 +3,7 @@ use std::ops::Range; use chrono::{DateTime, Utc}; use helium_proto::ServiceProviderAllocation; pub use promotions::daemon::PromotionDaemon; -pub use reward::RewardInfoColl; +pub use reward::ServiceProviderRewardInfos; pub mod dc_sessions; pub mod promotions; diff --git a/mobile_verifier/src/service_provider/reward.rs b/mobile_verifier/src/service_provider/reward.rs index b3019d24e..b25ec10a3 100644 --- a/mobile_verifier/src/service_provider/reward.rs +++ b/mobile_verifier/src/service_provider/reward.rs @@ -35,9 +35,9 @@ pub fn rewards_per_share( } } -/// Container for ['ServiceProvideRewardInfo'] +/// Container for all Service Provider rewarding #[derive(Debug)] -pub struct RewardInfoColl { +pub struct ServiceProviderRewardInfos { coll: Vec, total_sp_allocation: Decimal, all_transfer: Decimal, @@ -45,30 +45,32 @@ pub struct RewardInfoColl { reward_epoch: Range>, } +// Represents a single Service Providers information for rewarding, +// only used internally. #[derive(Debug, Clone, PartialEq)] -pub struct RewardInfo { +struct RewardInfo { // proto::ServiceProvider enum repr sp_id: i32, - // How much DC in sessions + // Total DC transferred for reward epoch dc: Decimal, + // % of total allocated rewards for dc transfer + dc_perc: Decimal, // % allocated from DC to promo rewards (found in db from file from on chain) allocated_promo_perc: Decimal, - // % of total allocated rewards for all service providers - dc_perc: Decimal, - // % of total allocated rewards going towards rewards + // % of total allocated rewards going towards promotions realized_promo_perc: Decimal, - // % usable dc percentage of total rewards + // % of total allocated rewards awarded for dc transfer realized_dc_perc: Decimal, - // % matched promotions from unallocated + // % matched promotions from unallocated, can never exceed realized_promo_perc matched_promo_perc: Decimal, // Rewards for the epoch promotion_rewards: ServiceProviderPromotions, } -impl RewardInfoColl { +impl ServiceProviderRewardInfos { pub fn new( dc_sessions: ServiceProviderDCSessions, promo_funds: ServiceProviderFunds, @@ -286,11 +288,11 @@ mod tests { use super::*; - use super::RewardInfoColl; + use super::ServiceProviderRewardInfos; #[test] fn no_rewards_if_none_allocated() { - let sp_infos = RewardInfoColl::new( + let sp_infos = ServiceProviderRewardInfos::new( ServiceProviderDCSessions::from([(0, dec!(100))]), ServiceProviderFunds::from([(0, 5000)]), ServiceProviderPromotions::from([PromotionRewardShare { @@ -310,7 +312,7 @@ mod tests { fn no_matched_rewards_if_no_unallocated() { let total_rewards = dec!(1000); - let sp_infos = RewardInfoColl::new( + let sp_infos = ServiceProviderRewardInfos::new( ServiceProviderDCSessions::from([(0, total_rewards)]), ServiceProviderFunds::from([(0, 5000)]), ServiceProviderPromotions::from([PromotionRewardShare { @@ -337,7 +339,7 @@ mod tests { let total_rewards = dec!(1100); let sp_session = dec!(1000); - let sp_infos = RewardInfoColl::new( + let sp_infos = ServiceProviderRewardInfos::new( ServiceProviderDCSessions::from([(0, sp_session)]), ServiceProviderFunds::from([(0, 10000)]), // All rewards allocated to promotions ServiceProviderPromotions::from([ @@ -373,7 +375,7 @@ mod tests { let total_rewards = dec!(11_000); let sp_session = dec!(1000); - let sp_infos = RewardInfoColl::new( + let sp_infos = ServiceProviderRewardInfos::new( ServiceProviderDCSessions::from([(0, sp_session)]), ServiceProviderFunds::from([(0, 10000)]), // All rewards allocated to promotions ServiceProviderPromotions::from([ @@ -409,7 +411,7 @@ mod tests { let total_rewards = dec!(1100); let sp_session = dec!(1000); - let sp_infos = RewardInfoColl::new( + let sp_infos = ServiceProviderRewardInfos::new( ServiceProviderDCSessions::from([(0, sp_session)]), ServiceProviderFunds::from([(0, 100)]), // Severely limit promotion rewards ServiceProviderPromotions::from([ From 4c50dc340c6034d37c5e42d946763e097fe35f89 Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Fri, 20 Sep 2024 17:21:21 -0700 Subject: [PATCH 20/31] fix up uncompiled tests --- mobile_verifier/src/reward_shares.rs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index 3ad7714aa..713438634 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -775,6 +775,7 @@ mod test { self, dc_sessions::ServiceProviderDCSessions, promotions::{funds::ServiceProviderFunds, rewards::ServiceProviderPromotions}, + ServiceProviderRewardInfos, }, speedtests::Speedtest, speedtests_average::SpeedtestAverage, @@ -2229,18 +2230,19 @@ mod test { let epoch = (now - Duration::hours(1))..now; let total_sp_rewards = service_provider::get_scheduled_tokens(&epoch); - let sp_reward_infos = service_provider::ServiceProviderRewardInfos::new( + let sp_reward_infos = ServiceProviderRewardInfos::new( ServiceProviderDCSessions::from([(sp1, dec!(1000))]), ServiceProviderFunds::default(), ServiceProviderPromotions::default(), total_sp_rewards, mobile_bone_price, + epoch.clone(), ); let mut sp_rewards = HashMap::::new(); let mut allocated_sp_rewards = 0_u64; - for (reward_amount, reward) in sp_reward_infos.iter_rewards(&epoch) { + for (reward_amount, reward) in sp_reward_infos.iter_rewards() { if let Some(MobileReward::ServiceProviderReward(r)) = reward.reward { sp_rewards.insert(r.service_provider_id, r.amount); assert_eq!(reward_amount, r.amount); @@ -2273,19 +2275,20 @@ mod test { let total_rewards_value_in_dc = mobile_bones_to_dc(total_sp_rewards_in_bones, mobile_bone_price); - let sp_reward_infos = service_provider::ServiceProviderRewardInfos::new( + let sp_reward_infos = ServiceProviderRewardInfos::new( // force the service provider to have spend more DC than total rewardable ServiceProviderDCSessions::from([(sp1, total_rewards_value_in_dc * dec!(2.0))]), ServiceProviderFunds::default(), ServiceProviderPromotions::default(), total_rewards_value_in_dc, mobile_bone_price, + epoch.clone(), ); let mut sp_rewards = HashMap::new(); let mut allocated_sp_rewards = 0_u64; - for (reward_amount, reward) in sp_reward_infos.iter_rewards(&epoch) { + for (reward_amount, reward) in sp_reward_infos.iter_rewards() { if let Some(MobileReward::ServiceProviderReward(r)) = reward.reward { sp_rewards.insert(r.service_provider_id, r.amount); assert_eq!(reward_amount, r.amount); @@ -2319,18 +2322,19 @@ mod test { let epoch = (now - Duration::hours(1))..now; let total_sp_rewards_in_bones = dec!(500_000_000) * dec!(1_000_000); - let sp_reward_infos = service_provider::ServiceProviderRewardInfos::new( + let sp_reward_infos = ServiceProviderRewardInfos::new( ServiceProviderDCSessions::from([(sp1, dec!(100_000_000))]), ServiceProviderFunds::default(), ServiceProviderPromotions::default(), total_sp_rewards_in_bones, mobile_bone_price, + epoch.clone(), ); let mut sp_rewards = HashMap::new(); let mut allocated_sp_rewards = 0_u64; - for (reward_amount, reward) in sp_reward_infos.iter_rewards(&epoch) { + for (reward_amount, reward) in sp_reward_infos.iter_rewards() { if let Some(MobileReward::ServiceProviderReward(r)) = reward.reward { sp_rewards.insert(r.service_provider_id, r.amount); assert_eq!(reward_amount, r.amount); @@ -2364,18 +2368,19 @@ mod test { let epoch = (now - Duration::hours(24))..now; let total_sp_rewards_in_bones = dec!(500_000_000) * dec!(1_000_000); - let sp_reward_infos = service_provider::ServiceProviderRewardInfos::new( + let sp_reward_infos = ServiceProviderRewardInfos::new( ServiceProviderDCSessions::from([(sp1, dec!(100_000_000_000))]), ServiceProviderFunds::default(), ServiceProviderPromotions::default(), total_sp_rewards_in_bones, mobile_bone_price, + epoch.clone(), ); let mut sp_rewards = HashMap::new(); let mut allocated_sp_rewards = 0_u64; - for (reward_amount, reward) in sp_reward_infos.iter_rewards(&epoch) { + for (reward_amount, reward) in sp_reward_infos.iter_rewards() { if let Some(MobileReward::ServiceProviderReward(r)) = reward.reward { sp_rewards.insert(r.service_provider_id, r.amount); assert_eq!(reward_amount, r.amount); From 5b972bd76f852e090c7ac62060bb1197de6ee717 Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Fri, 27 Sep 2024 09:31:04 -0700 Subject: [PATCH 21/31] add tests from good examples in original PR https://github.com/helium/oracles/pull/856 --- .../src/service_provider/reward.rs | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) diff --git a/mobile_verifier/src/service_provider/reward.rs b/mobile_verifier/src/service_provider/reward.rs index b25ec10a3..c22a03e65 100644 --- a/mobile_verifier/src/service_provider/reward.rs +++ b/mobile_verifier/src/service_provider/reward.rs @@ -290,6 +290,158 @@ mod tests { use super::ServiceProviderRewardInfos; + impl ServiceProviderRewardInfos { + fn iter_sp_rewards(&self, sp_id: i32) -> Vec { + let rewards_per_share = rewards_per_share( + self.all_transfer, + self.total_sp_allocation, + self.mobile_bone_price, + ); + let sp_rewards = self.total_sp_allocation * rewards_per_share; + + for info in self.coll.iter() { + if info.sp_id == sp_id { + let mut result = info.promo_rewards(sp_rewards, &self.reward_epoch); + result.push(info.carrier_reward(sp_rewards, &self.reward_epoch)); + return result.into_iter().map(|(_, x)| x).collect(); + } + } + vec![] + } + fn single_sp_rewards( + &self, + sp_id: i32, + ) -> (proto::PromotionReward, proto::ServiceProviderReward) { + let binding = self.iter_sp_rewards(sp_id); + let mut rewards = binding.iter(); + + let promo = rewards.next().cloned().unwrap().promotion_reward(); + let sp = rewards.next().cloned().unwrap().sp_reward(); + + (promo, sp) + } + } + + trait RewardExt { + fn promotion_reward(self) -> proto::PromotionReward; + fn sp_reward(self) -> proto::ServiceProviderReward; + } + + impl RewardExt for proto::MobileRewardShare { + fn promotion_reward(self) -> proto::PromotionReward { + match self.reward { + Some(proto::Reward::PromotionReward(promo)) => promo.clone(), + other => panic!("expected promotion reward, got {other:?}"), + } + } + + fn sp_reward(self) -> proto::ServiceProviderReward { + match self.reward { + Some(proto::Reward::ServiceProviderReward(promo)) => promo.clone(), + other => panic!("expected sp reward, got {other:?}"), + } + } + } + + #[test] + fn unallocated_reward_scaling_1() { + let sp_infos = ServiceProviderRewardInfos::new( + ServiceProviderDCSessions::from([(0, dec!(12)), (1, dec!(6))]), + ServiceProviderFunds::from([(0, 5000), (1, 5000)]), + ServiceProviderPromotions::from([ + PromotionRewardShare { + service_provider_id: 0, + rewardable_entity: Entity::SubscriberId(vec![0]), + shares: 1, + }, + PromotionRewardShare { + service_provider_id: 1, + rewardable_entity: Entity::SubscriberId(vec![1]), + shares: 1, + }, + ]), + dec!(100), + dec!(0.00001), + epoch(), + ); + + let (promo_1, sp_1) = sp_infos.single_sp_rewards(0); + assert_eq!(promo_1.service_provider_amount, 6); + assert_eq!(promo_1.matched_amount, 6); + assert_eq!(sp_1.amount, 6); + + let (promo_2, sp_2) = sp_infos.single_sp_rewards(1); + assert_eq!(promo_2.service_provider_amount, 3); + assert_eq!(promo_2.matched_amount, 3); + assert_eq!(sp_2.amount, 3); + } + + #[test] + fn unallocated_reward_scaling_2() { + let sp_infos = ServiceProviderRewardInfos::new( + ServiceProviderDCSessions::from([(0, dec!(12)), (1, dec!(6))]), + ServiceProviderFunds::from([(0, 5000), (1, 10000)]), + ServiceProviderPromotions::from([ + PromotionRewardShare { + service_provider_id: 0, + rewardable_entity: Entity::SubscriberId(vec![0]), + shares: 1, + }, + PromotionRewardShare { + service_provider_id: 1, + rewardable_entity: Entity::SubscriberId(vec![1]), + shares: 1, + }, + ]), + dec!(100), + dec!(0.00001), + epoch(), + ); + + let (promo_1, sp_1) = sp_infos.single_sp_rewards(0); + assert_eq!(promo_1.service_provider_amount, 6); + assert_eq!(promo_1.matched_amount, 6); + assert_eq!(sp_1.amount, 6); + + let (promo_2, sp_2) = sp_infos.single_sp_rewards(1); + assert_eq!(promo_2.service_provider_amount, 6); + assert_eq!(promo_2.matched_amount, 6); + assert_eq!(sp_2.amount, 0); + } + + #[test] + fn unallocated_reward_scaling_3() { + let sp_infos = ServiceProviderRewardInfos::new( + ServiceProviderDCSessions::from([(0, dec!(10)), (1, dec!(1000))]), + ServiceProviderFunds::from([(0, 10000), (1, 200)]), + ServiceProviderPromotions::from([ + PromotionRewardShare { + service_provider_id: 0, + rewardable_entity: Entity::SubscriberId(vec![0]), + shares: 1, + }, + PromotionRewardShare { + service_provider_id: 1, + rewardable_entity: Entity::SubscriberId(vec![1]), + shares: 1, + }, + ]), + dec!(2000), + dec!(0.00001), + epoch(), + ); + + let (promo_1, sp_1) = sp_infos.single_sp_rewards(0); + assert_eq!(promo_1.service_provider_amount, 10); + assert_eq!(promo_1.matched_amount, 10); + assert_eq!(sp_1.amount, 0); + + let (promo_2, sp_2) = sp_infos.single_sp_rewards(1); + assert_eq!(promo_2.service_provider_amount, 20); + assert_eq!(promo_2.matched_amount, 20); + assert_eq!(sp_2.amount, 980); + } + #[test] fn no_rewards_if_none_allocated() { let sp_infos = ServiceProviderRewardInfos::new( From d088627f77247ef81c9001403a81490a223dd83c Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Fri, 27 Sep 2024 13:37:42 -0700 Subject: [PATCH 22/31] if for some reason a key is duplicated, absorb both --- mobile_verifier/src/service_provider/dc_sessions.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mobile_verifier/src/service_provider/dc_sessions.rs b/mobile_verifier/src/service_provider/dc_sessions.rs index 296fabef6..68917ef09 100644 --- a/mobile_verifier/src/service_provider/dc_sessions.rs +++ b/mobile_verifier/src/service_provider/dc_sessions.rs @@ -94,6 +94,11 @@ where I: Into, { fn from(iter: F) -> Self { - Self(iter.into_iter().map(|(k, v)| (k.into(), v)).collect()) + // sum duplicate keys + let mut map = HashMap::new(); + for (k, v) in iter { + *map.entry(k.into()).or_insert(Decimal::ZERO) += v; + } + Self(map) } } From 7100362701027b049221af5bc6cc9af56b789642 Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Fri, 27 Sep 2024 13:41:59 -0700 Subject: [PATCH 23/31] add proptests for service provider rewards --- Cargo.lock | 110 +++++++++++++----- mobile_verifier/Cargo.toml | 1 + .../src/service_provider/reward.rs | 106 ++++++++++++++++- 3 files changed, 189 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 92819b246..971b57ae3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1649,6 +1649,21 @@ dependencies = [ "serde", ] +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bit_field" version = "0.10.1" @@ -3215,8 +3230,8 @@ dependencies = [ "serde_json", "sha2 0.10.8", "sqlx", - "strum 0.24.1", - "strum_macros 0.24.3", + "strum", + "strum_macros", "task-manager", "tempfile", "thiserror", @@ -3838,8 +3853,8 @@ dependencies = [ "prost-build", "serde", "serde_json", - "strum 0.26.3", - "strum_macros 0.26.4", + "strum", + "strum_macros", "tonic", "tonic-build", ] @@ -5184,6 +5199,7 @@ dependencies = [ "once_cell", "poc-metrics", "price", + "proptest", "prost", "rand 0.8.5", "regex", @@ -6067,6 +6083,26 @@ dependencies = [ "triggered", ] +[[package]] +name = "proptest" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags 2.5.0", + "lazy_static", + "num-traits", + "rand 0.8.5", + "rand_chacha 0.3.0", + "rand_xorshift", + "regex-syntax 0.8.3", + "rusty-fork", + "tempfile", + "unarray", +] + [[package]] name = "prost" version = "0.12.4" @@ -6209,6 +6245,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quinn" version = "0.10.2" @@ -6343,6 +6385,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "rand_xoshiro" version = "0.6.0" @@ -6917,6 +6968,18 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +[[package]] +name = "rusty-fork" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "ryu" version = "1.0.11" @@ -8664,35 +8727,13 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" -[[package]] -name = "strum" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" -dependencies = [ - "strum_macros 0.24.3", -] - [[package]] name = "strum" version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" dependencies = [ - "strum_macros 0.26.4", -] - -[[package]] -name = "strum_macros" -version = "0.24.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" -dependencies = [ - "heck 0.4.0", - "proc-macro2", - "quote", - "rustversion", - "syn 1.0.109", + "strum_macros", ] [[package]] @@ -9301,6 +9342,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c52b4cb7830f995903b2fcff3f523d21efc1c11f6c1596dd544b7925a64ff56" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicode-bidi" version = "0.3.15" @@ -9493,6 +9540,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + [[package]] name = "waker-fn" version = "1.1.0" diff --git a/mobile_verifier/Cargo.toml b/mobile_verifier/Cargo.toml index 29d0989ca..84e6c61b3 100644 --- a/mobile_verifier/Cargo.toml +++ b/mobile_verifier/Cargo.toml @@ -63,3 +63,4 @@ coverage-map = { path = "../coverage_map" } [dev-dependencies] backon = "0" +proptest = "1.5.0" diff --git a/mobile_verifier/src/service_provider/reward.rs b/mobile_verifier/src/service_provider/reward.rs index c22a03e65..3c9e7e7cc 100644 --- a/mobile_verifier/src/service_provider/reward.rs +++ b/mobile_verifier/src/service_provider/reward.rs @@ -284,7 +284,7 @@ mod tests { use file_store::promotion_reward::Entity; use helium_proto::services::poc_mobile::{MobileRewardShare, PromotionReward}; - use crate::service_provider::promotions::rewards::PromotionRewardShare; + use crate::service_provider::{self, promotions::rewards::PromotionRewardShare}; use super::*; @@ -617,4 +617,108 @@ mod tests { .collect() } } + + use proptest::prelude::*; + + prop_compose! { + fn arb_share()(sp_id in 0..10_i32, ent_id in 0..200u8, shares in 1..=100u64) -> PromotionRewardShare { + PromotionRewardShare { + service_provider_id: sp_id, + rewardable_entity: Entity::SubscriberId(vec![ent_id]), + shares + } + } + } + + prop_compose! { + fn arb_dc_session()( + sp_id in 0..10_i32, + // below 1 trillion + dc_session in (0..=1_000_000_000_000_u64).prop_map(Decimal::from) + ) -> (i32, Decimal) { + (sp_id, dc_session) + } + } + + prop_compose! { + fn arb_fund()(sp_id in 0..10_i32, bps in arb_bps()) -> (i32, u16) { + (sp_id, bps) + } + } + + prop_compose! { + fn arb_bps()(bps in 0..=10_000u16) -> u16 { bps } + } + + proptest! { + // #![proptest_config(ProptestConfig::with_cases(100_000))] + + #[test] + fn single_provider_does_not_overallocate( + dc_session in any::().prop_map(Decimal::from), + fund_bps in arb_bps(), + shares in prop::collection::vec(arb_share(), 0..10), + total_allocation in any::().prop_map(Decimal::from) + ) { + + let sp_infos = ServiceProviderRewardInfos::new( + ServiceProviderDCSessions::from([(0, dc_session)]), + ServiceProviderFunds::from([(0, fund_bps)]), + ServiceProviderPromotions::from(shares), + total_allocation, + dec!(0.00001), + epoch() + ); + + let total_perc= sp_infos.total_percent(); + assert!(total_perc <= dec!(1)); + + let mut allocated = dec!(0); + for (amount, _) in sp_infos.iter_rewards() { + allocated += Decimal::from(amount); + } + assert!(allocated <= total_allocation); + } + + #[test] + fn multiple_provider_does_not_overallocate( + dc_sessions in prop::collection::vec(arb_dc_session(), 0..10), + funds in prop::collection::vec(arb_fund(), 0..10), + promotions in prop::collection::vec(arb_share(), 0..100), + ) { + let epoch = epoch(); + let total_allocation = service_provider::get_scheduled_tokens(&epoch); + + let sp_infos = ServiceProviderRewardInfos::new( + ServiceProviderDCSessions::from(dc_sessions), + ServiceProviderFunds::from(funds), + ServiceProviderPromotions::from(promotions), + total_allocation, + dec!(0.00001), + epoch + ); + + let total_perc= sp_infos.total_percent(); + prop_assert!(total_perc <= dec!(1)); + + let mut allocated = dec!(0); + for (amount, _) in sp_infos.iter_rewards() { + allocated += Decimal::from(amount); + } + prop_assert!(allocated <= total_allocation); + } + + } + + impl RewardInfo { + fn total_percent(&self) -> Decimal { + self.realized_dc_perc + self.realized_promo_perc + self.matched_promo_perc + } + } + + impl ServiceProviderRewardInfos { + fn total_percent(&self) -> Decimal { + self.coll.iter().map(|x| x.total_percent()).sum() + } + } } From 02c031a19159080c76ea6d7a1b86cd1cc06f131d Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Fri, 27 Sep 2024 13:42:16 -0700 Subject: [PATCH 24/31] Found a rounding error truncating to 5 decimal places for a % can result in the summed allocation exceeding 100%. Thanks proptest --- mobile_verifier/src/service_provider/reward.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile_verifier/src/service_provider/reward.rs b/mobile_verifier/src/service_provider/reward.rs index 3c9e7e7cc..4eb08aff9 100644 --- a/mobile_verifier/src/service_provider/reward.rs +++ b/mobile_verifier/src/service_provider/reward.rs @@ -251,7 +251,7 @@ fn distribute_unalloc_over_limit(coll: &mut [RewardInfo], unallocated_perc: Deci continue; } let shares = sp.realized_promo_perc * dec!(100); - sp.matched_promo_perc = ((shares / total) * unallocated_perc).round_dp(5); + sp.matched_promo_perc = (shares / total) * unallocated_perc; } } From 3ad8630be7ce71450fef4722bea0aa834ec60532 Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Mon, 30 Sep 2024 12:24:27 -0700 Subject: [PATCH 25/31] Discussion has been had It has been noted that in tests a single bone is okay, as long as we're not going over the allocated bones --- mobile_verifier/tests/integrations/rewarder_sp_rewards.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/mobile_verifier/tests/integrations/rewarder_sp_rewards.rs b/mobile_verifier/tests/integrations/rewarder_sp_rewards.rs index 2450bc933..71cf82f0a 100644 --- a/mobile_verifier/tests/integrations/rewarder_sp_rewards.rs +++ b/mobile_verifier/tests/integrations/rewarder_sp_rewards.rs @@ -95,14 +95,12 @@ async fn test_service_provider_rewards(pool: PgPool) -> anyhow::Result<()> { .to_string() ); assert_eq!(5_999, sp_reward.amount); - // assert_eq!(6_000, sp_reward.amount); assert_eq!( UnallocatedRewardType::ServiceProvider as i32, unallocated_reward.reward_type ); assert_eq!(8_219_178_076_192, unallocated_reward.amount); - // assert_eq!(8_219_178_076_191, unallocated_reward.amount); // confirm the total rewards allocated matches expectations let expected_sum = From bd190880da032d4911b54cd8bd1cf5b28de79d60 Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Mon, 30 Sep 2024 12:25:44 -0700 Subject: [PATCH 26/31] commit commit let's not have the worst of both worlds where we comment out unused code. Remove the printlns, they're not hard to add back in if you need them. --- .../src/service_provider/reward.rs | 3 +- .../tests/integrations/common/mod.rs | 55 +++++++------------ 2 files changed, 21 insertions(+), 37 deletions(-) diff --git a/mobile_verifier/src/service_provider/reward.rs b/mobile_verifier/src/service_provider/reward.rs index 4eb08aff9..da460d39b 100644 --- a/mobile_verifier/src/service_provider/reward.rs +++ b/mobile_verifier/src/service_provider/reward.rs @@ -595,8 +595,7 @@ mod tests { fn epoch() -> Range> { let now = Utc::now(); - let epoch = now - Duration::hours(24)..now; - epoch + now - Duration::hours(24)..now } trait PromoRewardFiltersExt { diff --git a/mobile_verifier/tests/integrations/common/mod.rs b/mobile_verifier/tests/integrations/common/mod.rs index f2e5a75e3..d1e73dd47 100644 --- a/mobile_verifier/tests/integrations/common/mod.rs +++ b/mobile_verifier/tests/integrations/common/mod.rs @@ -135,65 +135,50 @@ impl MockFileSinkReceiver { pub async fn receive_gateway_reward(&mut self) -> GatewayReward { match self.receive("receive_gateway_reward").await { - Some(mobile_reward) => { - // println!("mobile_reward: {:?}", mobile_reward); - match mobile_reward.reward { - Some(MobileReward::GatewayReward(r)) => r, - _ => panic!("failed to get gateway reward"), - } - } + Some(mobile_reward) => match mobile_reward.reward { + Some(MobileReward::GatewayReward(r)) => r, + _ => panic!("failed to get gateway reward"), + }, None => panic!("failed to receive gateway reward"), } } pub async fn receive_service_provider_reward(&mut self) -> ServiceProviderReward { match self.receive("receive_service_provider_reward").await { - Some(mobile_reward) => { - // println!("mobile_reward: {:?}", mobile_reward); - match mobile_reward.reward { - Some(MobileReward::ServiceProviderReward(r)) => r, - _ => panic!("failed to get service provider reward"), - } - } + Some(mobile_reward) => match mobile_reward.reward { + Some(MobileReward::ServiceProviderReward(r)) => r, + _ => panic!("failed to get service provider reward"), + }, None => panic!("failed to receive service provider reward"), } } pub async fn receive_subscriber_reward(&mut self) -> SubscriberReward { match self.receive("receive_subscriber_reward").await { - Some(mobile_reward) => { - // println!("mobile_reward: {:?}", mobile_reward); - match mobile_reward.reward { - Some(MobileReward::SubscriberReward(r)) => r, - _ => panic!("failed to get subscriber reward"), - } - } + Some(mobile_reward) => match mobile_reward.reward { + Some(MobileReward::SubscriberReward(r)) => r, + _ => panic!("failed to get subscriber reward"), + }, None => panic!("failed to receive subscriber reward"), } } pub async fn receive_promotion_reward(&mut self) -> PromotionReward { match self.receive("receive_promotion_reward").await { - Some(mobile_reward) => { - // println!("mobile_reward: {:?}", mobile_reward); - match mobile_reward.reward { - Some(MobileReward::PromotionReward(r)) => r, - _ => panic!("failed to get promotion reward"), - } - } + Some(mobile_reward) => match mobile_reward.reward { + Some(MobileReward::PromotionReward(r)) => r, + _ => panic!("failed to get promotion reward"), + }, None => panic!("failed to receive promotion reward"), } } pub async fn receive_unallocated_reward(&mut self) -> UnallocatedReward { match self.receive("receive_unallocated_reward").await { - Some(mobile_reward) => { - // println!("mobile_reward: {:?}", mobile_reward); - match mobile_reward.reward { - Some(MobileReward::UnallocatedReward(r)) => r, - _ => panic!("failed to get unallocated reward"), - } - } + Some(mobile_reward) => match mobile_reward.reward { + Some(MobileReward::UnallocatedReward(r)) => r, + _ => panic!("failed to get unallocated reward"), + }, None => panic!("failed to receive unallocated reward"), } } From 1f482089f4d08a76698cc4ee185d32cbf212c405 Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Mon, 30 Sep 2024 14:45:37 -0700 Subject: [PATCH 27/31] fill out service provider reward allocations in reward_manifest --- mobile_verifier/src/rewarder.rs | 2 +- mobile_verifier/src/service_provider/mod.rs | 18 +++++++++++++++--- .../tests/integrations/rewarder_sp_rewards.rs | 13 +++++++++++-- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index 59ff9ede7..d20f864b4 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -313,7 +313,7 @@ where boosted_poc_bones_per_reward_share: Some(helium_proto::Decimal { value: poc_dc_shares.boost.to_string(), }), - sp_allocations: service_provider::reward_data_sp_allocations(), + sp_allocations: service_provider::reward_data_sp_allocations(&self.pool).await?, }; self.reward_manifests .write( diff --git a/mobile_verifier/src/service_provider/mod.rs b/mobile_verifier/src/service_provider/mod.rs index d45aa4138..9a6f90885 100644 --- a/mobile_verifier/src/service_provider/mod.rs +++ b/mobile_verifier/src/service_provider/mod.rs @@ -4,6 +4,7 @@ use chrono::{DateTime, Utc}; use helium_proto::ServiceProviderAllocation; pub use promotions::daemon::PromotionDaemon; pub use reward::ServiceProviderRewardInfos; +use sqlx::PgPool; pub mod dc_sessions; pub mod promotions; @@ -25,7 +26,18 @@ pub fn get_scheduled_tokens(reward_period: &Range>) -> rust_decima crate::reward_shares::get_scheduled_tokens_for_service_providers(duration) } -pub fn reward_data_sp_allocations() -> Vec { - // TODO: How do you go from a service provider id to the payer key? - vec![] +pub async fn reward_data_sp_allocations( + pool: &PgPool, +) -> anyhow::Result> { + let funds = db::fetch_promotion_funds(pool).await?; + let mut sp_allocations = vec![]; + + for (sp_id, bps) in funds.0.into_iter() { + sp_allocations.push(ServiceProviderAllocation { + service_provider: sp_id, + incentive_escrow_fund_bps: bps as u32, + }); + } + + Ok(sp_allocations) } diff --git a/mobile_verifier/tests/integrations/rewarder_sp_rewards.rs b/mobile_verifier/tests/integrations/rewarder_sp_rewards.rs index 71cf82f0a..336bd57d3 100644 --- a/mobile_verifier/tests/integrations/rewarder_sp_rewards.rs +++ b/mobile_verifier/tests/integrations/rewarder_sp_rewards.rs @@ -236,10 +236,10 @@ async fn test_service_provider_promotion_rewards(pool: PgPool) -> anyhow::Result // Ensure the cleanup job can run let mut txn = pool.begin().await?; - service_provider::promotions::rewards::clear_promotion_rewards(&mut txn, &Utc::now()).await?; + service_provider::db::clear_promotion_rewards(&mut txn, &Utc::now()).await?; txn.commit().await?; - let promos = service_provider::promotions::rewards::fetch_promotion_rewards( + let promos = service_provider::db::fetch_promotion_rewards( &pool, &carrier_client, &(epoch.start..Utc::now()), @@ -247,6 +247,15 @@ async fn test_service_provider_promotion_rewards(pool: PgPool) -> anyhow::Result .await?; assert!(promos.is_empty()); + let sp_allocations = service_provider::reward_data_sp_allocations(&pool).await?; + assert_eq!( + vec![helium_proto::ServiceProviderAllocation { + service_provider: 0, + incentive_escrow_fund_bps: 1500 + }], + sp_allocations + ); + Ok(()) } From 1abf20305a152ddd18396dc68fcaa520a03e4f90 Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Tue, 1 Oct 2024 10:01:21 -0700 Subject: [PATCH 28/31] Service Providers can use multiple keys for dc_transfer For accounting reasons, Service Providers can have multiple payer keys listed. We need to ensuure all their dc transfer are accumulated into their service provider entry when we are generating rewards. --- .../src/service_provider/dc_sessions.rs | 76 +++++++++++++++++-- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/mobile_verifier/src/service_provider/dc_sessions.rs b/mobile_verifier/src/service_provider/dc_sessions.rs index 68917ef09..1888c3bfa 100644 --- a/mobile_verifier/src/service_provider/dc_sessions.rs +++ b/mobile_verifier/src/service_provider/dc_sessions.rs @@ -39,7 +39,7 @@ pub struct ServiceProviderDCSessions(pub(crate) HashMap Decimal { @@ -94,11 +94,77 @@ where I: Into, { fn from(iter: F) -> Self { - // sum duplicate keys - let mut map = HashMap::new(); + let mut me = Self::default(); for (k, v) in iter { - *map.entry(k.into()).or_insert(Decimal::ZERO) += v; + me.insert(k.into(), v); } - Self(map) + me + } +} + +#[cfg(test)] +pub mod tests { + + use chrono::Duration; + use helium_proto::ServiceProvider; + + use crate::data_session::HotspotDataSession; + + use super::*; + + impl ServiceProviderDCSessions { + fn len(&self) -> usize { + self.0.len() + } + } + + #[sqlx::test] + fn multiple_payer_keys_accumulate_to_service_provider(pool: PgPool) -> anyhow::Result<()> { + // Client always resolves to single service provider no matter payer key + struct MockClient; + + #[async_trait::async_trait] + impl CarrierServiceVerifier for MockClient { + type Error = ClientError; + + async fn payer_key_to_service_provider<'a>( + &self, + _pubkey: &str, + ) -> Result { + Ok(ServiceProvider::HeliumMobile) + } + } + + // Save multiple data sessions with different payers + let one = HotspotDataSession { + pub_key: vec![0].into(), + payer: vec![0].into(), + upload_bytes: 1_000, + download_bytes: 1_000, + num_dcs: 2_000, + received_timestamp: Utc::now(), + }; + let two = HotspotDataSession { + pub_key: vec![1].into(), + payer: vec![1].into(), + upload_bytes: 1_000, + download_bytes: 1_000, + num_dcs: 2_000, + received_timestamp: Utc::now(), + }; + let mut txn = pool.begin().await?; + one.save(&mut txn).await?; + two.save(&mut txn).await?; + txn.commit().await?; + + let now = Utc::now(); + let epoch = now - Duration::hours(24)..now; + + // dc sessions should represent single payer, and all dc is combined + let map = fetch_dc_sessions(&pool, &MockClient, &epoch).await?; + assert_eq!(map.len(), 1); + assert_eq!(map.all_transfer(), Decimal::from(4_000)); + + Ok(()) } } From 6d0ba723ed0db654e38462316ad607107b10a638 Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Wed, 2 Oct 2024 10:31:55 -0700 Subject: [PATCH 29/31] fixup cargo deps after rebase --- Cargo.lock | 198 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 153 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 971b57ae3..23a59fcfb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1617,17 +1617,17 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "beacon" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=map/subscriber-referral#effd56a31ab83a1079c7819cd1f783f4141c9c9b" +source = "git+https://github.com/helium/proto?branch=master#197ff9c6cde7dc0d8334d6b4e27c58779e6a7ce0" dependencies = [ "base64 0.21.7", "byteorder", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", + "helium-proto", "prost", "rand 0.8.5", "rand_chacha 0.3.0", "rust_decimal", "serde", - "sha2 0.9.9", + "sha2 0.10.8", "thiserror", ] @@ -1791,7 +1791,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", + "helium-proto", "http 0.2.11", "http-serde", "humantime-serde", @@ -2132,7 +2132,16 @@ dependencies = [ [[package]] name = "circuit-breaker" version = "0.1.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" +source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +dependencies = [ + "anchor-gen", + "anchor-lang 0.30.1", +] + +[[package]] +name = "circuit-breaker" +version = "0.1.0" +source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -2633,7 +2642,7 @@ dependencies = [ "axum 0.7.4", "bs58 0.4.0", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", + "helium-proto", "http 0.2.11", "notify", "serde", @@ -2774,7 +2783,16 @@ dependencies = [ [[package]] name = "data-credits" version = "0.2.2" -source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" +source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +dependencies = [ + "anchor-gen", + "anchor-lang 0.30.1", +] + +[[package]] +name = "data-credits" +version = "0.2.2" +source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -3152,7 +3170,16 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "fanout" version = "0.1.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" +source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +dependencies = [ + "anchor-gen", + "anchor-lang 0.30.1", +] + +[[package]] +name = "fanout" +version = "0.1.0" +source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -3215,7 +3242,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", + "helium-proto", "hex-literal", "http 0.2.11", "lazy_static", @@ -3728,7 +3755,7 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "helium-anchor-gen" version = "0.1.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" +source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -3779,7 +3806,7 @@ dependencies = [ "bs58 0.5.0", "byteorder", "ed25519-compact", - "getrandom 0.1.16", + "getrandom 0.2.10", "k256", "lazy_static", "multihash", @@ -3796,8 +3823,17 @@ dependencies = [ [[package]] name = "helium-entity-manager" -version = "0.3.1" -source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" +version = "0.2.11" +source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +dependencies = [ + "anchor-gen", + "anchor-lang 0.30.1", +] + +[[package]] +name = "helium-entity-manager" +version = "0.2.11" +source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -3806,7 +3842,7 @@ dependencies = [ [[package]] name = "helium-lib" version = "0.0.0" -source = "git+https://github.com/helium/helium-wallet-rs.git?branch=master#b54819ac4c4bd73be37d25d7d6d48842bbc95ea9" +source = "git+https://github.com/helium/helium-wallet-rs.git?branch=master#4acf688beac3c507c33843a745516839e1f814b2" dependencies = [ "anchor-client", "anchor-spl", @@ -3819,7 +3855,7 @@ dependencies = [ "h3o", "helium-anchor-gen 0.1.0 (git+https://github.com/helium/helium-anchor-gen.git?branch=main)", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=master)", + "helium-proto", "hex", "hex-literal", "itertools", @@ -3846,7 +3882,7 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=map/subscriber-referral#effd56a31ab83a1079c7819cd1f783f4141c9c9b" +source = "git+https://github.com/helium/proto?branch=master#197ff9c6cde7dc0d8334d6b4e27c58779e6a7ce0" dependencies = [ "bytes", "prost", @@ -3860,9 +3896,9 @@ dependencies = [ ] [[package]] -name = "helium-proto" -version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=master#376765fe006051d6dcccf709def58e7ed291b845" +name = "helium-sub-daos" +version = "0.1.8" +source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -3871,7 +3907,7 @@ dependencies = [ [[package]] name = "helium-sub-daos" version = "0.1.8" -source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" +source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -3909,7 +3945,7 @@ dependencies = [ "async-trait", "chrono", "derive_builder", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", + "helium-proto", "hextree", "rust_decimal", "rust_decimal_macros", @@ -3925,7 +3961,16 @@ checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" [[package]] name = "hexboosting" version = "0.1.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" +source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +dependencies = [ + "anchor-gen", + "anchor-lang 0.30.1", +] + +[[package]] +name = "hexboosting" +version = "0.1.0" +source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -4325,7 +4370,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", + "helium-proto", "http 0.2.11", "humantime-serde", "metrics", @@ -4394,7 +4439,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", + "helium-proto", "hextree", "http 0.2.11", "http-serde", @@ -4436,7 +4481,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", + "helium-proto", "http 0.2.11", "http-serde", "humantime-serde", @@ -4478,7 +4523,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", + "helium-proto", "http-serde", "humantime-serde", "iot-config", @@ -4736,7 +4781,25 @@ dependencies = [ [[package]] name = "lazy-distributor" version = "0.2.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" +source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +dependencies = [ + "anchor-gen", + "anchor-lang 0.30.1", +] + +[[package]] +name = "lazy-distributor" +version = "0.2.0" +source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +dependencies = [ + "anchor-gen", + "anchor-lang 0.30.1", +] + +[[package]] +name = "lazy-transactions" +version = "0.2.0" +source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -4745,7 +4808,7 @@ dependencies = [ [[package]] name = "lazy-transactions" version = "0.2.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" +source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -5066,7 +5129,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", + "helium-proto", "hextree", "http 0.2.11", "http-serde", @@ -5106,7 +5169,7 @@ dependencies = [ "futures", "h3o", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", + "helium-proto", "mobile-config", "prost", "rand 0.8.5", @@ -5121,7 +5184,16 @@ dependencies = [ [[package]] name = "mobile-entity-manager" version = "0.1.3" -source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" +source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +dependencies = [ + "anchor-gen", + "anchor-lang 0.30.1", +] + +[[package]] +name = "mobile-entity-manager" +version = "0.1.3" +source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -5142,7 +5214,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", + "helium-proto", "http 0.2.11", "http-serde", "humantime-serde", @@ -5186,7 +5258,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", + "helium-proto", "hex-assignments", "hextree", "http-serde", @@ -5554,7 +5626,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" dependencies = [ - "proc-macro-crate 1.1.3", + "proc-macro-crate 3.1.0", "proc-macro2", "quote", "syn 2.0.58", @@ -5870,7 +5942,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", + "helium-proto", "http 0.2.11", "hyper 0.14.28", "jsonrpsee", @@ -5953,7 +6025,7 @@ dependencies = [ "futures-util", "helium-anchor-gen 0.1.0 (git+https://github.com/helium/helium-anchor-gen.git)", "helium-lib", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", + "helium-proto", "humantime-serde", "metrics", "metrics-exporter-prometheus", @@ -5977,7 +6049,16 @@ dependencies = [ [[package]] name = "price-oracle" version = "0.2.1" -source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" +source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +dependencies = [ + "anchor-gen", + "anchor-lang 0.30.1", +] + +[[package]] +name = "price-oracle" +version = "0.2.1" +source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -6069,7 +6150,7 @@ dependencies = [ "custom-tracing", "file-store", "futures", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", + "helium-proto", "humantime-serde", "metrics", "metrics-exporter-prometheus", @@ -6120,7 +6201,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80b776a1b2dc779f5ee0641f8ade0125bc1298dd41a9a0c16d8bd57b42d222b1" dependencies = [ "bytes", - "heck 0.4.0", + "heck 0.5.0", "itertools", "log", "multimap", @@ -6621,7 +6702,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=map/subscriber-referral)", + "helium-proto", "humantime-serde", "lazy_static", "metrics", @@ -6655,7 +6736,16 @@ dependencies = [ [[package]] name = "rewards-oracle" version = "0.2.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" +source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +dependencies = [ + "anchor-gen", + "anchor-lang 0.30.1", +] + +[[package]] +name = "rewards-oracle" +version = "0.2.0" +source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -9280,7 +9370,16 @@ dependencies = [ [[package]] name = "treasury-management" version = "0.2.0" -source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" +source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +dependencies = [ + "anchor-gen", + "anchor-lang 0.30.1", +] + +[[package]] +name = "treasury-management" +version = "0.2.0" +source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -9326,7 +9425,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", - "rand 0.7.3", + "rand 0.8.5", "static_assertions", ] @@ -9528,7 +9627,16 @@ dependencies = [ [[package]] name = "voter-stake-registry" version = "0.3.3" -source = "git+https://github.com/helium/helium-anchor-gen.git#54392f522436bc8a8c8b8c0a0b4ec407a28f66ed" +source = "git+https://github.com/helium/helium-anchor-gen.git?branch=main#761b839a71cc6d2aecf4be994af9d8206aeae0e1" +dependencies = [ + "anchor-gen", + "anchor-lang 0.30.1", +] + +[[package]] +name = "voter-stake-registry" +version = "0.3.3" +source = "git+https://github.com/helium/helium-anchor-gen.git#761b839a71cc6d2aecf4be994af9d8206aeae0e1" dependencies = [ "anchor-gen", "anchor-lang 0.30.1", @@ -9996,7 +10104,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "sha2 0.9.9", + "sha2 0.10.8", "thiserror", "twox-hash", "xorf", From c02d5ca659dcc093dabfa045940b5414fcd6ef5d Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Thu, 3 Oct 2024 09:57:23 -0400 Subject: [PATCH 30/31] Change promotion_fund to promotion-fund --- Cargo.lock | 2 +- promotion_fund/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 23a59fcfb..983472175 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6139,7 +6139,7 @@ dependencies = [ ] [[package]] -name = "promotion_fund" +name = "promotion-fund" version = "0.1.0" dependencies = [ "anyhow", diff --git a/promotion_fund/Cargo.toml b/promotion_fund/Cargo.toml index eedd3135f..d6d3418dd 100644 --- a/promotion_fund/Cargo.toml +++ b/promotion_fund/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "promotion_fund" +name = "promotion-fund" version = "0.1.0" description = "Service Provider promotion fund tracking for the Helium Network" authors.workspace = true From 41ef60e8080afa47d0504d7349cc9a962fbdf7df Mon Sep 17 00:00:00 2001 From: Michael Jeffrey Date: Thu, 3 Oct 2024 10:02:59 -0700 Subject: [PATCH 31/31] Read Service Provider Promotion values from unique bucket - Add settings `promotion_ingest` --- mobile_verifier/pkg/settings-template.toml | 8 ++++++++ mobile_verifier/src/cli/server.rs | 2 ++ mobile_verifier/src/service_provider/promotions/daemon.rs | 8 +++++--- mobile_verifier/src/settings.rs | 2 ++ 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/mobile_verifier/pkg/settings-template.toml b/mobile_verifier/pkg/settings-template.toml index 57b462eca..98bd7caeb 100644 --- a/mobile_verifier/pkg/settings-template.toml +++ b/mobile_verifier/pkg/settings-template.toml @@ -47,6 +47,14 @@ block = 0 # bucket = "mainnet-mobile-ingest" +[promotion_ingest] + +# Input bucket details for Service Provider Promotion Funds + +# Name of bucket to access ingest data. Required +# +bucket = "price" + # Region for bucket. Defaults to below # # region = "us-west-2" diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index d325581c7..50d4fb61e 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -46,6 +46,7 @@ impl Cmd { let store_base_path = std::path::Path::new(&settings.cache); let report_ingest = FileStore::from_settings(&settings.ingest).await?; + let promotion_fund_ingest = FileStore::from_settings(&settings.promotion_ingest).await?; // mobile config clients let gateway_client = GatewayClient::from_settings(&settings.config_client)?; @@ -200,6 +201,7 @@ impl Cmd { settings, file_upload.clone(), report_ingest.clone(), + promotion_fund_ingest, gateway_client.clone(), auth_client.clone(), entity_client.clone(), diff --git a/mobile_verifier/src/service_provider/promotions/daemon.rs b/mobile_verifier/src/service_provider/promotions/daemon.rs index aea62dc0f..57bf11c4f 100644 --- a/mobile_verifier/src/service_provider/promotions/daemon.rs +++ b/mobile_verifier/src/service_provider/promotions/daemon.rs @@ -61,11 +61,13 @@ impl ManagedTask for PromotionDaemon { } impl PromotionDaemon { + #[allow(clippy::too_many_arguments)] pub async fn create_managed_task( pool: PgPool, settings: &Settings, file_upload: FileUpload, - file_store: file_store::FileStore, + report_file_store: file_store::FileStore, + promotion_file_store: file_store::FileStore, gateway_info_resolver: GatewayClient, authorization_verifier: AuthorizationClient, entity_verifier: EntityClient, @@ -83,7 +85,7 @@ impl PromotionDaemon { let (promotion_rewards, promotion_rewards_server) = file_source::Continuous::msg_source::() .state(pool.clone()) - .store(file_store.clone()) + .store(report_file_store.clone()) .lookback(LookbackBehavior::StartAfter(settings.start_after)) .prefix(FileType::PromotionRewardIngestReport.to_string()) .create() @@ -92,7 +94,7 @@ impl PromotionDaemon { let (promotion_funds, promotion_funds_server) = file_source::Continuous::prost_source::() .state(pool.clone()) - .store(file_store) + .store(promotion_file_store) .lookback(LookbackBehavior::StartAfter(settings.start_after)) .prefix(FileType::ServiceProviderPromotionFund.to_string()) .create() diff --git a/mobile_verifier/src/settings.rs b/mobile_verifier/src/settings.rs index 54a31d9dd..0ad90d158 100644 --- a/mobile_verifier/src/settings.rs +++ b/mobile_verifier/src/settings.rs @@ -25,6 +25,8 @@ pub struct Settings { pub database: db_store::Settings, pub ingest: file_store::Settings, pub data_transfer_ingest: file_store::Settings, + /// S3 ingest reading Service Provider Promotion Funds + pub promotion_ingest: file_store::Settings, pub output: file_store::Settings, /// S3 bucket from which new data sets are downloaded for oracle boosting /// assignments