More fee rate sources

src/GWallet.Backend.Tests/ServerReference.fs

@@ -50,6 +50,33 @@ type ServerReference() =
             TimeSpan = timeSpan
         },dummy_now) |> Some
 
+    [<Test>]
+    member __.``averageBetween3DiscardingOutlier: basic test``() =
+        let res = TrustMinimizedEstimation.AverageBetween3DiscardingOutlier 1m 2m 3m
+        Assert.That(res, Is.EqualTo 2m)
+        ()
+
+    [<Test>]
+    member __.``averageBetween3DiscardingOutlier: nuanced tests``() =
+        let res = TrustMinimizedEstimation.AverageBetween3DiscardingOutlier 0m 2m 3m
+        Assert.That(res, Is.EqualTo 2.5m)
+
+        let res = TrustMinimizedEstimation.AverageBetween3DiscardingOutlier 2m 0m 3m
+        Assert.That(res, Is.EqualTo 2.5m)
+        ()
+
+        let res = TrustMinimizedEstimation.AverageBetween3DiscardingOutlier 0m 3m 2m
+        Assert.That(res, Is.EqualTo 2.5m)
+        ()
+
+        let res = TrustMinimizedEstimation.AverageBetween3DiscardingOutlier 3m 0m 2m
+        Assert.That(res, Is.EqualTo 2.5m)
+        ()
+
+        let res = TrustMinimizedEstimation.AverageBetween3DiscardingOutlier 3m 2m 0m
+        Assert.That(res, Is.EqualTo 2.5m)
+        ()
+
     [<Test>]
     member __.``order of servers is kept if non-hostname details are same``() =
         let serverWithHighestPriority =

src/GWallet.Backend/Currency.fs

@@ -1,10 +1,28 @@
 namespace GWallet.Backend
 
+open System
 open System.Linq
 open System.ComponentModel
 
 open GWallet.Backend.FSharpUtil.UwpHacks
 
+module TrustMinimizedEstimation =
+    let AverageBetween3DiscardingOutlier (one: decimal) (two: decimal) (three: decimal): decimal =
+        let sorted = List.sort [one; two; three]
+        let first = sorted.Item 0
+        let last = sorted.Item 2
+        let higher = Math.Max(first, last)
+        let intermediate = sorted.Item 1
+        let lower = Math.Min(first, last)
+
+        if (higher - intermediate = intermediate - lower) then
+            (higher + intermediate + lower) / 3m
+        // choose the two that are closest
+        elif (higher - intermediate) < (intermediate - lower) then
+            (higher + intermediate) / 2m
+        else
+            (lower + intermediate) / 2m
+
 // this attribute below is for Json.NET (Newtonsoft.Json) to be able to deserialize this as a dict key
 [<TypeConverter(typeof<StringTypeConverter>)>]
 type Currency =

src/GWallet.Backend/FiatValueEstimation.fs

@@ -1,10 +1,8 @@
 namespace GWallet.Backend
 
 open System
-open System.Net
 
 open FSharp.Data
-open Fsdk
 open FSharpx.Collections
 
 open GWallet.Backend.FSharpUtil.UwpHacks
@@ -1004,8 +1002,7 @@ module FiatValueEstimation =
         | CoinGecko
         | CoinDesk
 
-    let private QueryOnlineInternal currency (provider: PriceProvider): Async<Option<string*string>> = async {
-        use webClient = new WebClient()
+    let private QueryOnlineInternal currency (provider: PriceProvider): Async<Option<string*string>> =
         let tickerName =
             match currency,provider with
             | Currency.BTC,_ -> "bitcoin"
@@ -1018,7 +1015,7 @@ module FiatValueEstimation =
             | Currency.DAI,PriceProvider.CoinCap -> "multi-collateral-dai"
             | Currency.DAI,_ -> "dai"
 
-        try
+        async {
             let baseUrl =
                 match provider with
                 | PriceProvider.BitPay ->
@@ -1032,16 +1029,14 @@ module FiatValueEstimation =
                         failwith "CoinDesk API only provides bitcoin price"
                     "https://api.coindesk.com/v1/bpi/currentprice.json"
             let uri = Uri baseUrl
-            let task = webClient.DownloadStringTaskAsync uri
-            let! res = Async.AwaitTask task
-            return Some (tickerName,res)
-        with
-        | ex ->
-            if (FSharpUtil.FindException<WebException> ex).IsSome then
-                return None
-            else
-                return raise <| FSharpUtil.ReRaise ex
-    }
+
+            let! maybeResult = Networking.QueryRestApi uri
+            let tupleResult =
+                match maybeResult with
+                | None -> None
+                | Some result -> Some (tickerName, result)
+            return tupleResult
+        }
 
     let private QueryBitPay currency =
         async {

src/GWallet.Backend/GWallet.Backend-legacy.fsproj

@@ -69,6 +69,7 @@
     <Compile Include="UtxoCoin\UtxoCoinServer.fs" />
     <Compile Include="UtxoCoin\UtxoCoinMinerFee.fs" />
     <Compile Include="UtxoCoin\TransactionTypes.fs" />
+    <Compile Include="UtxoCoin\FeeRateEstimation.fs" />
     <Compile Include="UtxoCoin\UtxoCoinAccount.fs" />
     <Compile Include="Ether\EtherExceptions.fs" />
     <Compile Include="Ether\EtherMinerFee.fs" />

src/GWallet.Backend/GWallet.Backend.fsproj

@@ -38,6 +38,7 @@
     <Compile Include="UtxoCoin\UtxoCoinServer.fs" />
     <Compile Include="UtxoCoin\UtxoCoinMinerFee.fs" />
     <Compile Include="UtxoCoin\TransactionTypes.fs" />
+    <Compile Include="UtxoCoin\FeeRateEstimation.fs" />
     <Compile Include="UtxoCoin\UtxoCoinAccount.fs" />
     <Compile Include="Ether\EtherExceptions.fs" />
     <Compile Include="Ether\EtherMinerFee.fs" />

src/GWallet.Backend/Networking.fs

@@ -126,6 +126,21 @@ type ServerMisconfiguredException =
 
 module Networking =
 
+    let QueryRestApi (uri: Uri) =
+        async {
+            use webClient = new WebClient()
+            try
+                let task = webClient.DownloadStringTaskAsync uri
+                let! result = Async.AwaitTask task
+                return Some result
+            with
+            | ex ->
+                if (FSharpUtil.FindException<WebException> ex).IsSome then
+                    return None
+                else
+                    return raise <| FSharpUtil.ReRaise ex
+        }
+
     let FindExceptionToRethrow (ex: Exception) (newExceptionMsg): Option<Exception> =
         match FSharpUtil.FindException<SocketException> ex with
         | None ->

src/GWallet.Backend/UtxoCoin/ElectrumClient.fs

@@ -122,12 +122,12 @@ module ElectrumClient =
             return raise <| ServerMisconfiguredException(SPrintF1 "Fee estimation returned an invalid non-positive value %M"
                                                                   estimateFeeResult.Result)
 
-        let amountPerKB = estimateFeeResult.Result
-        let satPerKB = (NBitcoin.Money (amountPerKB, NBitcoin.MoneyUnit.BTC)).ToUnit NBitcoin.MoneyUnit.Satoshi
+        let btcPerKB = estimateFeeResult.Result
+        let satPerKB = (NBitcoin.Money (btcPerKB, NBitcoin.MoneyUnit.BTC)).ToUnit NBitcoin.MoneyUnit.Satoshi
         let satPerB = satPerKB / (decimal 1000)
         Infrastructure.LogDebug <| SPrintF2
-            "Electrum server gave us a fee rate of %M per KB = %M sat per B" amountPerKB satPerB
-        return amountPerKB
+            "Electrum server gave us a fee rate of %M per KB = %M sat per B" btcPerKB satPerB
+        return btcPerKB
     }
 
     let BroadcastTransaction (transactionInHex: string) (stratumServer: Async<StratumClient>) = async {

src/GWallet.Backend/UtxoCoin/FeeRateEstimation.fs

@@ -0,0 +1,183 @@
+namespace GWallet.Backend.UtxoCoin
+
+open System
+open System.Linq
+
+open GWallet.Backend
+open GWallet.Backend.FSharpUtil.UwpHacks
+
+open FSharp.Data
+open NBitcoin
+
+module FeeRateEstimation =
+
+    type Priority =
+    | Highest
+    | Low
+
+    type MempoolSpaceProvider = JsonProvider<"""
+        {
+          "fastestFee": 41,
+          "halfHourFee": 38,
+          "hourFee": 35,
+          "economyFee": 12,
+          "minimumFee": 6
+        }
+    """>
+
+    let MempoolSpaceRestApiUri = Uri "https://mempool.space/api/v1/fees/recommended"
+
+    type BlockchainInfoProvider = JsonProvider<"""
+        {
+          "limits": {
+            "min": 4,
+            "max": 16
+          },
+          "regular": 9,
+          "priority": 11
+        }
+    """>
+
+    let BlockchainInfoRestApiUri = Uri "https://api.blockchain.info/mempool/fees"
+
+    let private ToBrandedType(feeRatePerKB: decimal) (moneyUnit: MoneyUnit): FeeRate =
+        try
+            Money(feeRatePerKB, moneyUnit) |> FeeRate
+        with
+        | ex ->
+            // we need more info in case this bug shows again: https://gitlab.com/nblockchain/geewallet/issues/43
+            raise <| Exception(SPrintF2 "Could not create fee rate from %s %A"
+                                        (feeRatePerKB.ToString()) moneyUnit, ex)
+
+    let private QueryFeeRateToMempoolSpace (priority: Priority): Async<Option<FeeRate>> =
+        async {
+            let! maybeJson = Networking.QueryRestApi MempoolSpaceRestApiUri
+            match maybeJson with
+            | None -> return None
+            | Some json ->
+                let recommendedFees = MempoolSpaceProvider.Parse json
+                let highPrioFeeSatsPerB =
+                    // FIXME: at the moment of writing this, .FastestFee is even higher than what electrum servers recommend (15 vs 12)
+                    // (and .MinimumFee and .EconomyFee (3,6) seem too low, given that mempool.space website (not API) was giving 10,11,12)
+                    match priority with
+                    | Highest -> recommendedFees.FastestFee
+                    | Low -> recommendedFees.EconomyFee
+                    |> decimal
+                Infrastructure.LogDebug (SPrintF1 "mempool.space API gave us a fee rate of %M sat per B" highPrioFeeSatsPerB)
+                let satPerKB = highPrioFeeSatsPerB * (decimal 1000)
+                return Some <| ToBrandedType satPerKB MoneyUnit.Satoshi
+        }
+
+    let private QueryFeeRateToBlockchainInfo (priority: Priority): Async<Option<FeeRate>> =
+        async {
+            let! maybeJson = Networking.QueryRestApi BlockchainInfoRestApiUri
+            match maybeJson with
+            | None -> return None
+            | Some json ->
+                let recommendedFees = BlockchainInfoProvider.Parse json
+                let highPrioFeeSatsPerB =
+                    // FIXME: at the moment of writing this, both priority & regular give same number wtaf -> 9
+                    // (and .Limits.Min was 4, which seemed too low given that mempool.space website (not API) was giving 10,11,12;
+                    //  and .Limits.Max was too high, higher than what electrum servers were suggesting: 12)
+                    match priority with
+                    | Highest -> recommendedFees.Priority
+                    | Low -> recommendedFees.Regular
+                    |> decimal
+                Infrastructure.LogDebug (SPrintF1 "blockchain.info API gave us a fee rate of %M sat per B" highPrioFeeSatsPerB)
+                let satPerKB = highPrioFeeSatsPerB * (decimal 1000)
+                return Some <| ToBrandedType satPerKB MoneyUnit.Satoshi
+        }
+
+    let private AverageFee (feesFromDifferentServers: List<decimal>): decimal =
+        let avg = feesFromDifferentServers.Sum() / decimal feesFromDifferentServers.Length
+        avg
+
+    let private QueryFeeRateToElectrumServers (currency: Currency) (priority: Priority): Async<FeeRate> =
+        async {
+            //querying for 1 will always return -1 surprisingly...
+            let numBlocksToWait =
+                match currency, priority with
+                | Currency.BTC, Low ->
+                    6
+                | Currency.LTC, _
+                | _, Highest ->
+                    //querying for 1 will always return -1 surprisingly...
+                    2
+                | otherCurrency, otherPrio ->
+                    failwith <| SPrintF2 "UTXO-based currency %A not implemented ElectrumServer feeRate %A query" otherCurrency otherPrio
+
+            let estimateFeeJob = ElectrumClient.EstimateFee numBlocksToWait
+            let! btcPerKiloByteForFastTrans =
+                Server.Query currency (QuerySettings.FeeEstimation AverageFee) estimateFeeJob None
+            return ToBrandedType (decimal btcPerKiloByteForFastTrans) MoneyUnit.BTC
+        }
+
+    let QueryFeeRateInternal currency (priority: Priority) =
+        let electrumJob =
+            async {
+                try
+                    let! result = QueryFeeRateToElectrumServers currency priority
+                    return Some result
+                with
+                | :? NoneAvailableException ->
+                    return None
+            }
+
+        async {
+            match currency with
+            | Currency.LTC ->
+                let! electrumResult = electrumJob
+                return electrumResult
+            | Currency.BTC ->
+                let! bothJobs = Async.Parallel [electrumJob; QueryFeeRateToMempoolSpace priority; QueryFeeRateToBlockchainInfo priority]
+                let electrumResult = bothJobs.ElementAt 0
+                let mempoolSpaceResult = bothJobs.ElementAt 1
+                let blockchainInfoResult = bothJobs.ElementAt 2
+                match electrumResult, mempoolSpaceResult, blockchainInfoResult with
+                | None, None, None -> return None
+                | Some feeRate, None, None ->
+                    Infrastructure.LogDebug "Only electrum servers available for feeRate estimation"
+                    return Some feeRate
+                | None, Some feeRate, None ->
+                    Infrastructure.LogDebug "Only mempool.space API available for feeRate estimation"
+                    return Some feeRate
+                | None, None, Some feeRate ->
+                    Infrastructure.LogDebug "Only blockchain.info API available for feeRate estimation"
+                    return Some feeRate
+                | None, Some restApiFeeRate1, Some restApiFeeRate2 ->
+                    Infrastructure.LogDebug "Only REST APIs available for feeRate estimation"
+                    let average = AverageFee [decimal restApiFeeRate1.FeePerK.Satoshi; decimal restApiFeeRate2.FeePerK.Satoshi]
+                    let averageFeeRate = ToBrandedType average MoneyUnit.Satoshi
+                    Infrastructure.LogDebug (SPrintF1 "Average fee rate of %M sat per B" averageFeeRate.SatoshiPerByte)
+                    return Some averageFeeRate
+                | Some electrumFeeRate, Some restApiFeeRate, None ->
+                    let average = AverageFee [decimal electrumFeeRate.FeePerK.Satoshi; decimal restApiFeeRate.FeePerK.Satoshi]
+                    let averageFeeRate = ToBrandedType average MoneyUnit.Satoshi
+                    Infrastructure.LogDebug (SPrintF1 "Average fee rate of %M sat per B" averageFeeRate.SatoshiPerByte)
+                    return Some averageFeeRate
+                | Some electrumFeeRate, None, Some restApiFeeRate ->
+                    let average = AverageFee [decimal electrumFeeRate.FeePerK.Satoshi; decimal restApiFeeRate.FeePerK.Satoshi]
+                    let averageFeeRate = ToBrandedType average MoneyUnit.Satoshi
+                    Infrastructure.LogDebug (SPrintF1 "Average fee rate of %M sat per B" averageFeeRate.SatoshiPerByte)
+                    return Some averageFeeRate
+                | Some electrumFeeRate, Some restApiFeeRate1, Some restApiFeeRate2 ->
+                    let average =
+                        TrustMinimizedEstimation.AverageBetween3DiscardingOutlier
+                            (decimal electrumFeeRate.FeePerK.Satoshi)
+                            (decimal restApiFeeRate1.FeePerK.Satoshi)
+                            (decimal restApiFeeRate2.FeePerK.Satoshi)
+                    let averageFeeRate = ToBrandedType average MoneyUnit.Satoshi
+                    Infrastructure.LogDebug (SPrintF1 "Average fee rate of %M sat per B" averageFeeRate.SatoshiPerByte)
+                    return Some averageFeeRate
+            | currency ->
+                return failwith <| SPrintF1 "UTXO currency not supported yet?: %A" currency
+        }
+
+    let internal EstimateFeeRate currency (priority: Priority): Async<FeeRate> =
+        async {
+            let! maybeFeeRate = QueryFeeRateInternal currency priority
+            match maybeFeeRate with
+            | None -> return failwith "Sending when offline not supported, try sign-off?"
+            | Some feeRate -> return feeRate
+        }
+