client/{core, btc}: MultiTrade

client/asset/btc/coin_selection.go

@@ -4,6 +4,7 @@
 package btc
 
 import (
+	"math"
 	"math/rand"
 	"sort"
 	"time"
@@ -58,6 +59,22 @@ func orderEnough(val, lots, feeRate, initTxSizeBase, initTxSize uint64, segwit,
 	}
 }
 
+// reserveEnough generates a function that can be used as the enough argument
+// to the fund method. The function returns true if sum is greater than equal
+// to amt.
+func reserveEnough(amt uint64) func(_, sum uint64) (bool, uint64) {
+	return func(_, sum uint64) (bool, uint64) {
+		return sum >= amt, 0
+	}
+}
+
+func sumUTXOSize(set []*compositeUTXO) (tot uint64) {
+	for _, utxo := range set {
+		tot += uint64(utxo.input.VBytes())
+	}
+	return tot
+}
+
 func sumUTXOs(set []*compositeUTXO) (tot uint64) {
 	for _, utxo := range set {
 		tot += utxo.amount
@@ -66,12 +83,12 @@ func sumUTXOs(set []*compositeUTXO) (tot uint64) {
 }
 
 // subsetWithLeastSumGreaterThan attempts to select the subset of UTXOs with
-// the smallest total value greater than amt. It does this by making
+// the smallest total value that is enough. It does this by making
 // 1000 random selections and returning the best one. Each selection
 // involves two passes over the UTXOs. The first pass randomly selects
 // each UTXO with 50% probability. Then, the second pass selects any
-// unused UTXOs until the total value is greater than or equal to amt.
-func subsetWithLeastSumGreaterThan(amt uint64, utxos []*compositeUTXO) []*compositeUTXO {
+// unused UTXOs until the total value is enough.
+func subsetWithLeastOverFund(enough func(uint64, uint64) (bool, uint64), maxFund uint64, utxos []*compositeUTXO) []*compositeUTXO {
 	best := uint64(1 << 62)
 	var bestIncluded []bool
 	bestNumIncluded := 0
@@ -87,9 +104,9 @@ func subsetWithLeastSumGreaterThan(amt uint64, utxos []*compositeUTXO) []*compos
 	included := make([]bool, len(utxos))
 	const iterations = 1000
 
-searchLoop:
 	for nRep := 0; nRep < iterations; nRep++ {
 		var nTotal uint64
+		var totalSize uint64
 		var numIncluded int
 
 		for nPass := 0; nPass < 2; nPass++ {
@@ -104,20 +121,19 @@ searchLoop:
 					included[i] = true
 					numIncluded++
 					nTotal += shuffledUTXOs[i].amount
-					if nTotal >= amt {
-						if nTotal < best || (nTotal == best && numIncluded < bestNumIncluded) {
+					totalSize += uint64(shuffledUTXOs[i].input.VBytes())
+					if e, _ := enough(totalSize, nTotal); e {
+						if nTotal < best || (nTotal == best && numIncluded < bestNumIncluded) && nTotal <= maxFund {
 							best = nTotal
 							if bestIncluded == nil {
 								bestIncluded = make([]bool, len(shuffledUTXOs))
 							}
 							copy(bestIncluded, included)
 							bestNumIncluded = numIncluded
 						}
-						if nTotal == amt {
-							break searchLoop
-						}
 						included[i] = false
 						nTotal -= shuffledUTXOs[i].amount
+						totalSize -= uint64(shuffledUTXOs[i].input.VBytes())
 						numIncluded--
 					}
 				}
@@ -160,37 +176,57 @@ searchLoop:
 //
 // If the provided UTXO set has less combined value than the requested amount a
 // nil slice is returned.
-func leastOverFund(amt uint64, utxos []*compositeUTXO) []*compositeUTXO {
-	if amt == 0 || sumUTXOs(utxos) < amt {
-		return nil
+func leastOverFund(enough func(inputsSize, sum uint64) (bool, uint64), utxos []*compositeUTXO) []*compositeUTXO {
+	return leastOverFundWithLimit(enough, math.MaxUint64, utxos)
+}
+
+func leastOverFundWithLimit(enough func(inputsSize, sum uint64) (bool, uint64), maxFund uint64, utxos []*compositeUTXO) []*compositeUTXO {
+	// Remove the UTXOs that are larger than maxFund
+	var smallEnoughUTXOs []*compositeUTXO
+	idx := sort.Search(len(utxos), func(i int) bool {
+		utxo := utxos[i]
+		return utxo.amount > maxFund
+	})
+	if idx == len(utxos) {
+		smallEnoughUTXOs = utxos
+	} else {
+		smallEnoughUTXOs = utxos[:idx]
 	}
 
 	// Partition - smallest UTXO that is large enough to fully fund, and the set
 	// of smaller ones.
-	idx := sort.Search(len(utxos), func(i int) bool {
-		return utxos[i].amount >= amt
+	idx = sort.Search(len(smallEnoughUTXOs), func(i int) bool {
+		utxo := smallEnoughUTXOs[i]
+		e, _ := enough(uint64(utxo.input.VBytes()), utxo.amount)
+		return e
 	})
 	var small []*compositeUTXO
-	var single *compositeUTXO // only return this if smaller ones would use more
-	if idx == len(utxos) {    // no one is enough
-		small = utxos
+	var single *compositeUTXO         // only return this if smaller ones would use more
+	if idx == len(smallEnoughUTXOs) { // no one is enough
+		small = smallEnoughUTXOs
 	} else {
-		small = utxos[:idx]
-		single = utxos[idx]
+		small = smallEnoughUTXOs[:idx]
+		single = smallEnoughUTXOs[idx]
 	}
 
-	// Find a subset of the small UTXO set with smallest combined amount.
 	var set []*compositeUTXO
-	if sumUTXOs(small) >= amt {
-		set = subsetWithLeastSumGreaterThan(amt, small)
-	} else if single != nil {
-		return []*compositeUTXO{single}
+	smallSetTotalValue := sumUTXOs(small)
+	smallSetTotalSize := sumUTXOSize(small)
+	if e, _ := enough(smallSetTotalSize, smallSetTotalValue); !e {
+		if single != nil {
+			return []*compositeUTXO{single}
+		} else {
+			return nil
+		}
+	} else {
+		set = subsetWithLeastOverFund(enough, maxFund, small)
 	}
 
 	// Return the small UTXO subset if it is less than the single big UTXO.
 	if single != nil && single.amount < sumUTXOs(set) {
 		return []*compositeUTXO{single}
 	}
+
 	return set
 }
 

client/asset/btc/coin_selection_test.go

@@ -0,0 +1,182 @@
+package btc
+
+import (
+	"math/rand"
+	"reflect"
+	"sort"
+	"testing"
+	"time"
+
+	dexbtc "decred.org/dcrdex/dex/networks/btc"
+)
+
+func Test_leastOverFund(t *testing.T) {
+	enough := func(_, sum uint64) (bool, uint64) {
+		return sum >= 10e8, 0
+	}
+	newU := func(amt float64) *compositeUTXO {
+		return &compositeUTXO{
+			utxo: &utxo{
+				amount: uint64(amt) * 1e8,
+			},
+			input: &dexbtc.SpendInfo{},
+		}
+	}
+	tests := []struct {
+		name  string
+		utxos []*compositeUTXO
+		want  []*compositeUTXO
+	}{
+		{
+			"1,3",
+			[]*compositeUTXO{newU(1), newU(8), newU(9)},
+			[]*compositeUTXO{newU(1), newU(9)},
+		},
+		{
+			"1,2",
+			[]*compositeUTXO{newU(1), newU(9)},
+			[]*compositeUTXO{newU(1), newU(9)},
+		},
+		{
+			"1,2++",
+			[]*compositeUTXO{newU(2), newU(9)},
+			[]*compositeUTXO{newU(2), newU(9)},
+		},
+		{
+			"2,3++",
+			[]*compositeUTXO{newU(0), newU(2), newU(9)},
+			[]*compositeUTXO{newU(2), newU(9)},
+		},
+		{
+			"3",
+			[]*compositeUTXO{newU(0), newU(2), newU(10)},
+			[]*compositeUTXO{newU(10)},
+		},
+		{
+			"subset",
+			[]*compositeUTXO{newU(1), newU(9), newU(11)},
+			[]*compositeUTXO{newU(1), newU(9)},
+		},
+		{
+			"subset small bias",
+			[]*compositeUTXO{newU(3), newU(6), newU(7)},
+			[]*compositeUTXO{newU(3), newU(7)},
+		},
+		{
+			"single exception",
+			[]*compositeUTXO{newU(5), newU(7), newU(11)},
+			[]*compositeUTXO{newU(11)},
+		},
+		{
+			"1 of 1",
+			[]*compositeUTXO{newU(10)},
+			[]*compositeUTXO{newU(10)},
+		},
+		{
+			"ok nil",
+			[]*compositeUTXO{newU(1), newU(8)},
+			nil,
+		},
+		{
+			"ok",
+			[]*compositeUTXO{newU(1)},
+			nil,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := leastOverFund(enough, tt.utxos)
+			sort.Slice(got, func(i int, j int) bool {
+				return got[i].amount < got[j].amount
+			})
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("subset() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Fuzz_leastOverFund(f *testing.F) {
+	type seed struct {
+		amt uint64
+		n   int
+	}
+
+	rnd := rand.New(rand.NewSource(1))
+
+	seeds := make([]seed, 0, 40)
+	for i := 0; i < 10; i++ {
+		seeds = append(seeds, seed{
+			amt: uint64(rnd.Intn(40)),
+			n:   rnd.Intn(20000),
+		})
+	}
+
+	for _, seed := range seeds {
+		f.Add(seed.amt, seed.n)
+	}
+
+	newU := func(amt float64) *compositeUTXO {
+		return &compositeUTXO{
+			utxo: &utxo{
+				amount: uint64(amt * 1e8),
+			},
+			input: &dexbtc.SpendInfo{},
+		}
+	}
+
+	var totalDuration time.Duration
+	var totalUTXO int64
+
+	f.Fuzz(func(t *testing.T, amt uint64, n int) {
+		if n < 1 || n > 65535 || amt == 0 || amt > 21e6 {
+			t.Skip()
+		}
+		m := 2 * amt / uint64(n)
+		utxos := make([]*compositeUTXO, n)
+		for i := range utxos {
+			var v float64
+			if rand.Intn(2) > 0 {
+				v = rand.Float64()
+			}
+			if m != 0 {
+				v += float64(rand.Int63n(int64(m)))
+			}
+			if v > 40000 {
+				t.Skip()
+			}
+			utxos[i] = newU(v)
+		}
+		startTime := time.Now()
+		enough := func(_, sum uint64) (bool, uint64) {
+			return sum >= amt*1e8, 0
+		}
+		leastOverFund(enough, utxos)
+		totalDuration += time.Since(startTime)
+		totalUTXO += int64(n)
+	})
+
+	f.Logf("leastOverFund: average duration: %v, with average number of UTXOs: %v\n", totalDuration/100, totalUTXO/100)
+}
+
+func BenchmarkLeastOverFund(b *testing.B) {
+	// Same amounts every time.
+	rnd := rand.New(rand.NewSource(1))
+	utxos := make([]*compositeUTXO, 2_000)
+	for i := range utxos {
+		utxo := &compositeUTXO{
+			utxo: &utxo{
+				amount: uint64(rnd.Int31n(100) * 1e8),
+			},
+			input: &dexbtc.SpendInfo{},
+		}
+		utxos[i] = utxo
+	}
+	b.ResetTimer()
+	for n := 0; n < b.N; n++ {
+		enough := func(_, sum uint64) (bool, uint64) {
+			return sum >= 10_000*1e8, 0
+		}
+		leastOverFund(enough, utxos)
+	}
+}

client/asset/btc/livetest/livetest.go

@@ -255,35 +255,35 @@ func Run(t *testing.T, cfg *Config) {
 
 	// Gamma should only have 10 BTC utxos, so calling fund for less should only
 	// return 1 utxo.
-	utxos, _, err := rig.secondWallet.FundOrder(ord)
+	utxos, _, _, err := rig.secondWallet.FundOrder(ord)
 	if err != nil {
 		t.Fatalf("Funding error: %v", err)
 	}
 	utxo := utxos[0]
 
 	// UTXOs should be locked
-	utxos, _, _ = rig.secondWallet.FundOrder(ord)
+	utxos, _, _, _ = rig.secondWallet.FundOrder(ord)
 	if inUTXOs(utxo, utxos) {
 		t.Fatalf("received locked output")
 	}
 	rig.secondWallet.ReturnCoins([]asset.Coin{utxo})
 	rig.secondWallet.ReturnCoins(utxos)
 	// Make sure we get the first utxo back with Fund.
-	utxos, _, _ = rig.secondWallet.FundOrder(ord)
+	utxos, _, _, _ = rig.secondWallet.FundOrder(ord)
 	if !cfg.SplitTx && !inUTXOs(utxo, utxos) {
 		t.Fatalf("unlocked output not returned")
 	}
 	rig.secondWallet.ReturnCoins(utxos)
 
 	// Get a separate set of UTXOs for each contract.
 	setOrderValue(contractValue)
-	utxos1, _, err := rig.secondWallet.FundOrder(ord)
+	utxos1, _, _, err := rig.secondWallet.FundOrder(ord)
 	if err != nil {
 		t.Fatalf("error funding first contract: %v", err)
 	}
 	// Get a separate set of UTXOs for each contract.
 	setOrderValue(contractValue * 2)
-	utxos2, _, err := rig.secondWallet.FundOrder(ord)
+	utxos2, _, _, err := rig.secondWallet.FundOrder(ord)
 	if err != nil {
 		t.Fatalf("error funding second contract: %v", err)
 	}
@@ -493,7 +493,7 @@ func Run(t *testing.T, cfg *Config) {
 
 	// Have gamma send a swap contract to the alpha address.
 	setOrderValue(contractValue)
-	utxos, _, _ = rig.secondWallet.FundOrder(ord)
+	utxos, _, _, _ = rig.secondWallet.FundOrder(ord)
 	contract := &asset.Contract{
 		Address:    address,
 		Value:      contractValue,