From 7bffca82904984840a884d7b677e66e06849437c Mon Sep 17 00:00:00 2001 From: rooooooooob Date: Tue, 26 Oct 2021 20:22:41 -0500 Subject: [PATCH] TxBuilder: Implement CIP-0002 Coin Selection https://github.com/cardano-foundation/CIPs/blob/master/CIP-0002/CIP-0002.md key differences 1) we take into account adjuting for free (which is out of CIP2's scope) 2) LargestFirst supports multiassets (naively) --- rust/Cargo.lock | 70 ++++++++- rust/Cargo.toml | 1 + rust/src/address.rs | 6 +- rust/src/legacy_address/address.rs | 8 +- rust/src/lib.rs | 2 +- rust/src/tx_builder.rs | 234 ++++++++++++++++++++++++++++- rust/src/utils.rs | 27 +++- 7 files changed, 327 insertions(+), 21 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 21c94d13..a83362b5 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "aho-corasick" version = "0.7.15" @@ -67,6 +69,7 @@ dependencies = [ "num-bigint", "quickcheck", "quickcheck_macros", + "rand 0.8.4", "rand_chacha 0.1.1", "rand_os", "serde_json", @@ -186,7 +189,18 @@ checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" dependencies = [ "cfg-if 0.1.10", "libc", - "wasi", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.10.2+wasi-snapshot-preview1", ] [[package]] @@ -317,7 +331,7 @@ checksum = "a44883e74aa97ad63db83c4bf8ca490f02b2fc02f92575e720c8551e843c945f" dependencies = [ "env_logger", "log", - "rand", + "rand 0.7.3", "rand_core 0.5.1", ] @@ -347,11 +361,23 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ - "getrandom", + "getrandom 0.1.15", "libc", "rand_chacha 0.2.2", "rand_core 0.5.1", - "rand_hc", + "rand_hc 0.2.0", +] + +[[package]] +name = "rand" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.3", + "rand_hc 0.3.1", ] [[package]] @@ -374,6 +400,16 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.3", +] + [[package]] name = "rand_core" version = "0.3.1" @@ -395,7 +431,16 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" dependencies = [ - "getrandom", + "getrandom 0.1.15", +] + +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom 0.2.3", ] [[package]] @@ -407,6 +452,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_hc" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" +dependencies = [ + "rand_core 0.6.3", +] + [[package]] name = "rand_os" version = "0.1.3" @@ -529,6 +583,12 @@ version = "0.9.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + [[package]] name = "wasm-bindgen" version = "0.2.74" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 751014a3..f606289f 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -30,6 +30,7 @@ num-bigint = "0.4.0" # feature or this one clear_on_drop = { version = "0.2", features = ["no_cc"] } itertools = "0.10.1" +rand = "0.8.4" # non-wasm [target.'cfg(not(all(target_arch = "wasm32", not(target_os = "emscripten"))))'.dependencies] diff --git a/rust/src/address.rs b/rust/src/address.rs index 5a841a5c..6ab61a8f 100644 --- a/rust/src/address.rs +++ b/rust/src/address.rs @@ -159,7 +159,7 @@ impl Deserialize for StakeCredential { } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Eq, Ord, PartialEq, PartialOrd)] enum AddrType { Base(BaseAddress), Ptr(PointerAddress), @@ -169,7 +169,7 @@ enum AddrType { } #[wasm_bindgen] -#[derive(Clone, Debug)] +#[derive(Debug, Clone, Eq, Ord, PartialEq, PartialOrd)] pub struct ByronAddress(pub (crate) ExtendedAddr); #[wasm_bindgen] impl ByronAddress { @@ -256,7 +256,7 @@ impl ByronAddress { } #[wasm_bindgen] -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Eq, Ord, PartialEq, PartialOrd)] pub struct Address(AddrType); from_bytes!(Address, data, { diff --git a/rust/src/legacy_address/address.rs b/rust/src/legacy_address/address.rs index 73d91c17..52b07036 100644 --- a/rust/src/legacy_address/address.rs +++ b/rust/src/legacy_address/address.rs @@ -76,7 +76,7 @@ impl cbor_event::de::Deserialize for AddrType { type HDAddressPayload = Vec; -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] +#[derive(Debug, Clone, Eq, Hash, Ord, PartialEq, PartialOrd)] pub struct Attributes { pub derivation_path: Option, pub protocol_magic: Option, @@ -182,10 +182,10 @@ fn hash_spending_data(addr_type: AddrType, xpub: &XPub, attrs: &Attributes) -> [ } /// A valid cardano Address that is displayed in base58 -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] +#[derive(Debug, Clone, Eq, Hash, Ord, PartialEq, PartialOrd)] pub struct Addr(Vec); -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Eq, Hash, Ord, PartialEq, PartialOrd)] pub enum AddressMatchXPub { Yes, No, @@ -277,7 +277,7 @@ impl cbor_event::de::Deserialize for Addr { const EXTENDED_ADDR_LEN: usize = 28; /// A valid cardano address deconstructed -#[derive(Debug, PartialEq, Eq, Clone)] +#[derive(Debug, Clone, Eq, Ord, PartialEq, PartialOrd)] pub struct ExtendedAddr { pub addr: [u8; EXTENDED_ADDR_LEN], pub attributes: Attributes, diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 429437f3..74f38b11 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -382,7 +382,7 @@ impl TransactionInput { } #[wasm_bindgen] -#[derive(Clone, Debug)] +#[derive(Debug, Clone, Eq, Ord, PartialEq, PartialOrd)] pub struct TransactionOutput { address: Address, pub (crate) amount: Value, diff --git a/rust/src/tx_builder.rs b/rust/src/tx_builder.rs index fd40cd2e..50a501f8 100644 --- a/rust/src/tx_builder.rs +++ b/rust/src/tx_builder.rs @@ -1,7 +1,7 @@ use super::*; use super::fees; use super::utils; -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; // comes from witsVKeyNeeded in the Ledger spec fn witness_keys_for_cert(cert_enum: &Certificate, keys: &mut BTreeSet) { @@ -125,6 +125,12 @@ struct TxBuilderInput { amount: Value, // we need to keep track of the amount in the inputs for input selection } +#[wasm_bindgen] +pub enum CoinSelectionStrategyCIP2 { + LargestFirst, + RandomImprove, +} + #[wasm_bindgen] #[derive(Clone, Debug)] pub struct TransactionBuilder { @@ -144,13 +150,101 @@ pub struct TransactionBuilder { validity_start_interval: Option, input_types: MockWitnessSet, mint: Option, + inputs_auto_added: bool, } #[wasm_bindgen] impl TransactionBuilder { - // We have to know what kind of inputs these are to know what kind of mock witnesses to create since - // 1) mock witnesses have different lengths depending on the type which changes the expecting fee - // 2) Witnesses are a set so we need to get rid of duplicates to avoid over-estimating the fee + /// This automatically selects and adds inputs from {inputs} consisting of just enough to cover + /// the outputs that have already been added. + /// This should be called after adding all certs/outputs/etc and will be an error otherwise. + /// Uses CIP2: https://github.com/cardano-foundation/CIPs/blob/master/CIP-0002/CIP-0002.md + /// Adding a change output must be called after via TransactionBuilder::add_change_if_needed() + /// This function, diverging from CIP2, takes into account fees and will attempt to add additional + /// inputs to cover the minimum fees. This does not, however, set the txbuilder's fee. + pub fn add_inputs_from(&mut self, inputs: &TransactionUnspentOutputs, strategy: CoinSelectionStrategyCIP2) -> Result<(), JsError> { + let mut available_inputs = inputs.0.clone(); + let mut input_total = self + .get_explicit_input()? + .checked_add(&self.get_implicit_input()?)?; + let mut output_total = self + .get_explicit_output()? + .checked_add(&Value::new(&self.get_deposit()?))? + .checked_add(&Value::new(&self.min_fee()?))?; + match strategy { + CoinSelectionStrategyCIP2::LargestFirst => { + available_inputs.sort_by_key(|input| input.output.amount.coin); + // iterate in decreasing order of ADA-only value + for input in available_inputs.iter().rev() { + if input_total >= output_total { + break; + } + // differing from CIP2, we include the needed fees in the targets instead of just output values + let input_fee = self.fee_for_input(&input.output.address, &input.input, &input.output.amount)?; + self.add_input(&input.output.address, &input.input, &input.output.amount); + input_total = input_total.checked_add(&input.output.amount)?; + output_total = output_total.checked_add(&Value::new(&input_fee))?; + } + if input_total < output_total { + return Err(JsError::from_str("UTxO Balance Insufficient")); + } + }, + CoinSelectionStrategyCIP2::RandomImprove => { + use rand::Rng; + if self.outputs.0.iter().any(|output| output.amount.multiasset.is_some()) { + return Err(JsError::from_str("Multiasset values not supported by RandomImprove. Please use LargestFirst")); + } + // Phase 1: Random Selection + let mut associated_inputs: BTreeMap> = BTreeMap::new(); + let mut rng = rand::thread_rng(); + let mut outputs = self.outputs.0.clone(); + outputs.sort_by_key(|output| output.amount.coin); + for output in outputs.iter().rev() { + let mut added = Value::new(&Coin::zero()); + let mut needed = output.amount.clone(); + while added < needed { + if available_inputs.is_empty() { + return Err(JsError::from_str("UTxO Balance Insufficient")); + } + let random_index = rng.gen_range(0..available_inputs.len()); + let input = available_inputs.swap_remove(random_index); + // differing from CIP2, we include the needed fees in the targets instead of just output values + let input_fee = self.fee_for_input(&input.output.address, &input.input, &input.output.amount)?; + self.add_input(&input.output.address, &input.input, &input.output.amount); + input_total = input_total.checked_add(&input.output.amount)?; + output_total = output_total.checked_add(&Value::new(&input_fee))?; + needed = needed.checked_add(&Value::new(&input_fee))?; + added = added.checked_add(&input.output.amount)?; + associated_inputs.entry(output.clone()).or_default().push(input); + } + } + // Phase 2: Improvement + for output in outputs.iter_mut() { + let associated = associated_inputs.get_mut(output).unwrap(); + for input in associated.iter_mut() { + let random_index = rng.gen_range(0..available_inputs.len()); + let new_input = available_inputs.get_mut(random_index).unwrap(); + let cur = from_bignum(&input.output.amount.coin); + let new = from_bignum(&new_input.output.amount.coin); + let min = from_bignum(&output.amount.coin); + let ideal = 2 * min; + let max = 3 * min; + let move_closer = (ideal as i128 - new as i128).abs() < (ideal as i128 - cur as i128).abs(); + let not_exceed_max = new < max; + if move_closer && not_exceed_max { + std::mem::swap(input, new_input); + } + } + } + }, + } + + Ok(()) + } + + /// We have to know what kind of inputs these are to know what kind of mock witnesses to create since + /// 1) mock witnesses have different lengths depending on the type which changes the expecting fee + /// 2) Witnesses are a set so we need to get rid of duplicates to avoid over-estimating the fee pub fn add_key_input(&mut self, hash: &Ed25519KeyHash, input: &TransactionInput, amount: &Value) { self.inputs.push(TxBuilderInput { input: input.clone(), @@ -222,7 +316,7 @@ impl TransactionBuilder { } /// calculates how much the fee would increase if you added a given output - pub fn fee_for_input(&mut self, address: &Address, input: &TransactionInput, amount: &Value) -> Result { + pub fn fee_for_input(&self, address: &Address, input: &TransactionInput, amount: &Value) -> Result { let mut self_copy = self.clone(); // we need some value for these for it to be a a valid transaction @@ -334,7 +428,8 @@ impl TransactionBuilder { bootstraps: BTreeSet::new(), }, validity_start_interval: None, - mint: None + mint: None, + inputs_auto_added: false, } } @@ -1606,4 +1701,131 @@ mod tests { assert!(tx_builder.add_change_if_needed(&change_addr).is_err()) } + + fn make_input(input_hash_byte: u8, value: Value) -> TransactionUnspentOutput { + TransactionUnspentOutput::new( + &TransactionInput::new(&TransactionHash::from([input_hash_byte; 32]), 0), + &TransactionOutput::new(&Address::from_bech32("addr1vyy6nhfyks7wdu3dudslys37v252w2nwhv0fw2nfawemmnqs6l44z").unwrap(), &value) + ) + } + + #[test] + fn tx_builder_cip2_largest_first_increasing_fees() { + // we have a = 1 to test increasing fees when more inputs are added + let linear_fee = LinearFee::new(&to_bignum(1), &to_bignum(0)); + let mut tx_builder = TransactionBuilder::new( + &linear_fee, + &Coin::zero(), + &to_bignum(0), + &to_bignum(0), + 9999, + 9999 + ); + tx_builder.add_output(&TransactionOutput::new( + &Address::from_bech32("addr1vyy6nhfyks7wdu3dudslys37v252w2nwhv0fw2nfawemmnqs6l44z").unwrap(), + &Value::new(&to_bignum(1000)) + )).unwrap(); + let mut available_inputs = TransactionUnspentOutputs::new(); + available_inputs.add(&make_input(0u8, Value::new(&to_bignum(150)))); + available_inputs.add(&make_input(1u8, Value::new(&to_bignum(200)))); + available_inputs.add(&make_input(2u8, Value::new(&to_bignum(800)))); + available_inputs.add(&make_input(3u8, Value::new(&to_bignum(400)))); + available_inputs.add(&make_input(4u8, Value::new(&to_bignum(100)))); + tx_builder.add_inputs_from(&available_inputs, CoinSelectionStrategyCIP2::LargestFirst).unwrap(); + let change_addr = ByronAddress::from_base58("Ae2tdPwUPEZGUEsuMAhvDcy94LKsZxDjCbgaiBBMgYpR8sKf96xJmit7Eho").unwrap().to_address(); + let change_added = tx_builder.add_change_if_needed(&change_addr).unwrap(); + assert!(change_added); + let tx = tx_builder.build().unwrap(); + // change needed + assert_eq!(2, tx.outputs().len()); + assert_eq!(3, tx.inputs().len()); + // confirm order of only what is necessary + assert_eq!(2u8, tx.inputs().get(0).transaction_id().0[0]); + assert_eq!(3u8, tx.inputs().get(1).transaction_id().0[0]); + assert_eq!(1u8, tx.inputs().get(2).transaction_id().0[0]); + } + + + #[test] + fn tx_builder_cip2_largest_first_static_fees() { + // we have a = 0 so we know adding inputs/outputs doesn't change the fee so we can analyze more + let linear_fee = LinearFee::new(&to_bignum(0), &to_bignum(0)); + let mut tx_builder = TransactionBuilder::new( + &linear_fee, + &Coin::zero(), + &to_bignum(0), + &to_bignum(0), + 9999, + 9999 + ); + tx_builder.add_output(&TransactionOutput::new( + &Address::from_bech32("addr1vyy6nhfyks7wdu3dudslys37v252w2nwhv0fw2nfawemmnqs6l44z").unwrap(), + &Value::new(&to_bignum(1200)) + )).unwrap(); + let mut available_inputs = TransactionUnspentOutputs::new(); + available_inputs.add(&make_input(0u8, Value::new(&to_bignum(150)))); + available_inputs.add(&make_input(1u8, Value::new(&to_bignum(200)))); + available_inputs.add(&make_input(2u8, Value::new(&to_bignum(800)))); + available_inputs.add(&make_input(3u8, Value::new(&to_bignum(400)))); + available_inputs.add(&make_input(4u8, Value::new(&to_bignum(100)))); + tx_builder.add_inputs_from(&available_inputs, CoinSelectionStrategyCIP2::LargestFirst).unwrap(); + let change_addr = ByronAddress::from_base58("Ae2tdPwUPEZGUEsuMAhvDcy94LKsZxDjCbgaiBBMgYpR8sKf96xJmit7Eho").unwrap().to_address(); + let change_added = tx_builder.add_change_if_needed(&change_addr).unwrap(); + assert!(!change_added); + let tx = tx_builder.build().unwrap(); + // change not needed - should be exact + assert_eq!(1, tx.outputs().len()); + assert_eq!(2, tx.inputs().len()); + // confirm order of only what is necessary + assert_eq!(2u8, tx.inputs().get(0).transaction_id().0[0]); + assert_eq!(3u8, tx.inputs().get(1).transaction_id().0[0]); + } + + + #[test] + fn tx_builder_cip2_random_improve() { + // we have a = 1 to test increasing fees when more inputs are added + let linear_fee = LinearFee::new(&to_bignum(1), &to_bignum(0)); + let mut tx_builder = TransactionBuilder::new( + &linear_fee, + &Coin::zero(), + &to_bignum(0), + &to_bignum(0), + 9999, + 9999 + ); + const COST: u64 = 1000; + tx_builder.add_output(&TransactionOutput::new( + &Address::from_bech32("addr1vyy6nhfyks7wdu3dudslys37v252w2nwhv0fw2nfawemmnqs6l44z").unwrap(), + &Value::new(&to_bignum(COST)) + )).unwrap(); + let mut available_inputs = TransactionUnspentOutputs::new(); + available_inputs.add(&make_input(0u8, Value::new(&to_bignum(150)))); + available_inputs.add(&make_input(1u8, Value::new(&to_bignum(200)))); + available_inputs.add(&make_input(2u8, Value::new(&to_bignum(800)))); + available_inputs.add(&make_input(3u8, Value::new(&to_bignum(400)))); + available_inputs.add(&make_input(4u8, Value::new(&to_bignum(100)))); + available_inputs.add(&make_input(5u8, Value::new(&to_bignum(200)))); + available_inputs.add(&make_input(6u8, Value::new(&to_bignum(150)))); + tx_builder.add_inputs_from(&available_inputs, CoinSelectionStrategyCIP2::RandomImprove).unwrap(); + let change_addr = ByronAddress::from_base58("Ae2tdPwUPEZGUEsuMAhvDcy94LKsZxDjCbgaiBBMgYpR8sKf96xJmit7Eho").unwrap().to_address(); + let _change_added = tx_builder.add_change_if_needed(&change_addr).unwrap(); + let tx = tx_builder.build().unwrap(); + // we need to look up the values to ensure there's enough + let mut input_values = BTreeMap::new(); + for utxo in available_inputs.0.iter() { + input_values.insert(utxo.input.transaction_id(), utxo.output.amount.clone()); + } + let mut encountered = std::collections::HashSet::new(); + let mut input_total = Value::new(&Coin::zero()); + for input in tx.inputs.0.iter() { + let txid = input.transaction_id(); + if !encountered.insert(txid.clone()) { + panic!("Input {:?} duplicated", txid); + } + let value = input_values.get(&txid).unwrap(); + input_total = input_total.checked_add(value).unwrap(); + } + assert!(input_total >= Value::new(&tx_builder.min_fee().unwrap().checked_add(&to_bignum(COST)).unwrap())); + } } diff --git a/rust/src/utils.rs b/rust/src/utils.rs index 282cf67b..9331a478 100644 --- a/rust/src/utils.rs +++ b/rust/src/utils.rs @@ -68,8 +68,8 @@ macro_rules! to_from_bytes { #[wasm_bindgen] #[derive(Clone, Debug)] pub struct TransactionUnspentOutput { - input: TransactionInput, - output: TransactionOutput + pub(crate) input: TransactionInput, + pub(crate) output: TransactionOutput } to_from_bytes!(TransactionUnspentOutput); @@ -134,6 +134,29 @@ impl Deserialize for TransactionUnspentOutput { } } +#[wasm_bindgen] +#[derive(Clone, Debug)] +pub struct TransactionUnspentOutputs(pub(crate) Vec); + +#[wasm_bindgen] +impl TransactionUnspentOutputs { + pub fn new() -> Self { + Self(Vec::new()) + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn get(&self, index: usize) -> TransactionUnspentOutput { + self.0[index].clone() + } + + pub fn add(&mut self, elem: &TransactionUnspentOutput) { + self.0.push(elem.clone()); + } +} + // Generic u64 wrapper for platforms that don't support u64 or BigInt/etc // This is an unsigned type - no negative numbers. // Can be converted to/from plain rust