From 7ee2521ef5d68e4890991338ce1d65091e5cdf2f Mon Sep 17 00:00:00 2001 From: Denys Zadorozhnyi Date: Tue, 8 Feb 2022 16:34:49 +0200 Subject: [PATCH 01/21] add test case for incorrect BIP32 private child key derivation; --- .../secrets/ExtendedSecretKeySpec.scala | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKeySpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKeySpec.scala index 24d8831b63..b08da419a1 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKeySpec.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKeySpec.scala @@ -7,6 +7,8 @@ import org.scalatest.matchers.should.Matchers import org.scalatest.propspec.AnyPropSpec import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks import scorex.util.encode.Base58 +import org.ergoplatform.P2PKAddress +import org.ergoplatform.ErgoAddressEncoder class ExtendedSecretKeySpec extends AnyPropSpec @@ -16,6 +18,9 @@ class ExtendedSecretKeySpec val seedStr = "edge talent poet tortoise trumpet dose" val seed: Array[Byte] = Mnemonic.toSeed(SecretString.create(seedStr)) + def equalBase58(v1: Array[Byte], v2b58: String): Assertion = + Base58.encode(v1) shouldEqual v2b58 + property("key tree derivation from seed (test vectors from BIP32 check)") { val expectedRoot = "4rEDKLd17LX4xNR8ss4ithdqFRc3iFnTiTtQbanWJbCT" val cases = Seq( @@ -50,6 +55,25 @@ class ExtendedSecretKeySpec } } - def equalBase58(v1: Array[Byte], v2b58: String): Assertion = Base58.encode(v1) shouldEqual v2b58 + property("incorrect BIP32 key derivation (31 bit child key)") { + // see https://github.com/ergoplatform/ergo/issues/1627 for details + val seedStr = + "race relax argue hair sorry riot there spirit ready fetch food hedgehog hybrid mobile pretty" + val seed: Array[Byte] = Mnemonic.toSeed(SecretString.create(seedStr)) + val cases = Seq( + ("9ewv8sxJ1jfr6j3WUSbGPMTVx3TZgcJKdnjKCbJWhiJp5U62uhP", "m/44'/429'/0'/0/0") + ) + + val root = ExtendedSecretKey.deriveMasterKey(seed) + + val addressEncoder = ErgoAddressEncoder(ErgoAddressEncoder.MainnetNetworkPrefix) + + cases.foreach { + case (expectedP2PK, path) => + val derived = root.derive(DerivationPath.fromEncoded(path).get) + val p2pk = P2PKAddress(derived.publicKey.key)(addressEncoder) + p2pk.toString shouldEqual expectedP2PK + } + } } From a5e61159c400cd62407d8a1d26380e716fa9090f Mon Sep 17 00:00:00 2001 From: Denys Zadorozhnyi Date: Tue, 8 Feb 2022 19:32:20 +0200 Subject: [PATCH 02/21] add comment to ExtendedSecretKey derivation explaining less-than-32-bytes child key bug; --- .../org/ergoplatform/wallet/secrets/ExtendedSecretKey.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKey.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKey.scala index 90f8be2344..34928b9070 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKey.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKey.scala @@ -55,6 +55,10 @@ final class ExtendedSecretKey(val keyBytes: Array[Byte], object ExtendedSecretKey { + // For some seeds 'parentKey.keyBytes' is less than 32 bytes while BIP32 requires 32 bytes. + // The reason for this is childKey being small(ish) and 'BigIntegers.asUnsignedByteArray(childKey)' + // is less than 32 bytes while BIP32 requires 32 bytes. + // see https://github.com/ergoplatform/ergo/issues/1627 for details @scala.annotation.tailrec def deriveChildSecretKey(parentKey: ExtendedSecretKey, idx: Int): ExtendedSecretKey = { val keyCoded: Array[Byte] = From 34b981751c8c60f9d763f5e1303dba58d9ccea8c Mon Sep 17 00:00:00 2001 From: Denys Zadorozhnyi Date: Mon, 14 Feb 2022 17:07:20 +0200 Subject: [PATCH 03/21] introduce ExtendedSecretKey.usePre1627KeyDerivation and fix BIP32 secret key derivation; --- .../wallet/secrets/EncryptedSecret.scala | 15 ++++--- .../wallet/secrets/ExtendedKey.scala | 4 -- .../wallet/secrets/ExtendedPublicKey.scala | 6 +-- .../wallet/secrets/ExtendedSecretKey.scala | 29 ++++++++++---- .../wallet/secrets/JsonSecretStorage.scala | 40 ++++++++++++------- .../wallet/AddressGenerationDemo.java | 2 +- .../wallet/CreateTransactionDemo.java | 6 +-- .../ErgoProvingInterpreterSpec.scala | 2 +- .../interpreter/ErgoUnsafeProverSpec.scala | 2 +- .../wallet/secrets/DerivationPathSpec.scala | 6 +-- .../secrets/ExtendedPublicKeySpec.scala | 2 +- .../secrets/ExtendedSecretKeySpec.scala | 28 ++++++------- .../secrets/JsonSecretStorageSpec.scala | 4 +- .../transactions/TransactionBuilderSpec.scala | 2 +- .../wallet/utils/Generators.scala | 2 +- .../http/api/WalletApiRoute.scala | 14 ++++--- .../nodeView/wallet/ErgoWalletActor.scala | 6 +-- .../nodeView/wallet/ErgoWalletReader.scala | 4 +- .../nodeView/wallet/ErgoWalletService.scala | 11 +++-- .../nodeView/wallet/ErgoWalletSupport.scala | 2 +- .../http/routes/WalletApiRouteSpec.scala | 3 +- .../wallet/ErgoWalletServiceSpec.scala | 5 ++- .../persistence/WalletRegistryBenchmark.scala | 2 +- .../utils/ErgoTestConstants.scala | 2 +- .../scala/org/ergoplatform/utils/Stubs.scala | 2 +- 25 files changed, 117 insertions(+), 84 deletions(-) diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/EncryptedSecret.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/EncryptedSecret.scala index a055c6ed3d..d7d1f7f7c8 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/EncryptedSecret.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/EncryptedSecret.scala @@ -13,18 +13,21 @@ import scorex.util.encode.Base16 * @param iv - cipher initialization vector * @param authTag - message authentication tag * @param cipherParams - cipher params + * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation (see https://github.com/ergoplatform/ergo/issues/1627 for details) */ final case class EncryptedSecret(cipherText: String, salt: String, iv: String, authTag: String, - cipherParams: EncryptionSettings) + cipherParams: EncryptionSettings, usePre1627KeyDerivation: Option[Boolean]) object EncryptedSecret { def apply(cipherText: Array[Byte], salt: Array[Byte], iv: Array[Byte], authTag: Array[Byte], - cipherParams: EncryptionSettings): EncryptedSecret = { + cipherParams: EncryptionSettings, usePre1627KeyDerivation: Option[Boolean]): EncryptedSecret = { new EncryptedSecret( Base16.encode(cipherText), Base16.encode(salt), Base16.encode(iv), - Base16.encode(authTag), cipherParams) + Base16.encode(authTag), + cipherParams, + usePre1627KeyDerivation) } implicit object EncryptedSecretEncoder extends Encoder[EncryptedSecret] { @@ -35,7 +38,8 @@ object EncryptedSecret { "salt" -> secret.salt.asJson, "iv" -> secret.iv.asJson, "authTag" -> secret.authTag.asJson, - "cipherParams" -> secret.cipherParams.asJson + "cipherParams" -> secret.cipherParams.asJson, + "usePre1627KeyDerivation" -> secret.usePre1627KeyDerivation.asJson ) } @@ -50,7 +54,8 @@ object EncryptedSecret { iv <- cursor.downField("iv").as[String] authTag <- cursor.downField("authTag").as[String] cipherParams <- cursor.downField("cipherParams").as[EncryptionSettings] - } yield EncryptedSecret(cipherText, salt, iv, authTag, cipherParams) + usePre1627KeyDerivation <- cursor.downField("usePre1627KeyDerivation").as[Option[Boolean]] + } yield EncryptedSecret(cipherText, salt, iv, authTag, cipherParams, usePre1627KeyDerivation) } } diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedKey.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedKey.scala index 6fb3b03dbc..fd11e311c2 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedKey.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedKey.scala @@ -15,10 +15,6 @@ package org.ergoplatform.wallet.secrets */ trait ExtendedKey[T <: ExtendedKey[T]] { - val keyBytes: Array[Byte] - - val chainCode: Array[Byte] - val path: DerivationPath /** Returns subtype reference. diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedPublicKey.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedPublicKey.scala index 21ca2c8b30..ad3955bf0f 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedPublicKey.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedPublicKey.scala @@ -16,9 +16,9 @@ import scala.annotation.tailrec * Public key, its chain code and path in key tree. * (see: https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki) */ -final class ExtendedPublicKey(val keyBytes: Array[Byte], - val chainCode: Array[Byte], - val path: DerivationPath) +final class ExtendedPublicKey(private[secrets] val keyBytes: Array[Byte], + private[secrets] val chainCode: Array[Byte], + val path: DerivationPath) extends ExtendedKey[ExtendedPublicKey] { def selfReflection: ExtendedPublicKey = this diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKey.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKey.scala index 34928b9070..03126c3a04 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKey.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKey.scala @@ -16,8 +16,9 @@ import sigmastate.interpreter.CryptoConstants * Secret, its chain code and path in key tree. * (see: https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki) */ -final class ExtendedSecretKey(val keyBytes: Array[Byte], - val chainCode: Array[Byte], +final class ExtendedSecretKey(private[secrets] val keyBytes: Array[Byte], + private[secrets] val chainCode: Array[Byte], + private[secrets] val usePre1627KeyDerivation: Boolean, val path: DerivationPath) extends ExtendedKey[ExtendedSecretKey] with SecretKey { @@ -73,8 +74,16 @@ object ExtendedSecretKey { .mod(CryptoConstants.groupOrder) if (childKeyProtoDecoded.compareTo(CryptoConstants.groupOrder) >= 0 || childKey.equals(BigInteger.ZERO)) deriveChildSecretKey(parentKey, idx + 1) - else - new ExtendedSecretKey(BigIntegers.asUnsignedByteArray(childKey), childChainCode, parentKey.path.extended(idx)) + else { + val keyBytes = if (parentKey.usePre1627KeyDerivation) { + // maybe less than 32 bytes if childKey is small enough + BigIntegers.asUnsignedByteArray(childKey) + } else { + // padded with leading zeroes to 32 bytes + BigIntegers.asUnsignedByteArray(Constants.SecretKeyLength, childKey) + } + new ExtendedSecretKey(keyBytes, childChainCode, parentKey.usePre1627KeyDerivation, parentKey.path.extended(idx)) + } } def deriveChildPublicKey(parentKey: ExtendedSecretKey, idx: Int): ExtendedPublicKey = { @@ -84,9 +93,15 @@ object ExtendedSecretKey { new ExtendedPublicKey(derivedPk, derivedSecret.chainCode, derivedPath) } - def deriveMasterKey(seed: Array[Byte]): ExtendedSecretKey = { + + /** + * Derives master secret key from the seed + * @param seed - seed bytes + * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation (see https://github.com/ergoplatform/ergo/issues/1627 for details) + */ + def deriveMasterKey(seed: Array[Byte], usePre1627KeyDerivation: Boolean): ExtendedSecretKey = { val (masterKey, chainCode) = HmacSHA512.hash(Constants.BitcoinSeed, seed).splitAt(Constants.SecretKeyLength) - new ExtendedSecretKey(masterKey, chainCode, DerivationPath.MasterPath) + new ExtendedSecretKey(masterKey, chainCode, usePre1627KeyDerivation, DerivationPath.MasterPath) } } @@ -108,7 +123,7 @@ object ExtendedSecretKeySerializer extends ErgoWalletSerializer[ExtendedSecretKe val chainCode = r.getBytes(Constants.SecretKeyLength) val pathLen = r.getUInt().toIntExact val path = DerivationPathSerializer.parseBytes(r.getBytes(pathLen)) - new ExtendedSecretKey(keyBytes, chainCode, path) + new ExtendedSecretKey(keyBytes, chainCode, false, path) } } diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/JsonSecretStorage.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/JsonSecretStorage.scala index 105822af3f..38e0fda512 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/JsonSecretStorage.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/JsonSecretStorage.scala @@ -39,9 +39,11 @@ final class JsonSecretStorage(val secretFile: File, encryptionSettings: Encrypti * @param mnemonicPassOpt - optional SecretString mnemonic password to be erased after use. */ override def checkSeed(mnemonic: SecretString, mnemonicPassOpt: Option[SecretString]): Boolean = { - val seed = Mnemonic.toSeed(mnemonic, mnemonicPassOpt) - val secret = ExtendedSecretKey.deriveMasterKey(seed) - unlockedSecret.fold(false)(s => secret.equals(s)) + unlockedSecret.fold(false){ uSecret => + val seed = Mnemonic.toSeed(mnemonic, mnemonicPassOpt) + val secret = ExtendedSecretKey.deriveMasterKey(seed, uSecret.usePre1627KeyDerivation) + secret.equals(uSecret) + } } /** @@ -58,20 +60,20 @@ final class JsonSecretStorage(val secretFile: File, encryptionSettings: Encrypti .flatMap(txt => Base16.decode(encryptedSecret.salt) .flatMap(salt => Base16.decode(encryptedSecret.iv) .flatMap(iv => Base16.decode(encryptedSecret.authTag) - .map(tag => (txt, salt, iv, tag)) + .map(tag => (txt, salt, iv, tag, encryptedSecret.usePre1627KeyDerivation)) ) ) ) - .flatMap { case (cipherText, salt, iv, tag) => { + .flatMap { case (cipherText, salt, iv, tag, usePre1627KeyDerivation) => { val res = crypto.AES.decrypt(cipherText, pass.getData(), salt, iv, tag)(encryptionSettings) pass.erase() res + .map(seed => unlockedSecret = Some(ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation.getOrElse(true)))) } } } - .fold(Failure(_), Success(_)) + . fold(Failure(_), Success(_)) .flatten - .map(seed => unlockedSecret = Some(ExtendedSecretKey.deriveMasterKey(seed))) } /** @@ -87,13 +89,16 @@ final class JsonSecretStorage(val secretFile: File, encryptionSettings: Encrypti object JsonSecretStorage { /** - * Initializes storage instance with new wallet file encrypted with the given `pass`. - */ - def init(seed: Array[Byte], pass: SecretString)(settings: SecretStorageSettings): JsonSecretStorage = { + * Initializes storage instance with new wallet file encrypted with the given `pass`. + * @param seed - seed bytes + * @param pass - encryption password + * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation (see https://github.com/ergoplatform/ergo/issues/1627 for details) + */ + def init(seed: Array[Byte], pass: SecretString, usePre1627KeyDerivation: Boolean)(settings: SecretStorageSettings): JsonSecretStorage = { val iv = scorex.utils.Random.randomBytes(crypto.AES.NonceBitsLen / 8) val salt = scorex.utils.Random.randomBytes(32) val (ciphertext, tag) = crypto.AES.encrypt(seed, pass.getData(), salt, iv)(settings.encryption) - val encryptedSecret = EncryptedSecret(ciphertext, salt, iv, tag, settings.encryption) + val encryptedSecret = EncryptedSecret(ciphertext, salt, iv, tag, settings.encryption, Some(usePre1627KeyDerivation)) val uuid = UUID.nameUUIDFromBytes(ciphertext) new File(settings.secretDir).mkdirs() val file = new File(s"${settings.secretDir}/$uuid.json") @@ -110,14 +115,19 @@ object JsonSecretStorage { } /** - * Initializes storage with the seed derived from an existing mnemonic phrase. - */ + * Initializes storage with the seed derived from an existing mnemonic phrase. + * @param mnemonic - mnemonic phase + * @param mnemonicPassOpt - optional mnemonic password + * @param encryptionPass - encryption password + * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation (see https://github.com/ergoplatform/ergo/issues/1627 for details) + */ def restore(mnemonic: SecretString, mnemonicPassOpt: Option[SecretString], encryptionPass: SecretString, - settings: SecretStorageSettings): JsonSecretStorage = { + settings: SecretStorageSettings, + usePre1627KeyDerivation: Boolean): JsonSecretStorage = { val seed = Mnemonic.toSeed(mnemonic, mnemonicPassOpt) - init(seed, encryptionPass)(settings) + init(seed, encryptionPass, usePre1627KeyDerivation)(settings) } def readFile(settings: SecretStorageSettings): Try[JsonSecretStorage] = { diff --git a/ergo-wallet/src/test/java/org/ergoplatform/wallet/AddressGenerationDemo.java b/ergo-wallet/src/test/java/org/ergoplatform/wallet/AddressGenerationDemo.java index 39568e63a4..2daa664862 100644 --- a/ergo-wallet/src/test/java/org/ergoplatform/wallet/AddressGenerationDemo.java +++ b/ergo-wallet/src/test/java/org/ergoplatform/wallet/AddressGenerationDemo.java @@ -19,7 +19,7 @@ public static byte[] secretSeedFromMnemonic(String mnemonic) { } public static ExtendedSecretKey masterSecretFromSeed(byte[] seed) { - ExtendedSecretKey rootSk = ExtendedSecretKey.deriveMasterKey(seed); + ExtendedSecretKey rootSk = ExtendedSecretKey.deriveMasterKey(seed, false); return rootSk; } diff --git a/ergo-wallet/src/test/java/org/ergoplatform/wallet/CreateTransactionDemo.java b/ergo-wallet/src/test/java/org/ergoplatform/wallet/CreateTransactionDemo.java index f7a9d1cc35..258b0f4f57 100644 --- a/ergo-wallet/src/test/java/org/ergoplatform/wallet/CreateTransactionDemo.java +++ b/ergo-wallet/src/test/java/org/ergoplatform/wallet/CreateTransactionDemo.java @@ -33,7 +33,7 @@ public static void createTransaction() throws Exception { // Create an address byte[] entropy = Random.randomBytes(32); - ExtendedSecretKey extendedSecretKey = ExtendedSecretKey.deriveMasterKey(entropy); + ExtendedSecretKey extendedSecretKey = ExtendedSecretKey.deriveMasterKey(entropy, false); ErgoAddress myAddress = P2PKAddress.apply(extendedSecretKey.privateInput().publicImage(), encoder); int transferAmt = 25000000; // amount to transfer @@ -118,11 +118,11 @@ public static void createTransactionMultipleKeys() throws Exception { // Create second address byte[] entropy1 = Random.randomBytes(32); - ExtendedSecretKey extendedSecretKey1 = ExtendedSecretKey.deriveMasterKey(entropy1); + ExtendedSecretKey extendedSecretKey1 = ExtendedSecretKey.deriveMasterKey(entropy1, false); ErgoAddress changeAddress = P2PKAddress.apply(extendedSecretKey1.privateInput().publicImage(), encoder); byte[] entropy2 = Random.randomBytes(32); - ExtendedSecretKey extendedSecretKey2 = ExtendedSecretKey.deriveMasterKey(entropy2); + ExtendedSecretKey extendedSecretKey2 = ExtendedSecretKey.deriveMasterKey(entropy2, false); int transferAmt = 25000000; // amount to transfer diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/ErgoProvingInterpreterSpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/ErgoProvingInterpreterSpec.scala index bb3111fe97..438770a1f1 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/ErgoProvingInterpreterSpec.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/ErgoProvingInterpreterSpec.scala @@ -23,7 +23,7 @@ class ErgoProvingInterpreterSpec with Generators with InterpreterSpecCommon { - private def obtainSecretKey() = ExtendedSecretKey.deriveMasterKey(Random.randomBytes(32)) + private def obtainSecretKey() = ExtendedSecretKey.deriveMasterKey(Random.randomBytes(32), usePre1627KeyDerivation = false) it should "produce proofs with primitive secrets" in { val extendedSecretKey = obtainSecretKey() diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/ErgoUnsafeProverSpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/ErgoUnsafeProverSpec.scala index 3cd92d0a0c..2be289322c 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/ErgoUnsafeProverSpec.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/ErgoUnsafeProverSpec.scala @@ -17,7 +17,7 @@ class ErgoUnsafeProverSpec it should "produce the same proof as a fully-functional prover" in { val entropy = Random.randomBytes(32) - val extendedSecretKey = ExtendedSecretKey.deriveMasterKey(entropy) + val extendedSecretKey = ExtendedSecretKey.deriveMasterKey(entropy, usePre1627KeyDerivation = false) val fullProver = ErgoProvingInterpreter(extendedSecretKey, parameters) val unsafeProver = ErgoUnsafeProver diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/DerivationPathSpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/DerivationPathSpec.scala index 71ccc10faa..0a598b32a4 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/DerivationPathSpec.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/DerivationPathSpec.scala @@ -33,7 +33,7 @@ class DerivationPathSpec val seed = Mnemonic.toSeed(SecretString.create(mnemonic), None) - val masterKey = ExtendedSecretKey.deriveMasterKey(seed) + val masterKey = ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation = false) val dp = DerivationPath.nextPath(IndexedSeq(masterKey), usePreEip3Derivation = false).get val sk = masterKey.derive(dp) val pk = sk.publicKey.key @@ -69,7 +69,7 @@ class DerivationPathSpec val seed = Mnemonic.toSeed(SecretString.create(mnemonic), None) - val masterKey = ExtendedSecretKey.deriveMasterKey(seed) + val masterKey = ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation = false) val dp = DerivationPath.nextPath(IndexedSeq(masterKey), usePreEip3Derivation = true).get val sk = masterKey.derive(dp) val pk = sk.publicKey.key @@ -106,7 +106,7 @@ class DerivationPathSpec val seed = Mnemonic.toSeed(SecretString.create(mnemonic), None) - val masterKey = ExtendedSecretKey.deriveMasterKey(seed) + val masterKey = ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation = false) P2PKAddress(masterKey.publicKey.key).toString() shouldBe address masterKey.path shouldBe masterKeyDerivation diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/ExtendedPublicKeySpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/ExtendedPublicKeySpec.scala index b768329f34..2cd869c66e 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/ExtendedPublicKeySpec.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/ExtendedPublicKeySpec.scala @@ -8,7 +8,7 @@ class ExtendedPublicKeySpec with ScalaCheckPropertyChecks with Generators { - val rootSecret: ExtendedSecretKey = ExtendedSecretKey.deriveMasterKey(seed) + val rootSecret: ExtendedSecretKey = ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation = false) property("public key tree derivation from seed (test vectors from BIP32 check)") { val expectedRoot = "kTV6HY41wXZVSqdpoe1heA8pBZFEN2oq5T59ZCMpqKKJ" diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKeySpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKeySpec.scala index b08da419a1..f3ff7f401a 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKeySpec.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKeySpec.scala @@ -29,7 +29,7 @@ class ExtendedSecretKeySpec ("DWMp3L9JZiywxSb5gSjc5dYxPwEZ6KkmasNiHD6VRcpJ", Index.hardIndex(2)) ) - val root = ExtendedSecretKey.deriveMasterKey(seed) + val root = ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation = false) equalBase58(root.keyBytes, expectedRoot) @@ -47,7 +47,7 @@ class ExtendedSecretKeySpec ("DWMp3L9JZiywxSb5gSjc5dYxPwEZ6KkmasNiHD6VRcpJ", "m/1/2/2'") ) - val root = ExtendedSecretKey.deriveMasterKey(seed) + val root = ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation = false) cases.foreach { case (expectedKey, path) => val derived = root.derive(DerivationPath.fromEncoded(path).get) @@ -55,25 +55,25 @@ class ExtendedSecretKeySpec } } - property("incorrect BIP32 key derivation (31 bit child key)") { + property("1627 BIP32 key derivation fix (31 bit child key)") { // see https://github.com/ergoplatform/ergo/issues/1627 for details val seedStr = "race relax argue hair sorry riot there spirit ready fetch food hedgehog hybrid mobile pretty" val seed: Array[Byte] = Mnemonic.toSeed(SecretString.create(seedStr)) - val cases = Seq( - ("9ewv8sxJ1jfr6j3WUSbGPMTVx3TZgcJKdnjKCbJWhiJp5U62uhP", "m/44'/429'/0'/0/0") - ) + val path = "m/44'/429'/0'/0/0" + val addressEncoder = ErgoAddressEncoder(ErgoAddressEncoder.MainnetNetworkPrefix) - val root = ExtendedSecretKey.deriveMasterKey(seed) + val pre1627DerivedSecretKey = ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation = true) + .derive(DerivationPath.fromEncoded(path).get) - val addressEncoder = ErgoAddressEncoder(ErgoAddressEncoder.MainnetNetworkPrefix) + P2PKAddress(pre1627DerivedSecretKey.publicKey.key)(addressEncoder).toString shouldEqual + "9ewv8sxJ1jfr6j3WUSbGPMTVx3TZgcJKdnjKCbJWhiJp5U62uhP" - cases.foreach { - case (expectedP2PK, path) => - val derived = root.derive(DerivationPath.fromEncoded(path).get) - val p2pk = P2PKAddress(derived.publicKey.key)(addressEncoder) - p2pk.toString shouldEqual expectedP2PK - } + val fixedDerivedSecretKey = ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation = false) + .derive(DerivationPath.fromEncoded(path).get) + + P2PKAddress(fixedDerivedSecretKey.publicKey.key)(addressEncoder).toString shouldEqual + "9eYMpbGgBf42bCcnB2nG3wQdqPzpCCw5eB1YaWUUen9uCaW3wwm" } } diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/JsonSecretStorageSpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/JsonSecretStorageSpec.scala index 3d9f4f5c79..6aed7975de 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/JsonSecretStorageSpec.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/JsonSecretStorageSpec.scala @@ -18,7 +18,7 @@ class JsonSecretStorageSpec forAll(entropyGen, passwordGen, encryptionSettingsGen) { (seed, pass, cryptoSettings) => val dir = createTempDir val settings = SecretStorageSettings(dir.getAbsolutePath, cryptoSettings) - val storage = JsonSecretStorage.init(seed, SecretString.create(pass))(settings) + val storage = JsonSecretStorage.init(seed, SecretString.create(pass), usePre1627KeyDerivation = false)(settings) storage.isLocked shouldBe true @@ -34,7 +34,7 @@ class JsonSecretStorageSpec forAll(entropyGen, passwordGen, encryptionSettingsGen) { (seed, pass, cryptoSettings) => val dir = createTempDir val settings = SecretStorageSettings(dir.getAbsolutePath, cryptoSettings) - val storage = JsonSecretStorage.init(seed, SecretString.create(pass))(settings) + val storage = JsonSecretStorage.init(seed, SecretString.create(pass), usePre1627KeyDerivation = false)(settings) storage.unlock(SecretString.create(pass)) diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/transactions/TransactionBuilderSpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/transactions/TransactionBuilderSpec.scala index ec94247038..3f41745308 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/transactions/TransactionBuilderSpec.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/transactions/TransactionBuilderSpec.scala @@ -29,7 +29,7 @@ class TransactionBuilderSpec extends WalletTestHelpers with Matchers { val seedStr = "edge talent poet tortoise trumpet dose" val seed: Array[Byte] = Mnemonic.toSeed(SecretString.create(seedStr)) - val rootSecret: ExtendedSecretKey = ExtendedSecretKey.deriveMasterKey(seed) + val rootSecret: ExtendedSecretKey = ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation = false) val currentHeight = 0 val minBoxValue = BoxSelector.MinBoxValue diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/Generators.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/Generators.scala index 8083d97a99..9fc73b1398 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/Generators.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/Generators.scala @@ -145,7 +145,7 @@ trait Generators { def extendedSecretGen: Gen[ExtendedSecretKey] = for { seed <- Gen.const(Constants.SecretKeyLength).map(scorex.utils.Random.randomBytes) - } yield ExtendedSecretKey.deriveMasterKey(seed) + } yield ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation = false) def extendedPubKeyGen: Gen[ExtendedPublicKey] = extendedSecretGen.map(_.publicKey) diff --git a/src/main/scala/org/ergoplatform/http/api/WalletApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/WalletApiRoute.scala index 256034fb42..816f84bca7 100644 --- a/src/main/scala/org/ergoplatform/http/api/WalletApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/WalletApiRoute.scala @@ -80,11 +80,13 @@ case class WalletApiRoute(readersHolder: ActorRef, .fold(_ => reject, s => provide(s)) } - private val restoreRequest: Directive1[(String, String, Option[String])] = entity(as[Json]).flatMap { p => + private val restoreRequest: Directive1[(Boolean, String, String, Option[String])] = entity(as[Json]).flatMap { p => p.hcursor.downField("pass").as[String] - .flatMap(pass => p.hcursor.downField("mnemonic").as[String] - .flatMap(mnemo => p.hcursor.downField("mnemonicPass").as[Option[String]] - .map(mnemoPassOpt => (pass, mnemo, mnemoPassOpt)) + .flatMap(usePre1627KeyDerivation => p.hcursor.downField("usePre1627KeyDerivation").as[Boolean] + .flatMap(pass => p.hcursor.downField("mnemonic").as[String] + .flatMap(mnemo => p.hcursor.downField("mnemonicPass").as[Option[String]] + .map(mnemoPassOpt => (pass, usePre1627KeyDerivation, mnemo, mnemoPassOpt)) + ) ) ) .fold(_ => reject, s => provide(s)) @@ -397,8 +399,8 @@ case class WalletApiRoute(readersHolder: ActorRef, } def restoreWalletR: Route = (path("restore") & post & restoreRequest) { - case (pass, mnemo, mnemoPassOpt) => - withWalletOp(_.restoreWallet(SecretString.create(pass), SecretString.create(mnemo), mnemoPassOpt.map(SecretString.create(_)))) { + case (usePre1627KeyDerivation, pass, mnemo, mnemoPassOpt) => + withWalletOp(_.restoreWallet(SecretString.create(pass), SecretString.create(mnemo), mnemoPassOpt.map(SecretString.create(_)), usePre1627KeyDerivation)) { _.fold( e => BadRequest(e.getMessage), _ => ApiResponse.toRoute(ApiResponse.OK) diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletActor.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletActor.scala index 6e95385e02..0e46e13372 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletActor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletActor.scala @@ -107,8 +107,8 @@ class ErgoWalletActor(settings: ErgoSettings, } // Restore wallet with mnemonic if secret is not set yet - case RestoreWallet(mnemonic, mnemonicPassOpt, walletPass) if !state.secretIsSet(settings.walletSettings.testMnemonic) => - ergoWalletService.restoreWallet(state, settings, mnemonic, mnemonicPassOpt, walletPass) match { + case RestoreWallet(mnemonic, mnemonicPassOpt, walletPass, usePre1627KeyDerivation) if !state.secretIsSet(settings.walletSettings.testMnemonic) => + ergoWalletService.restoreWallet(state, settings, mnemonic, mnemonicPassOpt, walletPass, usePre1627KeyDerivation) match { case Success(newState) => log.info("Wallet is restored") context.become(loadedWallet(newState)) @@ -635,7 +635,7 @@ object ErgoWalletActor extends ScorexLogging { * @param mnemonicPassOpt * @param walletPass */ - final case class RestoreWallet(mnemonic: SecretString, mnemonicPassOpt: Option[SecretString], walletPass: SecretString) + final case class RestoreWallet(mnemonic: SecretString, mnemonicPassOpt: Option[SecretString], walletPass: SecretString, usePre1627KeyDerivation: Boolean) /** * Unlock wallet with wallet password diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletReader.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletReader.scala index b0030b0304..37da56bb99 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletReader.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletReader.scala @@ -39,8 +39,8 @@ trait ErgoWalletReader extends NodeViewComponent { (walletActor ? InitWallet(pass, mnemonicPassOpt)).mapTo[Try[SecretString]] def restoreWallet(encryptionPass: SecretString, mnemonic: SecretString, - mnemonicPassOpt: Option[SecretString] = None): Future[Try[Unit]] = - (walletActor ? RestoreWallet(mnemonic, mnemonicPassOpt, encryptionPass)).mapTo[Try[Unit]] + mnemonicPassOpt: Option[SecretString] = None, usePre1627KeyDerivation: Boolean): Future[Try[Unit]] = + (walletActor ? RestoreWallet(mnemonic, mnemonicPassOpt, encryptionPass, usePre1627KeyDerivation)).mapTo[Try[Unit]] def unlockWallet(pass: SecretString): Future[Try[Unit]] = (walletActor ? UnlockWallet(pass)).mapTo[Try[Unit]] diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala index 4edd26fd6f..f742f71c51 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala @@ -66,13 +66,15 @@ trait ErgoWalletService { * @param mnemonic that was used to initialize wallet with * @param mnemonicPassOpt that was used to initialize wallet with * @param walletPass that was used to initialize wallet with + * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation (see https://github.com/ergoplatform/ergo/issues/1627 for details) * @return new wallet state */ def restoreWallet(state: ErgoWalletState, settings: ErgoSettings, mnemonic: SecretString, mnemonicPassOpt: Option[SecretString], - walletPass: SecretString): Try[ErgoWalletState] + walletPass: SecretString, + usePre1627KeyDerivation: Boolean): Try[ErgoWalletState] /** * Decrypt underlying encrypted storage using `walletPass` and update public keys @@ -292,7 +294,7 @@ class ErgoWalletServiceImpl(override val ergoSettings: ErgoSettings) extends Erg log.info("Initializing wallet") def initStorage(mnemonic: SecretString): Try[JsonSecretStorage] = - Try(JsonSecretStorage.init(Mnemonic.toSeed(mnemonic, mnemonicPassOpt), walletPass)(walletSettings.secretStorage)) + Try(JsonSecretStorage.init(Mnemonic.toSeed(mnemonic, mnemonicPassOpt), walletPass, usePre1627KeyDerivation = false)(walletSettings.secretStorage)) val result = new Mnemonic(walletSettings.mnemonicPhraseLanguage, walletSettings.seedStrengthBits) @@ -316,11 +318,12 @@ class ErgoWalletServiceImpl(override val ergoSettings: ErgoSettings) extends Erg settings: ErgoSettings, mnemonic: SecretString, mnemonicPassOpt: Option[SecretString], - walletPass: SecretString): Try[ErgoWalletState] = + walletPass: SecretString, + usePre1627KeyDerivation: Boolean): Try[ErgoWalletState] = if (settings.nodeSettings.isFullBlocksPruned) Failure(new IllegalArgumentException("Unable to restore wallet when pruning is enabled")) else - Try(JsonSecretStorage.restore(mnemonic, mnemonicPassOpt, walletPass, settings.walletSettings.secretStorage)) + Try(JsonSecretStorage.restore(mnemonic, mnemonicPassOpt, walletPass, settings.walletSettings.secretStorage, usePre1627KeyDerivation)) .flatMap { secretStorage => // remove old wallet state, see https://github.com/ergoplatform/ergo/issues/1313 recreateRegistry(state, settings).flatMap { stateV1 => diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala index 23c77cf8f1..5b94f72c72 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala @@ -37,7 +37,7 @@ trait ErgoWalletSupport extends ScorexLogging { def buildProverFromMnemonic(mnemonic: SecretString, keysQty: Option[Int], parameters: Parameters): ErgoProvingInterpreter = { val seed = Mnemonic.toSeed(mnemonic) - val rootSk = ExtendedSecretKey.deriveMasterKey(seed) + val rootSk = ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation = false) val childSks = keysQty.toIndexedSeq.flatMap(x => (0 until x).map(rootSk.child)) ErgoProvingInterpreter(rootSk +: childSks, parameters) } diff --git a/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala index 5c08932ea3..5dd094a619 100644 --- a/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala @@ -136,7 +136,8 @@ class WalletApiRouteSpec extends AnyFlatSpec } it should "restore wallet" in { - Post(prefix + "/restore", Json.obj("pass" -> "1234".asJson, "mnemonic" -> WalletActorStub.mnemonic.asJson)) ~> + Post(prefix + "/restore", Json.obj("pass" -> "1234".asJson, "mnemonic" -> WalletActorStub.mnemonic.asJson, + "usePre1627KeyDerivation" -> false.asJson)) ~> route ~> check(status shouldBe StatusCodes.OK) } diff --git a/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala b/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala index 48f8a5e942..b03fe5c179 100644 --- a/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala @@ -44,7 +44,7 @@ class ErgoWalletServiceSpec private implicit val x: WalletFixture = new WalletFixture(settings, parameters, getCurrentView(_).vault) implicit override val generatorDrivenConfig: PropertyCheckConfiguration = PropertyCheckConfiguration(minSuccessful = 4, sizeRange = 4) private lazy val pks = getPublicKeys.toList - private val masterKey = ExtendedSecretKey.deriveMasterKey(Mnemonic.toSeed(SecretString.create("edge talent poet tortoise trumpet dose"))) + private val masterKey = ExtendedSecretKey.deriveMasterKey(Mnemonic.toSeed(SecretString.create("edge talent poet tortoise trumpet dose")), usePre1627KeyDerivation = false) override def afterAll(): Unit = try super.afterAll() finally x.stop() @@ -76,7 +76,8 @@ class ErgoWalletServiceSpec settingsWithPruning, mnemonic = SecretString.create("x"), mnemonicPassOpt = None, - walletPass = SecretString.create("y") + walletPass = SecretString.create("y"), + usePre1627KeyDerivation = false ).failed.get.getMessage shouldBe "Unable to restore wallet when pruning is enabled" } } diff --git a/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletRegistryBenchmark.scala b/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletRegistryBenchmark.scala index 15313c65f8..db0f19b76e 100644 --- a/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletRegistryBenchmark.scala +++ b/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletRegistryBenchmark.scala @@ -36,7 +36,7 @@ object WalletRegistryBenchmark extends App with ErgoTestConstants { val registry = WalletRegistry(settings).get val storage = WalletStorage.readOrCreate(settings) - val rootSecret = ExtendedSecretKey.deriveMasterKey(Array.fill(32)(0: Byte)) + val rootSecret = ExtendedSecretKey.deriveMasterKey(Array.fill(32)(0: Byte), usePre1627KeyDerivation = false) val derivedSecrets = (1 to 15000).map { i => val k = rootSecret.derive(DerivationPath.fromEncoded(s"m/44'/429'/0'/0/$i").get) diff --git a/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala b/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala index db4e4a547e..45b2f37391 100644 --- a/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala +++ b/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala @@ -66,7 +66,7 @@ trait ErgoTestConstants extends ScorexLogging { val emptyProverResult: ProverResult = ProverResult(Array.emptyByteArray, ContextExtension.empty) lazy val defaultSeed: Array[Byte] = Mnemonic.toSeed(settings.walletSettings.testMnemonic.fold[SecretString](SecretString.empty())(SecretString.create(_))) - val defaultRootSecret: ExtendedSecretKey = ExtendedSecretKey.deriveMasterKey(defaultSeed) + val defaultRootSecret: ExtendedSecretKey = ExtendedSecretKey.deriveMasterKey(defaultSeed, usePre1627KeyDerivation = false) val defaultChildSecrets: IndexedSeq[ExtendedSecretKey] = settings.walletSettings.testKeysQty .toIndexedSeq .flatMap(x => (0 until x).map(defaultRootSecret.child)) diff --git a/src/test/scala/org/ergoplatform/utils/Stubs.scala b/src/test/scala/org/ergoplatform/utils/Stubs.scala index c7d0139081..b821755551 100644 --- a/src/test/scala/org/ergoplatform/utils/Stubs.scala +++ b/src/test/scala/org/ergoplatform/utils/Stubs.scala @@ -263,7 +263,7 @@ trait Stubs extends ErgoGenerators with ErgoTestHelpers with ChainGenerator with object WalletActorStub { val mnemonic = "abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon" val path = DerivationPath(List(0, 1, 2), publicBranch = false) - val secretKey = ExtendedSecretKey.deriveMasterKey(Mnemonic.toSeed(SecretString.create(mnemonic))).derive(path) + val secretKey = ExtendedSecretKey.deriveMasterKey(Mnemonic.toSeed(SecretString.create(mnemonic)), usePre1627KeyDerivation = false).derive(path) val address = P2PKAddress(proveDlogGen.sample.get) val walletBoxN_N: WalletBox = WalletBox( From fc50861207d07e4d059acc78a095047949a39a95 Mon Sep 17 00:00:00 2001 From: Denys Zadorozhnyi Date: Mon, 14 Feb 2022 17:53:18 +0200 Subject: [PATCH 04/21] update openapi.yaml with added usePre1627KeyDerivation option for wallet restore endpoint; --- src/main/resources/api/openapi.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index 187e9da69f..9b67ab3b56 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -1560,6 +1560,7 @@ components: required: - pass - mnemonic + - usePre1627KeyDerivation properties: pass: type: string @@ -1570,6 +1571,9 @@ components: mnemonicPass: type: string description: Optional pass to password-protect mnemonic seed + usePre1627KeyDerivation: + type: boolean + description: use incorrect(previous) BIP32 key derivation (see https://github.com/ergoplatform/ergo/issues/1627 for details). It's recommended to set to 'true' if the original wallet was created by ergo node before v4.0.24. CheckWallet: type: object From f6a0f9f238dfdba3ac1ab48663e618924820a664 Mon Sep 17 00:00:00 2001 From: Denys Zadorozhnyi Date: Tue, 15 Feb 2022 09:52:49 +0200 Subject: [PATCH 05/21] fix comment; --- .../ergoplatform/wallet/secrets/ExtendedSecretKey.scala | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKey.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKey.scala index 03126c3a04..acf3e3ba84 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKey.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKey.scala @@ -56,10 +56,6 @@ final class ExtendedSecretKey(private[secrets] val keyBytes: Array[Byte], object ExtendedSecretKey { - // For some seeds 'parentKey.keyBytes' is less than 32 bytes while BIP32 requires 32 bytes. - // The reason for this is childKey being small(ish) and 'BigIntegers.asUnsignedByteArray(childKey)' - // is less than 32 bytes while BIP32 requires 32 bytes. - // see https://github.com/ergoplatform/ergo/issues/1627 for details @scala.annotation.tailrec def deriveChildSecretKey(parentKey: ExtendedSecretKey, idx: Int): ExtendedSecretKey = { val keyCoded: Array[Byte] = @@ -76,7 +72,8 @@ object ExtendedSecretKey { deriveChildSecretKey(parentKey, idx + 1) else { val keyBytes = if (parentKey.usePre1627KeyDerivation) { - // maybe less than 32 bytes if childKey is small enough + // maybe less than 32 bytes if childKey is small enough while BIP32 requires 32 bytes. + // see https://github.com/ergoplatform/ergo/issues/1627 for details BigIntegers.asUnsignedByteArray(childKey) } else { // padded with leading zeroes to 32 bytes From 9e0c5ea185a9ce69279c7f1d619a0c1b00cc9eef Mon Sep 17 00:00:00 2001 From: Denys Zadorozhnyi Date: Tue, 15 Feb 2022 12:11:17 +0200 Subject: [PATCH 06/21] add WalletApiRoute test for missing key derivation option rejection; --- .../scala/org/ergoplatform/http/api/WalletApiRoute.scala | 3 ++- .../org/ergoplatform/http/routes/WalletApiRouteSpec.scala | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/scala/org/ergoplatform/http/api/WalletApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/WalletApiRoute.scala index 816f84bca7..b8a4394862 100644 --- a/src/main/scala/org/ergoplatform/http/api/WalletApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/WalletApiRoute.scala @@ -24,6 +24,7 @@ import scorex.util.encode.Base16 import scala.concurrent.Future import scala.concurrent.duration._ import scala.util.{Failure, Success, Try} +import akka.http.scaladsl.server.MissingQueryParamRejection case class WalletApiRoute(readersHolder: ActorRef, nodeViewActorRef: ActorRef, @@ -89,7 +90,7 @@ case class WalletApiRoute(readersHolder: ActorRef, ) ) ) - .fold(_ => reject, s => provide(s)) + .fold(e => reject(MissingQueryParamRejection(e.toString())), s => provide(s)) } private val checkRequest: Directive1[(String, Option[String])] = entity(as[Json]).flatMap { p => diff --git a/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala index 5dd094a619..df998daf35 100644 --- a/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala @@ -20,6 +20,7 @@ import org.scalatest.matchers.should.Matchers import scala.util.{Random, Try} import scala.concurrent.duration._ +import akka.http.scaladsl.server.MissingQueryParamRejection class WalletApiRouteSpec extends AnyFlatSpec with Matchers @@ -141,6 +142,10 @@ class WalletApiRouteSpec extends AnyFlatSpec route ~> check(status shouldBe StatusCodes.OK) } + it should "not restore wallet without key derivation method specified" in { + Post(prefix + "/restore", Json.obj("pass" -> "1234".asJson, "mnemonic" -> WalletActorStub.mnemonic.asJson)) ~> + route ~> check(rejection shouldBe a[MissingQueryParamRejection]) + } it should "unlock wallet" in { Post(prefix + "/unlock", Json.obj("pass" -> "1234".asJson)) ~> route ~> check { From 5a0756add25cde3c76c08225b9b8482209695afb Mon Sep 17 00:00:00 2001 From: Denys Zadorozhnyi Date: Tue, 15 Feb 2022 12:55:22 +0200 Subject: [PATCH 07/21] add usePre1627KeyDerivation tests in JsonSecretStorage for new, restored and pre-1627(migration) wallets; --- .../secrets/JsonSecretStorageSpec.scala | 55 ++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/JsonSecretStorageSpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/JsonSecretStorageSpec.scala index 6aed7975de..50e473e284 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/JsonSecretStorageSpec.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/JsonSecretStorageSpec.scala @@ -6,6 +6,10 @@ import org.scalatest.matchers.should.Matchers import org.scalatest.propspec.AnyPropSpec import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks import org.ergoplatform.wallet.interface4j.SecretString +import org.scalacheck.Arbitrary +import org.ergoplatform.wallet.settings.EncryptionSettings +import java.io.{File, PrintWriter} +import java.util.UUID class JsonSecretStorageSpec extends AnyPropSpec @@ -15,10 +19,11 @@ class JsonSecretStorageSpec with FileUtils { property("initialization and unlock") { - forAll(entropyGen, passwordGen, encryptionSettingsGen) { (seed, pass, cryptoSettings) => + forAll(entropyGen, passwordGen, encryptionSettingsGen, Arbitrary.arbBool.arbitrary) { + (seed, pass, cryptoSettings, usePre1627KeyDerivation) => val dir = createTempDir val settings = SecretStorageSettings(dir.getAbsolutePath, cryptoSettings) - val storage = JsonSecretStorage.init(seed, SecretString.create(pass), usePre1627KeyDerivation = false)(settings) + val storage = JsonSecretStorage.init(seed, SecretString.create(pass), usePre1627KeyDerivation)(settings) storage.isLocked shouldBe true @@ -27,6 +32,9 @@ class JsonSecretStorageSpec unlockTry shouldBe 'success storage.isLocked shouldBe false + + // wallet should use explicitly specified BIP32 key derivation + storage.secret.get.usePre1627KeyDerivation shouldBe usePre1627KeyDerivation } } @@ -50,4 +58,47 @@ class JsonSecretStorageSpec } } + property("restore from mnemonic") { + forAll(mnemonicGen, passwordGen, encryptionSettingsGen, Arbitrary.arbBool.arbitrary) { + (mnemonic, pass, cryptoSettings, usePre1627KeyDerivation) => + val dir = createTempDir + val settings = SecretStorageSettings(dir.getAbsolutePath, cryptoSettings) + val storage = JsonSecretStorage.restore(SecretString.create(mnemonic.toString()), + mnemonicPassOpt = None, + encryptionPass = SecretString.create(pass), + settings = settings, + usePre1627KeyDerivation) + + val _ = storage.unlock(SecretString.create(pass)) + + // wallet should use explicitly specified BIP32 key derivation + storage.secret.get.usePre1627KeyDerivation shouldBe usePre1627KeyDerivation + } + } + + property("pre 1627 key derivation is used for loaded storage with missing usePre1627KeyDerivation") { + val dir = createTempDir + // mock JSON file without usePre1627KeyDerivation property set, simulating pre 1627 wallet file + val jsonFileRaw = """ + {"cipherText":"e134f488c52a87ccb0287fdd164bdb3b67f04c33ba94eab169e802df7a082addd434e1f36dccbf5362f97a9e57ef97879807bdc632072fb2b3ae9a9b08a6caf6","salt":"988e9d31c675bf6012c235e4c238f22649285140544b17600e2887f655b74ae7","iv":"dff8c6b120cdfaac4192e9c1","authTag":"de1c6443808263b749f4c29f146208ba","cipherParams":{"prf":"HmacSHA1","c":7,"dkLen":256}} + """ + val pass = List("N", "m").toString() + val encryptionSettings = EncryptionSettings("HmacSHA1", 7, 256) + + new File(dir.getAbsolutePath()).mkdirs() + val uuid = UUID.randomUUID() + val file = new File(s"${dir}/$uuid.json") + val outWriter = new PrintWriter(file) + outWriter.write(jsonFileRaw) + outWriter.close() + + val settings = SecretStorageSettings(dir.getAbsolutePath, encryptionSettings) + val storage = JsonSecretStorage.readFile(settings).get + + val _ = storage.unlock(SecretString.create(pass)) + + // loaded wallet should use pre 1627 key derivation + storage.secret.get.usePre1627KeyDerivation shouldBe true + } + } From 4d4897956b81a6b51a558229817a1eddfcb93506 Mon Sep 17 00:00:00 2001 From: Denys Zadorozhnyi Date: Tue, 15 Feb 2022 15:13:48 +0200 Subject: [PATCH 08/21] fix doc comments; --- .../org/ergoplatform/wallet/secrets/JsonSecretStorage.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/JsonSecretStorage.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/JsonSecretStorage.scala index 38e0fda512..4bdfe2e7cb 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/JsonSecretStorage.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/JsonSecretStorage.scala @@ -92,7 +92,7 @@ object JsonSecretStorage { * Initializes storage instance with new wallet file encrypted with the given `pass`. * @param seed - seed bytes * @param pass - encryption password - * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation (see https://github.com/ergoplatform/ergo/issues/1627 for details) + * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation, expected to be true for new wallets, and false for old pre-1627 wallets (see https://github.com/ergoplatform/ergo/issues/1627 for details) */ def init(seed: Array[Byte], pass: SecretString, usePre1627KeyDerivation: Boolean)(settings: SecretStorageSettings): JsonSecretStorage = { val iv = scorex.utils.Random.randomBytes(crypto.AES.NonceBitsLen / 8) @@ -119,7 +119,7 @@ object JsonSecretStorage { * @param mnemonic - mnemonic phase * @param mnemonicPassOpt - optional mnemonic password * @param encryptionPass - encryption password - * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation (see https://github.com/ergoplatform/ergo/issues/1627 for details) + * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation, expected to be true for new wallets, and false for old pre-1627 wallets (see https://github.com/ergoplatform/ergo/issues/1627 for details) */ def restore(mnemonic: SecretString, mnemonicPassOpt: Option[SecretString], From 380384c4da2b389031232caf0e25002332160892 Mon Sep 17 00:00:00 2001 From: Denys Zadorozhnyi Date: Tue, 15 Feb 2022 18:23:20 +0200 Subject: [PATCH 09/21] fix doc comments; --- .../org/ergoplatform/wallet/secrets/EncryptedSecret.scala | 2 +- .../org/ergoplatform/wallet/secrets/ExtendedSecretKey.scala | 2 +- .../org/ergoplatform/wallet/secrets/JsonSecretStorage.scala | 4 ++-- .../org/ergoplatform/nodeView/wallet/ErgoWalletService.scala | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/EncryptedSecret.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/EncryptedSecret.scala index d7d1f7f7c8..54b0ebca7a 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/EncryptedSecret.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/EncryptedSecret.scala @@ -13,7 +13,7 @@ import scorex.util.encode.Base16 * @param iv - cipher initialization vector * @param authTag - message authentication tag * @param cipherParams - cipher params - * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation (see https://github.com/ergoplatform/ergo/issues/1627 for details) + * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation, expected to be false for new wallets, and true for old pre-1627 wallets (see https://github.com/ergoplatform/ergo/issues/1627 for details) */ final case class EncryptedSecret(cipherText: String, salt: String, iv: String, authTag: String, cipherParams: EncryptionSettings, usePre1627KeyDerivation: Option[Boolean]) diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKey.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKey.scala index acf3e3ba84..35825a0800 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKey.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKey.scala @@ -94,7 +94,7 @@ object ExtendedSecretKey { /** * Derives master secret key from the seed * @param seed - seed bytes - * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation (see https://github.com/ergoplatform/ergo/issues/1627 for details) + * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation, expected to be false for new wallets, and true for old pre-1627 wallets (see https://github.com/ergoplatform/ergo/issues/1627 for details) */ def deriveMasterKey(seed: Array[Byte], usePre1627KeyDerivation: Boolean): ExtendedSecretKey = { val (masterKey, chainCode) = HmacSHA512.hash(Constants.BitcoinSeed, seed).splitAt(Constants.SecretKeyLength) diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/JsonSecretStorage.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/JsonSecretStorage.scala index 4bdfe2e7cb..566907d26f 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/JsonSecretStorage.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/JsonSecretStorage.scala @@ -92,7 +92,7 @@ object JsonSecretStorage { * Initializes storage instance with new wallet file encrypted with the given `pass`. * @param seed - seed bytes * @param pass - encryption password - * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation, expected to be true for new wallets, and false for old pre-1627 wallets (see https://github.com/ergoplatform/ergo/issues/1627 for details) + * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation, expected to be false for new wallets, and true for old pre-1627 wallets (see https://github.com/ergoplatform/ergo/issues/1627 for details) */ def init(seed: Array[Byte], pass: SecretString, usePre1627KeyDerivation: Boolean)(settings: SecretStorageSettings): JsonSecretStorage = { val iv = scorex.utils.Random.randomBytes(crypto.AES.NonceBitsLen / 8) @@ -119,7 +119,7 @@ object JsonSecretStorage { * @param mnemonic - mnemonic phase * @param mnemonicPassOpt - optional mnemonic password * @param encryptionPass - encryption password - * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation, expected to be true for new wallets, and false for old pre-1627 wallets (see https://github.com/ergoplatform/ergo/issues/1627 for details) + * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation, expected to be false for new wallets, and true for old pre-1627 wallets (see https://github.com/ergoplatform/ergo/issues/1627 for details) */ def restore(mnemonic: SecretString, mnemonicPassOpt: Option[SecretString], diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala index f742f71c51..1b70118977 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala @@ -66,7 +66,7 @@ trait ErgoWalletService { * @param mnemonic that was used to initialize wallet with * @param mnemonicPassOpt that was used to initialize wallet with * @param walletPass that was used to initialize wallet with - * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation (see https://github.com/ergoplatform/ergo/issues/1627 for details) + * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation, expected to be false for new wallets, and true for old pre-1627 wallets (see https://github.com/ergoplatform/ergo/issues/1627 for details) * @return new wallet state */ def restoreWallet(state: ErgoWalletState, From 32ff3ef67995b7b9286f4e8d92935279a7ff9d82 Mon Sep 17 00:00:00 2001 From: Denys Zadorozhnyi Date: Thu, 22 Sep 2022 09:48:30 +0300 Subject: [PATCH 10/21] fix CreateTransactionDemo build; --- .../java/org/ergoplatform/wallet/CreateTransactionDemo.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ergo-wallet/src/test/java/org/ergoplatform/wallet/CreateTransactionDemo.java b/ergo-wallet/src/test/java/org/ergoplatform/wallet/CreateTransactionDemo.java index 258b0f4f57..a8f38ba40f 100644 --- a/ergo-wallet/src/test/java/org/ergoplatform/wallet/CreateTransactionDemo.java +++ b/ergo-wallet/src/test/java/org/ergoplatform/wallet/CreateTransactionDemo.java @@ -73,7 +73,7 @@ public static void createMultiPaymentTransaction() { // Create an address byte[] entropy = Random.randomBytes(32); - ExtendedSecretKey extendedSecretKey = ExtendedSecretKey.deriveMasterKey(entropy); + ExtendedSecretKey extendedSecretKey = ExtendedSecretKey.deriveMasterKey(entropy, false); ErgoAddress myAddress = P2PKAddress.apply(extendedSecretKey.privateInput().publicImage(), encoder); int feeAmt = 1000000; // minimal fee amount From 71f62a9f9aa376c0eb2c04911b529ade4e9f51b4 Mon Sep 17 00:00:00 2001 From: Denys Zadorozhnyi Date: Thu, 22 Sep 2022 11:12:48 +0300 Subject: [PATCH 11/21] Use pre-1627 derivation in ergo node; --- src/main/resources/api/openapi.yaml | 4 ---- .../ergoplatform/http/api/WalletApiRoute.scala | 17 +++++++---------- .../nodeView/wallet/ErgoWalletActor.scala | 6 +++--- .../nodeView/wallet/ErgoWalletReader.scala | 4 ++-- .../nodeView/wallet/ErgoWalletService.scala | 11 ++++------- .../nodeView/wallet/ErgoWalletSupport.scala | 2 +- .../http/routes/WalletApiRouteSpec.scala | 9 +-------- .../nodeView/wallet/ErgoWalletServiceSpec.scala | 5 ++--- .../ergoplatform/utils/ErgoTestConstants.scala | 2 +- .../scala/org/ergoplatform/utils/Stubs.scala | 2 +- 10 files changed, 22 insertions(+), 40 deletions(-) diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index 9b67ab3b56..187e9da69f 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -1560,7 +1560,6 @@ components: required: - pass - mnemonic - - usePre1627KeyDerivation properties: pass: type: string @@ -1571,9 +1570,6 @@ components: mnemonicPass: type: string description: Optional pass to password-protect mnemonic seed - usePre1627KeyDerivation: - type: boolean - description: use incorrect(previous) BIP32 key derivation (see https://github.com/ergoplatform/ergo/issues/1627 for details). It's recommended to set to 'true' if the original wallet was created by ergo node before v4.0.24. CheckWallet: type: object diff --git a/src/main/scala/org/ergoplatform/http/api/WalletApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/WalletApiRoute.scala index b8a4394862..f432fe9d1a 100644 --- a/src/main/scala/org/ergoplatform/http/api/WalletApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/WalletApiRoute.scala @@ -24,7 +24,6 @@ import scorex.util.encode.Base16 import scala.concurrent.Future import scala.concurrent.duration._ import scala.util.{Failure, Success, Try} -import akka.http.scaladsl.server.MissingQueryParamRejection case class WalletApiRoute(readersHolder: ActorRef, nodeViewActorRef: ActorRef, @@ -81,16 +80,14 @@ case class WalletApiRoute(readersHolder: ActorRef, .fold(_ => reject, s => provide(s)) } - private val restoreRequest: Directive1[(Boolean, String, String, Option[String])] = entity(as[Json]).flatMap { p => + private val restoreRequest: Directive1[(String, String, Option[String])] = entity(as[Json]).flatMap { p => p.hcursor.downField("pass").as[String] - .flatMap(usePre1627KeyDerivation => p.hcursor.downField("usePre1627KeyDerivation").as[Boolean] - .flatMap(pass => p.hcursor.downField("mnemonic").as[String] - .flatMap(mnemo => p.hcursor.downField("mnemonicPass").as[Option[String]] - .map(mnemoPassOpt => (pass, usePre1627KeyDerivation, mnemo, mnemoPassOpt)) - ) + .flatMap(pass => p.hcursor.downField("mnemonic").as[String] + .flatMap(mnemo => p.hcursor.downField("mnemonicPass").as[Option[String]] + .map(mnemoPassOpt => (pass, mnemo, mnemoPassOpt)) ) ) - .fold(e => reject(MissingQueryParamRejection(e.toString())), s => provide(s)) + .fold(e => reject, s => provide(s)) } private val checkRequest: Directive1[(String, Option[String])] = entity(as[Json]).flatMap { p => @@ -400,8 +397,8 @@ case class WalletApiRoute(readersHolder: ActorRef, } def restoreWalletR: Route = (path("restore") & post & restoreRequest) { - case (usePre1627KeyDerivation, pass, mnemo, mnemoPassOpt) => - withWalletOp(_.restoreWallet(SecretString.create(pass), SecretString.create(mnemo), mnemoPassOpt.map(SecretString.create(_)), usePre1627KeyDerivation)) { + case (pass, mnemo, mnemoPassOpt) => + withWalletOp(_.restoreWallet(SecretString.create(pass), SecretString.create(mnemo), mnemoPassOpt.map(SecretString.create(_)))) { _.fold( e => BadRequest(e.getMessage), _ => ApiResponse.toRoute(ApiResponse.OK) diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletActor.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletActor.scala index 0e46e13372..6e95385e02 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletActor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletActor.scala @@ -107,8 +107,8 @@ class ErgoWalletActor(settings: ErgoSettings, } // Restore wallet with mnemonic if secret is not set yet - case RestoreWallet(mnemonic, mnemonicPassOpt, walletPass, usePre1627KeyDerivation) if !state.secretIsSet(settings.walletSettings.testMnemonic) => - ergoWalletService.restoreWallet(state, settings, mnemonic, mnemonicPassOpt, walletPass, usePre1627KeyDerivation) match { + case RestoreWallet(mnemonic, mnemonicPassOpt, walletPass) if !state.secretIsSet(settings.walletSettings.testMnemonic) => + ergoWalletService.restoreWallet(state, settings, mnemonic, mnemonicPassOpt, walletPass) match { case Success(newState) => log.info("Wallet is restored") context.become(loadedWallet(newState)) @@ -635,7 +635,7 @@ object ErgoWalletActor extends ScorexLogging { * @param mnemonicPassOpt * @param walletPass */ - final case class RestoreWallet(mnemonic: SecretString, mnemonicPassOpt: Option[SecretString], walletPass: SecretString, usePre1627KeyDerivation: Boolean) + final case class RestoreWallet(mnemonic: SecretString, mnemonicPassOpt: Option[SecretString], walletPass: SecretString) /** * Unlock wallet with wallet password diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletReader.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletReader.scala index 37da56bb99..b0030b0304 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletReader.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletReader.scala @@ -39,8 +39,8 @@ trait ErgoWalletReader extends NodeViewComponent { (walletActor ? InitWallet(pass, mnemonicPassOpt)).mapTo[Try[SecretString]] def restoreWallet(encryptionPass: SecretString, mnemonic: SecretString, - mnemonicPassOpt: Option[SecretString] = None, usePre1627KeyDerivation: Boolean): Future[Try[Unit]] = - (walletActor ? RestoreWallet(mnemonic, mnemonicPassOpt, encryptionPass, usePre1627KeyDerivation)).mapTo[Try[Unit]] + mnemonicPassOpt: Option[SecretString] = None): Future[Try[Unit]] = + (walletActor ? RestoreWallet(mnemonic, mnemonicPassOpt, encryptionPass)).mapTo[Try[Unit]] def unlockWallet(pass: SecretString): Future[Try[Unit]] = (walletActor ? UnlockWallet(pass)).mapTo[Try[Unit]] diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala index 1b70118977..76eef9cfbe 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala @@ -66,15 +66,13 @@ trait ErgoWalletService { * @param mnemonic that was used to initialize wallet with * @param mnemonicPassOpt that was used to initialize wallet with * @param walletPass that was used to initialize wallet with - * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation, expected to be false for new wallets, and true for old pre-1627 wallets (see https://github.com/ergoplatform/ergo/issues/1627 for details) * @return new wallet state */ def restoreWallet(state: ErgoWalletState, settings: ErgoSettings, mnemonic: SecretString, mnemonicPassOpt: Option[SecretString], - walletPass: SecretString, - usePre1627KeyDerivation: Boolean): Try[ErgoWalletState] + walletPass: SecretString): Try[ErgoWalletState] /** * Decrypt underlying encrypted storage using `walletPass` and update public keys @@ -294,7 +292,7 @@ class ErgoWalletServiceImpl(override val ergoSettings: ErgoSettings) extends Erg log.info("Initializing wallet") def initStorage(mnemonic: SecretString): Try[JsonSecretStorage] = - Try(JsonSecretStorage.init(Mnemonic.toSeed(mnemonic, mnemonicPassOpt), walletPass, usePre1627KeyDerivation = false)(walletSettings.secretStorage)) + Try(JsonSecretStorage.init(Mnemonic.toSeed(mnemonic, mnemonicPassOpt), walletPass, usePre1627KeyDerivation = true)(walletSettings.secretStorage)) val result = new Mnemonic(walletSettings.mnemonicPhraseLanguage, walletSettings.seedStrengthBits) @@ -318,12 +316,11 @@ class ErgoWalletServiceImpl(override val ergoSettings: ErgoSettings) extends Erg settings: ErgoSettings, mnemonic: SecretString, mnemonicPassOpt: Option[SecretString], - walletPass: SecretString, - usePre1627KeyDerivation: Boolean): Try[ErgoWalletState] = + walletPass: SecretString): Try[ErgoWalletState] = if (settings.nodeSettings.isFullBlocksPruned) Failure(new IllegalArgumentException("Unable to restore wallet when pruning is enabled")) else - Try(JsonSecretStorage.restore(mnemonic, mnemonicPassOpt, walletPass, settings.walletSettings.secretStorage, usePre1627KeyDerivation)) + Try(JsonSecretStorage.restore(mnemonic, mnemonicPassOpt, walletPass, settings.walletSettings.secretStorage, usePre1627KeyDerivation = true)) .flatMap { secretStorage => // remove old wallet state, see https://github.com/ergoplatform/ergo/issues/1313 recreateRegistry(state, settings).flatMap { stateV1 => diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala index 5b94f72c72..4ab529a426 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala @@ -37,7 +37,7 @@ trait ErgoWalletSupport extends ScorexLogging { def buildProverFromMnemonic(mnemonic: SecretString, keysQty: Option[Int], parameters: Parameters): ErgoProvingInterpreter = { val seed = Mnemonic.toSeed(mnemonic) - val rootSk = ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation = false) + val rootSk = ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation = true) val childSks = keysQty.toIndexedSeq.flatMap(x => (0 until x).map(rootSk.child)) ErgoProvingInterpreter(rootSk +: childSks, parameters) } diff --git a/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala index df998daf35..98458156c2 100644 --- a/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala @@ -20,7 +20,6 @@ import org.scalatest.matchers.should.Matchers import scala.util.{Random, Try} import scala.concurrent.duration._ -import akka.http.scaladsl.server.MissingQueryParamRejection class WalletApiRouteSpec extends AnyFlatSpec with Matchers @@ -137,14 +136,8 @@ class WalletApiRouteSpec extends AnyFlatSpec } it should "restore wallet" in { - Post(prefix + "/restore", Json.obj("pass" -> "1234".asJson, "mnemonic" -> WalletActorStub.mnemonic.asJson, - "usePre1627KeyDerivation" -> false.asJson)) ~> - route ~> check(status shouldBe StatusCodes.OK) - } - - it should "not restore wallet without key derivation method specified" in { Post(prefix + "/restore", Json.obj("pass" -> "1234".asJson, "mnemonic" -> WalletActorStub.mnemonic.asJson)) ~> - route ~> check(rejection shouldBe a[MissingQueryParamRejection]) + route ~> check(status shouldBe StatusCodes.OK) } it should "unlock wallet" in { diff --git a/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala b/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala index b03fe5c179..d712f38c5d 100644 --- a/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala @@ -44,7 +44,7 @@ class ErgoWalletServiceSpec private implicit val x: WalletFixture = new WalletFixture(settings, parameters, getCurrentView(_).vault) implicit override val generatorDrivenConfig: PropertyCheckConfiguration = PropertyCheckConfiguration(minSuccessful = 4, sizeRange = 4) private lazy val pks = getPublicKeys.toList - private val masterKey = ExtendedSecretKey.deriveMasterKey(Mnemonic.toSeed(SecretString.create("edge talent poet tortoise trumpet dose")), usePre1627KeyDerivation = false) + private val masterKey = ExtendedSecretKey.deriveMasterKey(Mnemonic.toSeed(SecretString.create("edge talent poet tortoise trumpet dose")), usePre1627KeyDerivation = true) override def afterAll(): Unit = try super.afterAll() finally x.stop() @@ -76,8 +76,7 @@ class ErgoWalletServiceSpec settingsWithPruning, mnemonic = SecretString.create("x"), mnemonicPassOpt = None, - walletPass = SecretString.create("y"), - usePre1627KeyDerivation = false + walletPass = SecretString.create("y") ).failed.get.getMessage shouldBe "Unable to restore wallet when pruning is enabled" } } diff --git a/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala b/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala index 45b2f37391..623db7041f 100644 --- a/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala +++ b/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala @@ -66,7 +66,7 @@ trait ErgoTestConstants extends ScorexLogging { val emptyProverResult: ProverResult = ProverResult(Array.emptyByteArray, ContextExtension.empty) lazy val defaultSeed: Array[Byte] = Mnemonic.toSeed(settings.walletSettings.testMnemonic.fold[SecretString](SecretString.empty())(SecretString.create(_))) - val defaultRootSecret: ExtendedSecretKey = ExtendedSecretKey.deriveMasterKey(defaultSeed, usePre1627KeyDerivation = false) + val defaultRootSecret: ExtendedSecretKey = ExtendedSecretKey.deriveMasterKey(defaultSeed, usePre1627KeyDerivation = true) val defaultChildSecrets: IndexedSeq[ExtendedSecretKey] = settings.walletSettings.testKeysQty .toIndexedSeq .flatMap(x => (0 until x).map(defaultRootSecret.child)) diff --git a/src/test/scala/org/ergoplatform/utils/Stubs.scala b/src/test/scala/org/ergoplatform/utils/Stubs.scala index b821755551..01b13100ca 100644 --- a/src/test/scala/org/ergoplatform/utils/Stubs.scala +++ b/src/test/scala/org/ergoplatform/utils/Stubs.scala @@ -263,7 +263,7 @@ trait Stubs extends ErgoGenerators with ErgoTestHelpers with ChainGenerator with object WalletActorStub { val mnemonic = "abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon" val path = DerivationPath(List(0, 1, 2), publicBranch = false) - val secretKey = ExtendedSecretKey.deriveMasterKey(Mnemonic.toSeed(SecretString.create(mnemonic)), usePre1627KeyDerivation = false).derive(path) + val secretKey = ExtendedSecretKey.deriveMasterKey(Mnemonic.toSeed(SecretString.create(mnemonic)), usePre1627KeyDerivation = true).derive(path) val address = P2PKAddress(proveDlogGen.sample.get) val walletBoxN_N: WalletBox = WalletBox( From 4e8e98e09e7759af196f0e0ceee13ee6923097e4 Mon Sep 17 00:00:00 2001 From: pragmaxim Date: Wed, 28 Sep 2022 13:58:48 +0200 Subject: [PATCH 12/21] SIGTERM should be handled as remote shutdown --- .../scala/scorex/core/app/Application.scala | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/src/main/scala/scorex/core/app/Application.scala b/src/main/scala/scorex/core/app/Application.scala index 300b218f8f..9df3b390e4 100644 --- a/src/main/scala/scorex/core/app/Application.scala +++ b/src/main/scala/scorex/core/app/Application.scala @@ -3,6 +3,7 @@ package scorex.core.app import akka.actor.{ActorRef, ActorSystem} import akka.http.scaladsl.Http import akka.http.scaladsl.server.{ExceptionHandler, RejectionHandler, Route} +import org.ergoplatform.ErgoApp import org.ergoplatform.settings.ErgoSettings import scorex.core.api.http.{ApiErrorHandler, ApiRejectionHandler, ApiRoute, CompositeHttpService} import scorex.core.network._ @@ -17,9 +18,6 @@ import scala.concurrent.ExecutionContext trait Application extends ScorexLogging { - import scorex.core.network.NetworkController.ReceivableMessages.ShutdownNetwork - - //settings val ergoSettings: ErgoSettings implicit val scorexSettings: ScorexSettings @@ -100,20 +98,8 @@ trait Application extends ScorexLogging { Runtime.getRuntime.addShutdownHook(new Thread() { override def run() { log.error("Unexpected shutdown") - stopAll() + ErgoApp.shutdownSystem() } }) } - - def stopAll(): Unit = synchronized { - log.info("Stopping network services") - upnpGateway.foreach(_.deletePort(scorexSettings.network.bindAddress.getPort)) - networkControllerRef ! ShutdownNetwork - - log.info("Stopping actors (incl. block generator)") - actorSystem.terminate().onComplete { _ => - log.info("Exiting from the app...") - System.exit(0) - } - } } From 45e4788d278b217b04830a1b80cf54a0291cd209 Mon Sep 17 00:00:00 2001 From: Alex Chepurnoy Date: Sat, 1 Oct 2022 20:53:19 +0300 Subject: [PATCH 13/21] 104 version set --- src/main/resources/api/openapi.yaml | 2 +- src/main/resources/application.conf | 2 +- src/main/resources/mainnet.conf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index 1709ee50fb..2016ee2819 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.2" info: - version: "4.0.103" + version: "4.0.104" title: Ergo Node API description: API docs for Ergo Node. Models are shared between all Ergo products contact: diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index d125cf180b..436d20a14c 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -400,7 +400,7 @@ scorex { nodeName = "ergo-node" # Network protocol version to be sent in handshakes - appVersion = 4.0.103 + appVersion = 4.0.104 # Network agent name. May contain information about client code # stack, starting from core code-base up to the end graphical interface. diff --git a/src/main/resources/mainnet.conf b/src/main/resources/mainnet.conf index 4b5439078a..61d958fd94 100644 --- a/src/main/resources/mainnet.conf +++ b/src/main/resources/mainnet.conf @@ -76,7 +76,7 @@ scorex { network { magicBytes = [1, 0, 2, 4] bindAddress = "0.0.0.0:9030" - nodeName = "ergo-mainnet-4.0.103" + nodeName = "ergo-mainnet-4.0.104" nodeName = ${?NODENAME} knownPeers = [ "213.239.193.208:9030", From e321014acf8ecc253da07273cf1e17d309675902 Mon Sep 17 00:00:00 2001 From: Alex Chepurnoy Date: Sun, 2 Oct 2022 19:24:12 +0300 Subject: [PATCH 14/21] fixed height for eip37 activation --- .../modifierprocessors/HeadersProcessor.scala | 37 +------------------ 1 file changed, 2 insertions(+), 35 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala index d77a056edb..36010fc66d 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala @@ -19,7 +19,6 @@ import scorex.core.consensus.ProgressInfo import scorex.core.consensus.ModifierSemanticValidity import scorex.core.utils.ScorexEncoding import scorex.core.validation.{InvalidModifier, ModifierValidator, ValidationResult, ValidationState} -import scorex.crypto.hash.Blake2b256 import scorex.db.ByteArrayWrapper import scorex.util._ @@ -258,18 +257,6 @@ trait HeadersProcessor extends ToDownloadProcessor with ScorexLogging with Score protected def heightIdsKey(height: Int): ByteArrayWrapper = ByteArrayWrapper(Algos.hash(Ints.toByteArray(height))) - private val EIP37VotingParameter: Byte = 6 // input cost, set 6 = 2100 for voting on EIP-37 - - private val eip37Key = Blake2b256.hash("eip37 activation height") - - private def storeEip37ActivationHeight(eip37ActivationHeight: Int) = { - historyStorage.insert(eip37Key, Ints.toByteArray(eip37ActivationHeight)) - } - - private def eip37ActivationHeight: Option[Int] = { - historyStorage.get(scorex.util.bytesToId(eip37Key)).map(Ints.fromByteArray) - } - /** * Calculate difficulty for the next block * @@ -279,29 +266,9 @@ trait HeadersProcessor extends ToDownloadProcessor with ScorexLogging with Score def requiredDifficultyAfter(parent: Header): Difficulty = { val parentHeight = parent.height - val minActivationHeight = 843776 - val maxActivationHeight = 843776 + 98304 - val checkActivationPeriod = 128 - val activationVotesChecked = 256 - val activationThreshold = 232 - - // todo: this EIP-37 activation checking code could be removed after activation - if (settings.chainSettings.isMainnet && - parentHeight > minActivationHeight && - parentHeight <= maxActivationHeight && - parentHeight % checkActivationPeriod == 0 && - eip37ActivationHeight.isEmpty) { - val chain = headerChainBack(activationVotesChecked, parent, _ => false) - val votesFor = chain.headers.map(_.votes).map(_.contains(EIP37VotingParameter)).count(_ == true) - val eip37Activated = votesFor >= activationThreshold - if (eip37Activated) { - log.info(s"EIP-37 activated on ${parentHeight + 1}, votes for: $votesFor") - storeEip37ActivationHeight(parentHeight + 1) - } - } + val eip37ActivationHeight = 844673 - if (parentHeight > minActivationHeight && parentHeight + 1 >= eip37ActivationHeight.getOrElse(Int.MaxValue)) { - // by eip37VotedOn definition could be on mainnet only + if (settings.networkType.isMainNet && parentHeight + 1 >= eip37ActivationHeight) { val epochLength = 128 // epoch length after EIP-37 activation if (parentHeight % epochLength == 0) { val heights = difficultyCalculator.previousHeadersRequiredForRecalculation(parentHeight + 1, epochLength) From 87dec4f0737b61e9f276f5dbb8152a0f02385bb9 Mon Sep 17 00:00:00 2001 From: Alex Chepurnoy Date: Tue, 4 Oct 2022 17:23:44 +0300 Subject: [PATCH 15/21] no voting for parameter #6 anymore --- .../mining/CandidateGenerator.scala | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala b/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala index bce5cebace..43633e8d5b 100644 --- a/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala +++ b/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala @@ -495,28 +495,11 @@ object CandidateGenerator extends ScorexLogging { newParams.blockVersion ) } else { - val preVotes = currentParams.vote( + val votes = currentParams.vote( ergoSettings.votingTargets.targets, stateContext.votingData.epochVotes, voteForSoftFork ) - // we allow to put vote for parameter #6 (input cost) used for EIP-37 activation, - // even if epoch wasn't started with such a vote - val voteForEip37 = 2400 // 6 == 2400 in the config - val votes = if(newHeight > 843776 && - newHeight < 844800 + 2048 && - ergoSettings.votingTargets.targets.get(Parameters.InputCostIncrease).contains(voteForEip37) && - !preVotes.contains(Parameters.InputCostIncrease)) { - val idx = preVotes.indexWhere(_ == Parameters.NoParameter) - if(idx != -1) { - preVotes.update(idx, Parameters.InputCostIncrease) - preVotes - } else { - preVotes - } - } else { - preVotes - } ( interlinksExtension, votes, From 61e4fc9a904b0408f7973d0365d37fc6717c9b25 Mon Sep 17 00:00:00 2001 From: Alex Chepurnoy Date: Wed, 5 Oct 2022 14:43:14 +0300 Subject: [PATCH 16/21] 6 = 2400 voting removed --- src/main/resources/mainnet.conf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/resources/mainnet.conf b/src/main/resources/mainnet.conf index 4b5439078a..bb317f971f 100644 --- a/src/main/resources/mainnet.conf +++ b/src/main/resources/mainnet.conf @@ -66,10 +66,6 @@ ergo { maxTransactionCost = 4900000 } - voting { - 6 = 2400 - } - } scorex { From 4b98b289543309c971d4ebbcb6230f0e88842036 Mon Sep 17 00:00:00 2001 From: Alex Chepurnoy Date: Thu, 6 Oct 2022 17:26:27 +0300 Subject: [PATCH 17/21] equals and hashcode for UnconfirmedTransaction --- .../scala/org/ergoplatform/mining/CandidateGenerator.scala | 4 +++- .../modifiers/mempool/UnconfirmedTransaction.scala | 6 ++++++ .../org/ergoplatform/nodeView/mempool/ErgoMemPool.scala | 2 +- .../org/ergoplatform/nodeView/mempool/OrderedTxPool.scala | 4 ++-- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala b/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala index 43633e8d5b..a24dd82c22 100644 --- a/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala +++ b/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala @@ -158,7 +158,9 @@ class CandidateGenerator( _ ! StatusReply.error(s"Candidate generation failed : ${ex.getMessage}") ) case Some(Success((candidate, eliminatedTxs))) => - if (eliminatedTxs.ids.nonEmpty) viewHolderRef ! eliminatedTxs + if (eliminatedTxs.ids.nonEmpty) { + viewHolderRef ! eliminatedTxs + } val generationTook = System.currentTimeMillis() - start log.info(s"Generated new candidate in $generationTook ms") context.become( diff --git a/src/main/scala/org/ergoplatform/modifiers/mempool/UnconfirmedTransaction.scala b/src/main/scala/org/ergoplatform/modifiers/mempool/UnconfirmedTransaction.scala index e6bace01e4..5c99a0e4e3 100644 --- a/src/main/scala/org/ergoplatform/modifiers/mempool/UnconfirmedTransaction.scala +++ b/src/main/scala/org/ergoplatform/modifiers/mempool/UnconfirmedTransaction.scala @@ -30,6 +30,12 @@ case class UnconfirmedTransaction(transaction: ErgoTransaction, copy(lastCost = Some(cost), lastCheckedTime = System.currentTimeMillis()) } + override def equals(obj: Any): Boolean = obj match { + case that: UnconfirmedTransaction => that.id == id + case _ => false + } + + override def hashCode(): Int = id.hashCode() } object UnconfirmedTransaction { diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala index fc6c5faf3d..6b10aa4161 100644 --- a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala +++ b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala @@ -188,7 +188,7 @@ class ErgoMemPool private[mempool](private[mempool] val pool: OrderedTxPool, val validationStartTime = System.currentTimeMillis() val blacklistedTransactions = nodeSettings.blacklistedTransactions - if(blacklistedTransactions.nonEmpty && blacklistedTransactions.contains(tx.id)) { + if (blacklistedTransactions.nonEmpty && blacklistedTransactions.contains(tx.id)) { val exc = new Exception("blacklisted tx") this.invalidate(unconfirmedTx) -> new ProcessingOutcome.Invalidated(exc, validationStartTime) } else { diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala index 1a6fda31cf..00dfc64af6 100644 --- a/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala +++ b/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala @@ -96,7 +96,7 @@ case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedT invalidatedTxIds, outputs -- tx.outputs.map(_.id), inputs -- tx.inputs.map(_.boxId) - ).updateFamily(unconfirmedTx, -wtx.weight, System.currentTimeMillis(), 0) + ).updateFamily(unconfirmedTx, -wtx.weight, System.currentTimeMillis(), depth = 0) case None => this } } @@ -111,7 +111,7 @@ case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedT invalidatedTxIds.put(tx.id), outputs -- tx.outputs.map(_.id), inputs -- tx.inputs.map(_.boxId) - ).updateFamily(unconfirmedTx, -wtx.weight, System.currentTimeMillis(), 0) + ).updateFamily(unconfirmedTx, -wtx.weight, System.currentTimeMillis(), depth = 0) case None => OrderedTxPool(orderedTransactions, transactionsRegistry, invalidatedTxIds.put(tx.id), outputs, inputs) } From 99c12371adf414934939ccb0fb97b114ce6711bd Mon Sep 17 00:00:00 2001 From: Alex Chepurnoy Date: Thu, 6 Oct 2022 18:06:55 +0300 Subject: [PATCH 18/21] contains reverted --- .../network/ErgoNodeViewSynchronizer.scala | 11 ++++++----- .../ergoplatform/nodeView/mempool/ErgoMemPool.scala | 10 +++++++++- .../ergoplatform/nodeView/mempool/OrderedTxPool.scala | 2 +- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala index 3925140495..2198633d21 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala @@ -762,6 +762,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, if (txAcceptanceFilter) { val unknownMods = + //todo: filter out transactions invalidated in the mempool? invData.ids.filter(mid => deliveryTracker.status(mid, modifierTypeId, Seq(mp)) == ModifiersStatus.Unknown) // filter out transactions that were already applied to history val notApplied = unknownMods.filterNot(blockAppliedTxsCache.mightContain) @@ -1033,22 +1034,22 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, } } - case FailedOnRecheckTransaction(_, _) => - // do nothing for now + case FailedOnRecheckTransaction(id, _) => + declined.put(id, System.currentTimeMillis()) case SyntacticallySuccessfulModifier(modTypeId, modId) => deliveryTracker.setHeld(modId, modTypeId) case RecoverableFailedModification(modTypeId, modId, e) => - logger.debug(s"Setting recoverable failed modifier ${modId} as Unknown", e) + logger.debug(s"Setting recoverable failed modifier $modId as Unknown", e) deliveryTracker.setUnknown(modId, modTypeId) case SyntacticallyFailedModification(modTypeId, modId, e) => - logger.debug(s"Invalidating syntactically failed modifier ${modId}", e) + logger.debug(s"Invalidating syntactically failed modifier $modId", e) deliveryTracker.setInvalid(modId, modTypeId).foreach(penalizeMisbehavingPeer) case SemanticallyFailedModification(modTypeId, modId, e) => - logger.debug(s"Invalidating semantically failed modifier ${modId}", e) + logger.debug(s"Invalidating semantically failed modifier $modId", e) deliveryTracker.setInvalid(modId, modTypeId).foreach(penalizeMisbehavingPeer) case ChangedHistory(newHistoryReader: ErgoHistory) => diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala index 6b10aa4161..eb67c33028 100644 --- a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala +++ b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala @@ -229,7 +229,15 @@ class ErgoMemPool private[mempool](private[mempool] val pool: OrderedTxPool, acceptIfNoDoubleSpend(unconfirmedTx, validationStartTime) } } else { - val exc = new Exception(s"Pool can not accept transaction ${tx.id}, it is invalidated earlier or the pool is full") + val contains = this.contains(tx.id) + val msg = if(contains) { + s"Pool can not accept transaction ${tx.id}, it is already in the mempool" + } else if(pool.size == settings.nodeSettings.mempoolCapacity) { + s"Pool can not accept transaction ${tx.id}, the mempool is full" + } else { + s"Pool can not accept transaction ${tx.id}" + } + val exc = new Exception(msg) this -> new ProcessingOutcome.Declined(exc, validationStartTime) } } else { diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala index 00dfc64af6..5fd3b75590 100644 --- a/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala +++ b/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala @@ -142,7 +142,7 @@ case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedT * @return - true, if transaction is in the pool or invalidated earlier, false otherwise */ def contains(id: ModifierId): Boolean = { - transactionsRegistry.contains(id) || isInvalidated(id) + transactionsRegistry.contains(id) } def isInvalidated(id: ModifierId): Boolean = invalidatedTxIds.mightContain(id) From 9f243027468ba53485fff4f489a1006b8e1e754a Mon Sep 17 00:00:00 2001 From: Alex Chepurnoy Date: Thu, 6 Oct 2022 18:10:23 +0300 Subject: [PATCH 19/21] log on removal --- .../scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala index eb67c33028..829b013d80 100644 --- a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala +++ b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala @@ -109,6 +109,7 @@ class ErgoMemPool private[mempool](private[mempool] val pool: OrderedTxPool, } def remove(unconfirmedTransaction: UnconfirmedTransaction): ErgoMemPool = { + log.debug(s"Removing transaction ${unconfirmedTransaction.id} from the mempool") val tx = unconfirmedTransaction.transaction val wtx = pool.transactionsRegistry.get(tx.id) val updStats = wtx.map(wgtx => stats.add(System.currentTimeMillis(), wgtx)) From 982071cd9bbabee10b9d70dc591b0fb2edab954a Mon Sep 17 00:00:00 2001 From: Alex Chepurnoy Date: Thu, 6 Oct 2022 22:41:27 +0300 Subject: [PATCH 20/21] todo improved --- .../org/ergoplatform/network/ErgoNodeViewSynchronizer.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala index 2198633d21..15e05a4ef0 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala @@ -762,7 +762,8 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, if (txAcceptanceFilter) { val unknownMods = - //todo: filter out transactions invalidated in the mempool? + // todo: filter out transactions invalidated in the mempool, + // todo: see https://github.com/ergoplatform/ergo/issues/1863 invData.ids.filter(mid => deliveryTracker.status(mid, modifierTypeId, Seq(mp)) == ModifiersStatus.Unknown) // filter out transactions that were already applied to history val notApplied = unknownMods.filterNot(blockAppliedTxsCache.mightContain) From 381624d12650489f30e5d549a09cf739e6c80c2f Mon Sep 17 00:00:00 2001 From: Alex Chepurnoy Date: Fri, 7 Oct 2022 00:39:30 +0300 Subject: [PATCH 21/21] invalidated and pool size logging --- .../org/ergoplatform/network/ErgoNodeViewSynchronizer.scala | 4 ++-- .../org/ergoplatform/nodeView/mempool/ErgoMemPool.scala | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala index 15e05a4ef0..747dab25cf 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala @@ -97,7 +97,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, * The node stops to accept transactions if declined table reaches this max size. It prevents spam attacks trying * to bloat the table (or exhaust node's CPU) */ - private val MaxDeclined = 400 + private val MaxDeclined = 1000 /** * No more than this number of unparsed transactions can be cached @@ -968,7 +968,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, // helper method to clear declined transactions after some off, so the node may accept them again private def clearDeclined(): Unit = { - val clearTimeout = FiniteDuration(10, MINUTES) + val clearTimeout = FiniteDuration(20, MINUTES) val now = System.currentTimeMillis() val toRemove = declined.filter { case (_, time) => diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala index 829b013d80..45671bd75e 100644 --- a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala +++ b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala @@ -185,7 +185,13 @@ class ErgoMemPool private[mempool](private[mempool] val pool: OrderedTxPool, def process(unconfirmedTx: UnconfirmedTransaction, state: ErgoState[_]): (ErgoMemPool, ProcessingOutcome) = { val tx = unconfirmedTx.transaction + + val invalidatedCnt = this.pool.invalidatedTxIds.approximateElementCount + val poolSize = this.size + log.info(s"Processing mempool transaction: $tx") + log.debug(s"Mempool: invalidated transactions: $invalidatedCnt, pool size: $poolSize") + val validationStartTime = System.currentTimeMillis() val blacklistedTransactions = nodeSettings.blacklistedTransactions