Merge pull request #764 from input-output-hk/etcm-109-ecip-1099

[ETCM-109] ECIP-1099 implementation

src/ets/scala/io/iohk/ethereum/ets/blockchain/BlockchainTestConfig.scala

@@ -36,7 +36,6 @@ object BlockchainTestConfig {
     daoForkConfig = None,
     accountStartNonce = UInt256.Zero,
     bootstrapNodes = Set(),
-
     // TODO: only place where this was supposed to be used but now it seems it's not, remove? Issue: EC-312
     gasTieBreaker = false,
     ethCompatibleStorage = true,
@@ -45,7 +44,8 @@ object BlockchainTestConfig {
     phoenixBlockNumber = Long.MaxValue,
     ecip1098BlockNumber = Long.MaxValue,
     treasuryAddress = Address(0),
-    ecip1097BlockNumber = Long.MaxValue
+    ecip1097BlockNumber = Long.MaxValue,
+    ecip1099BlockNumber = Long.MaxValue
   )
 
   val FrontierConfig = BaseBlockchainConfig.copy(
@@ -75,7 +75,8 @@ object BlockchainTestConfig {
     daoForkConfig = Some(
       new DaoForkConfig {
         override val forkBlockNumber: BigInt = 5
-        override val forkBlockHash = ByteString(Hex.decode("f6d7ef1087b5fd94eada533cf8a563f78c3944a2f8ae850e80935d20dc3b7315"))
+        override val forkBlockHash =
+          ByteString(Hex.decode("f6d7ef1087b5fd94eada533cf8a563f78c3944a2f8ae850e80935d20dc3b7315"))
         override val blockExtraData = Some(ByteString(Hex.decode("64616f2d686172642d666f726b")))
         override val range = 10
         override val refundContract = Some(Address("bf4ed7b27f1d666546e30d74d50d173d20bca754"))
@@ -231,8 +232,13 @@ object BlockchainTestConfig {
     maxCodeSize = Some(24576),
     byzantiumBlockNumber = -1,
     constantinopleBlockNumber = 0,
-    monetaryPolicyConfig =
-      MonetaryPolicyConfig(5000000, 0.2, BigInt("5000000000000000000"), BigInt("3000000000000000000"), BigInt("2000000000000000000"))
+    monetaryPolicyConfig = MonetaryPolicyConfig(
+      5000000,
+      0.2,
+      BigInt("5000000000000000000"),
+      BigInt("3000000000000000000"),
+      BigInt("2000000000000000000")
+    )
   )
   val ConstantinopleFixConfig = BaseBlockchainConfig.copy(
     frontierBlockNumber = -1,
@@ -245,8 +251,13 @@ object BlockchainTestConfig {
     byzantiumBlockNumber = -1,
     constantinopleBlockNumber = -1,
     petersburgBlockNumber = 0,
-    monetaryPolicyConfig =
-      MonetaryPolicyConfig(5000000, 0.2, BigInt("5000000000000000000"), BigInt("3000000000000000000"), BigInt("2000000000000000000"))
+    monetaryPolicyConfig = MonetaryPolicyConfig(
+      5000000,
+      0.2,
+      BigInt("5000000000000000000"),
+      BigInt("3000000000000000000"),
+      BigInt("2000000000000000000")
+    )
   )
   val IstanbulConfig = BaseBlockchainConfig.copy(
     frontierBlockNumber = -1,
@@ -260,8 +271,13 @@ object BlockchainTestConfig {
     constantinopleBlockNumber = -1,
     petersburgBlockNumber = -1,
     istanbulBlockNumber = 0,
-    monetaryPolicyConfig =
-      MonetaryPolicyConfig(5000000, 0.2, BigInt("5000000000000000000"), BigInt("3000000000000000000"), BigInt("2000000000000000000"))
+    monetaryPolicyConfig = MonetaryPolicyConfig(
+      5000000,
+      0.2,
+      BigInt("5000000000000000000"),
+      BigInt("3000000000000000000"),
+      BigInt("2000000000000000000")
+    )
   )
   val Eip158ToByzantiumAt5Config = BaseBlockchainConfig.copy(
     frontierBlockNumber = -1,
@@ -286,8 +302,13 @@ object BlockchainTestConfig {
     maxCodeSize = Some(24576),
     byzantiumBlockNumber = 0,
     constantinopleBlockNumber = 5,
-    monetaryPolicyConfig =
-      MonetaryPolicyConfig(5000000, 0.2, BigInt("5000000000000000000"), BigInt("3000000000000000000"), BigInt("2000000000000000000"))
+    monetaryPolicyConfig = MonetaryPolicyConfig(
+      5000000,
+      0.2,
+      BigInt("5000000000000000000"),
+      BigInt("3000000000000000000"),
+      BigInt("2000000000000000000")
+    )
   )
 }
 
@@ -299,7 +320,7 @@ object Validators {
   val eip150Validators = ValidatorsExecutor(Eip150Config, Protocol.Ethash)
   val frontierToHomesteadValidators = ValidatorsExecutor(FrontierToHomesteadAt5, Protocol.Ethash)
   val homesteadToEipValidators = ValidatorsExecutor(HomesteadToEIP150At5, Protocol.Ethash)
-  val homesteadToDaoValidators= ValidatorsExecutor(HomesteadToDaoAt5, Protocol.Ethash)
+  val homesteadToDaoValidators = ValidatorsExecutor(HomesteadToDaoAt5, Protocol.Ethash)
   val eip158Validators = ValidatorsExecutor(Eip158Config, Protocol.Ethash)
   val byzantiumValidators = ValidatorsExecutor(ByzantiumConfig, Protocol.Ethash)
   val constantinopleValidators = ValidatorsExecutor(ConstantinopleConfig, Protocol.Ethash)
@@ -314,17 +335,24 @@ object ValidatorsWithSkippedPoW {
 
   import BlockchainTestConfig._
 
-  val frontierValidators =  ValidatorsExecutor(FrontierConfig, new EthashTestBlockHeaderValidator(FrontierConfig))
+  val frontierValidators = ValidatorsExecutor(FrontierConfig, new EthashTestBlockHeaderValidator(FrontierConfig))
   val homesteadValidators = ValidatorsExecutor(HomesteadConfig, new EthashTestBlockHeaderValidator(HomesteadConfig))
   val eip150Validators = ValidatorsExecutor(Eip150Config, new EthashTestBlockHeaderValidator(Eip150Config))
-  val frontierToHomesteadValidators = ValidatorsExecutor(FrontierToHomesteadAt5, new EthashTestBlockHeaderValidator(FrontierToHomesteadAt5))
-  val homesteadToEipValidators = ValidatorsExecutor(HomesteadToEIP150At5, new EthashTestBlockHeaderValidator(HomesteadToEIP150At5))
-  val homesteadToDaoValidators= ValidatorsExecutor(HomesteadToDaoAt5, new EthashTestBlockHeaderValidator(HomesteadToDaoAt5))
+  val frontierToHomesteadValidators =
+    ValidatorsExecutor(FrontierToHomesteadAt5, new EthashTestBlockHeaderValidator(FrontierToHomesteadAt5))
+  val homesteadToEipValidators =
+    ValidatorsExecutor(HomesteadToEIP150At5, new EthashTestBlockHeaderValidator(HomesteadToEIP150At5))
+  val homesteadToDaoValidators =
+    ValidatorsExecutor(HomesteadToDaoAt5, new EthashTestBlockHeaderValidator(HomesteadToDaoAt5))
   val eip158Validators = ValidatorsExecutor(Eip158Config, new EthashTestBlockHeaderValidator(Eip158Config))
   val byzantiumValidators = ValidatorsExecutor(ByzantiumConfig, new EthashTestBlockHeaderValidator(ByzantiumConfig))
-  val constantinopleValidators = ValidatorsExecutor(ConstantinopleConfig, new EthashTestBlockHeaderValidator(ConstantinopleConfig))
-  val constantinopleFixValidators = ValidatorsExecutor(ConstantinopleFixConfig, new EthashTestBlockHeaderValidator(ConstantinopleFixConfig))
+  val constantinopleValidators =
+    ValidatorsExecutor(ConstantinopleConfig, new EthashTestBlockHeaderValidator(ConstantinopleConfig))
+  val constantinopleFixValidators =
+    ValidatorsExecutor(ConstantinopleFixConfig, new EthashTestBlockHeaderValidator(ConstantinopleFixConfig))
   val istanbulValidators = ValidatorsExecutor(IstanbulConfig, new EthashTestBlockHeaderValidator(IstanbulConfig))
-  val eip158ToByzantiumValidators = ValidatorsExecutor(Eip158ToByzantiumAt5Config, new EthashTestBlockHeaderValidator(Eip158ToByzantiumAt5Config))
-  val byzantiumToConstantinopleAt5 = ValidatorsExecutor(ByzantiumToConstantinopleAt5, new EthashTestBlockHeaderValidator(ByzantiumToConstantinopleAt5))
+  val eip158ToByzantiumValidators =
+    ValidatorsExecutor(Eip158ToByzantiumAt5Config, new EthashTestBlockHeaderValidator(Eip158ToByzantiumAt5Config))
+  val byzantiumToConstantinopleAt5 =
+    ValidatorsExecutor(ByzantiumToConstantinopleAt5, new EthashTestBlockHeaderValidator(ByzantiumToConstantinopleAt5))
 }

src/it/scala/io/iohk/ethereum/txExecTest/ECIP1017Test.scala

@@ -47,7 +47,8 @@ class ECIP1017Test extends AnyFlatSpec with Matchers {
       petersburgBlockNumber = Long.MaxValue,
       ecip1098BlockNumber = Long.MaxValue,
       treasuryAddress = Address(0),
-      ecip1097BlockNumber = Long.MaxValue
+      ecip1097BlockNumber = Long.MaxValue,
+      ecip1099BlockNumber = Long.MaxValue
     )
     val ec = ExecutionContext.fromExecutor(Executors.newFixedThreadPool(4))
 

src/it/scala/io/iohk/ethereum/txExecTest/ForksTest.scala

@@ -46,7 +46,8 @@ class ForksTest extends AnyFlatSpec with Matchers {
       petersburgBlockNumber = Long.MaxValue,
       ecip1098BlockNumber = Long.MaxValue,
       treasuryAddress = Address(0),
-      ecip1097BlockNumber = Long.MaxValue
+      ecip1097BlockNumber = Long.MaxValue,
+      ecip1099BlockNumber = Long.MaxValue
     )
 
     val noErrors = a[Right[_, Seq[Receipt]]]

src/main/resources/chains/etc-chain.conf

@@ -88,6 +88,10 @@
   # Has to be equal or greater than ecip1098-block-number
   ecip1097-block-number = "1000000000000000000"
 
+  # Epoch calibration block number
+  # https://ecips.ethereumclassic.org/ECIPs/ecip-1099
+  ecip1099-block-number = "11700000"
+
   # DAO fork configuration (Ethereum HF/Classic split)
   # https://blog.ethereum.org/2016/07/20/hard-fork-completed/
   dao {

src/main/resources/chains/eth-chain.conf

@@ -88,6 +88,10 @@
   # Has to be equal or greater than ecip1098-block-number
   ecip1097-block-number = "1000000000000000000"
 
+  # Epoch calibration block number
+  # https://ecips.ethereumclassic.org/ECIPs/ecip-1099
+  ecip1099-block-number = "1000000000000000000"
+
   # DAO fork configuration (Ethereum HF/Classic split)
   # https://blog.ethereum.org/2016/07/20/hard-fork-completed/
   dao {

src/main/resources/chains/mordor-chain.conf

@@ -88,6 +88,11 @@
   # Has to be equal or greater than ecip1098-block-number
   ecip1097-block-number = "1000000000000000000"
 
+
+  # Epoch calibration block number
+  # https://ecips.ethereumclassic.org/ECIPs/ecip-1099
+  ecip1099-block-number = "2520000"
+
   # DAO fork configuration (Ethereum HF/Classic split)
   # https://blog.ethereum.org/2016/07/20/hard-fork-completed/
   dao = null

src/main/resources/chains/ropsten-chain.conf

@@ -91,6 +91,11 @@
   # Has to be equal or greater than ecip1098-block-number
   ecip1097-block-number = "1000000000000000000"
 
+
+  # Epoch calibration block number
+  # https://ecips.ethereumclassic.org/ECIPs/ecip-1099
+  ecip1099-block-number = "1000000000000000000"
+
   # DAO fork configuration (Ethereum HF/Classic split)
   # https://blog.ethereum.org/2016/07/20/hard-fork-completed/
   dao {

src/main/resources/chains/test-chain.conf

@@ -88,6 +88,11 @@
   # Has to be equal or greater than ecip1098-block-number
   ecip1097-block-number = "1000000000000000000"
 
+
+  # Epoch calibration block number
+  # https://ecips.ethereumclassic.org/ECIPs/ecip-1099
+  ecip1099-block-number = "1000000000000000000"
+
   # DAO fork configuration (Ethereum HF/Classic split)
   # https://blog.ethereum.org/2016/07/20/hard-fork-completed/
   dao {

src/main/resources/chains/testnet-internal-chain.conf

@@ -88,6 +88,10 @@
     # Has to be equal or greater than ecip1098-block-number
     ecip1097-block-number = "1000000000000000000"
 
+    # Epoch calibration block number
+    # https://ecips.ethereumclassic.org/ECIPs/ecip-1099
+    ecip1099-block-number = "1000000000000000000"
+
     # DAO fork configuration (Ethereum HF/Classic split)
     # https://blog.ethereum.org/2016/07/20/hard-fork-completed/
     dao = null

src/main/scala/io/iohk/ethereum/consensus/ethash/EthashBlockCreator.scala

@@ -24,6 +24,7 @@ class EthashBlockCreator(
   lazy val miningConfig = fullConsensusConfig.specific
   private lazy val coinbase: Address = consensusConfig.coinbase
   private lazy val blockGenerator: EthashBlockGenerator = consensus.blockGenerator
+  lazy val blockchainConfig = consensus.blockchainConfig
 
   def getBlockForMining(parentBlock: Block, withTransactions: Boolean = true): Future[PendingBlock] = {
     val transactions =

src/main/scala/io/iohk/ethereum/consensus/ethash/EthashConsensus.scala

@@ -28,7 +28,7 @@ import scala.concurrent.duration._
 class EthashConsensus private (
     val vm: VMImpl,
     blockchain: BlockchainImpl,
-    blockchainConfig: BlockchainConfig,
+    val blockchainConfig: BlockchainConfig,
     val config: FullConsensusConfig[EthashConfig],
     val validators: ValidatorsExecutor,
     val blockGenerator: EthashBlockGenerator,

src/main/scala/io/iohk/ethereum/consensus/ethash/EthashMiner.scala

@@ -56,12 +56,12 @@ class EthashMiner(
 
   def processMining(): Unit = {
     val parentBlock = blockchain.getBestBlock()
-    val epoch = EthashUtils.epoch(parentBlock.header.number.toLong + 1)
-
+    val blockNumber = parentBlock.header.number.toLong + 1
+    val epoch = EthashUtils.epoch(blockNumber, blockCreator.blockchainConfig.ecip1099BlockNumber.toLong)
     val (dag, dagSize) = (currentEpoch, currentEpochDag, currentEpochDagSize) match {
       case (Some(`epoch`), Some(dag), Some(dagSize)) => (dag, dagSize)
       case _ =>
-        val seed = EthashUtils.seed(epoch)
+        val seed = EthashUtils.seed(blockNumber)
         val dagSize = EthashUtils.dagSize(epoch)
         val dagNumHashes = (dagSize / EthashUtils.HASH_BYTES).toInt
         val dag =
@@ -122,7 +122,7 @@ class EthashMiner(
     val outputStream = new FileOutputStream(dagFile(seed).getAbsolutePath)
     outputStream.write(DagFilePrefix.toArray[Byte])
 
-    val cache = EthashUtils.makeCache(epoch)
+    val cache = EthashUtils.makeCache(epoch, seed)
     val res = new Array[Array[Int]](dagNumHashes)
 
     (0 until dagNumHashes).foreach { i =>

src/main/scala/io/iohk/ethereum/consensus/ethash/EthashUtils.scala

@@ -38,9 +38,6 @@ object EthashUtils {
   // cache growth per epoch
   final val CACHE_BYTES_GROWTH: Long = BigInt(2).pow(17).toLong
 
-  // blocks per epoch
-  final val EPOCH_LENGTH: Int = 30000
-
   // width of mix
   final val MIX_BYTES: Int = 128
 
@@ -56,14 +53,27 @@ object EthashUtils {
   // number of accesses in hashimoto loop
   final val ACCESSES: Int = 64
 
+  // blocks per epoch (before ecip-1099)
+  final val EPOCH_LENGTH_BEFORE_ECIP_1099: Int = 30000
+
+  // blocks per epoch (after ecip-1099)
+  final val EPOCH_LENGTH_AFTER_ECIP_1099: Int = 60000
+
   // scalastyle:on magic.number
 
-  def seed(epoch: Long): ByteString = {
+  private def epochBeforeEcip1099(blockNumber: Long): Long = blockNumber / EPOCH_LENGTH_BEFORE_ECIP_1099
+
+  def seed(blockNumber: Long): ByteString = {
+    val epoch = epochBeforeEcip1099(blockNumber)
     (BigInt(0) until epoch)
       .foldLeft(ByteString(Hex.decode("00" * 32))) { case (b, _) => kec256(b) }
   }
 
-  def epoch(blockNumber: Long): Long = blockNumber / EPOCH_LENGTH
+  private def calcEpochLength(blockNumber: Long, ecip1099ActivationBlock: Long): Long =
+    if (blockNumber < ecip1099ActivationBlock) EPOCH_LENGTH_BEFORE_ECIP_1099 else EPOCH_LENGTH_AFTER_ECIP_1099
+
+  def epoch(blockNumber: Long, ecip1099ActivationBlock: Long): Long =
+    blockNumber / calcEpochLength(blockNumber, ecip1099ActivationBlock)
 
   def cacheSize(epoch: Long): Long = {
     val sz = (CACHE_BYTES_INIT + CACHE_BYTES_GROWTH * epoch) - HASH_BYTES
@@ -91,11 +101,11 @@ object EthashUtils {
     else isPrime(n, 3)
   }
 
-  def makeCache(epoch: Long): Array[Int] = {
+  def makeCache(epoch: Long, seed: ByteString): Array[Int] = {
     /* watch out, arrays are mutable here */
 
     val n = (cacheSize(epoch) / HASH_BYTES).toInt
-    val s = seed(epoch).toArray[Byte]
+    val s = seed.toArray[Byte]
 
     val bytes = new Array[Array[Byte]](n)
     bytes(0) = kec512(s)

src/main/scala/io/iohk/ethereum/consensus/ethash/validators/EthashBlockHeaderValidator.scala

@@ -1,13 +1,15 @@
 package io.iohk.ethereum.consensus.ethash
 package validators
 
+import akka.util.ByteString
 import io.iohk.ethereum.consensus.difficulty.DifficultyCalculator
 import io.iohk.ethereum.consensus.ethash.difficulty.EthashDifficultyCalculator
 import io.iohk.ethereum.consensus.validators.BlockHeaderError.HeaderPoWError
 import io.iohk.ethereum.consensus.validators.{BlockHeaderError, BlockHeaderValid, BlockHeaderValidatorSkeleton}
 import io.iohk.ethereum.crypto
 import io.iohk.ethereum.domain.BlockHeader
 import io.iohk.ethereum.utils.BlockchainConfig
+import monix.execution.atomic.{Atomic, AtomicAny}
 
 /**
   * A block header validator for Ethash.
@@ -17,9 +19,8 @@ class EthashBlockHeaderValidator(blockchainConfig: BlockchainConfig)
   import EthashBlockHeaderValidator._
 
   // NOTE the below comment is from before PoW decoupling
-  // we need concurrent map since validators can be used from multiple places
-  protected val powCaches: java.util.concurrent.ConcurrentMap[Long, PowCacheData] =
-    new java.util.concurrent.ConcurrentHashMap[Long, PowCacheData]()
+  // we need atomic since validators can be used from multiple places
+  protected val powCaches: AtomicAny[List[PowCacheData]] = Atomic(List.empty[PowCacheData])
 
   protected def difficulty: DifficultyCalculator = new EthashDifficultyCalculator(blockchainConfig)
 
@@ -39,25 +40,21 @@ class EthashBlockHeaderValidator(blockchainConfig: BlockchainConfig)
   protected def validatePoW(blockHeader: BlockHeader): Either[BlockHeaderError, BlockHeaderValid] = {
     import EthashUtils._
 
-    import scala.collection.JavaConverters._
-
-    def getPowCacheData(epoch: Long): PowCacheData = {
-      Option(powCaches.get(epoch)) match {
-        case Some(pcd) => pcd
-        case None =>
-          val data = new PowCacheData(cache = EthashUtils.makeCache(epoch), dagSize = EthashUtils.dagSize(epoch))
-
-          val keys = powCaches.keySet().asScala
-          val keysToRemove = keys.toSeq.sorted.take(keys.size - MaxPowCaches + 1)
-          keysToRemove.foreach(powCaches.remove)
-
-          powCaches.put(epoch, data)
-
-          data
+    def getPowCacheData(epoch: Long, seed: ByteString): PowCacheData = {
+      powCaches.transformAndExtract { cache =>
+        cache.find(_.epoch == epoch) match {
+          case Some(pcd) => (pcd, cache)
+          case None =>
+            val data =
+              PowCacheData(epoch, cache = EthashUtils.makeCache(epoch, seed), dagSize = EthashUtils.dagSize(epoch))
+            (data, (data :: cache).take(MaxPowCaches))
+        }
       }
     }
 
-    val powCacheData = getPowCacheData(epoch(blockHeader.number.toLong))
+    val epoch = EthashUtils.epoch(blockHeader.number.toLong, blockchainConfig.ecip1099BlockNumber.toLong)
+    val seed = EthashUtils.seed(blockHeader.number.toLong)
+    val powCacheData = getPowCacheData(epoch, seed)
 
     val proofOfWork = hashimotoLight(
       crypto.kec256(BlockHeader.getEncodedWithoutNonce(blockHeader)),
@@ -75,5 +72,5 @@ class EthashBlockHeaderValidator(blockchainConfig: BlockchainConfig)
 object EthashBlockHeaderValidator {
   final val MaxPowCaches: Int = 2 // maximum number of epochs for which PoW cache is stored in memory
 
-  class PowCacheData(val cache: Array[Int], val dagSize: Long)
+  case class PowCacheData(epoch: Long, cache: Array[Int], dagSize: Long)
 }