From 63808acaa99a674d85e1b37ffe738717181eb98a Mon Sep 17 00:00:00 2001 From: Ivano Pagano Date: Mon, 7 Sep 2020 16:35:11 +0200 Subject: [PATCH] Add a lower bound on fees' timestamps in the computation query --- .../common/testkit/util/RandomTestKit.scala | 2 +- .../tezos/TezosDatabaseOperations.scala | 39 +++++++++++++++++-- .../tezos/TezosDatabaseOperationsTest.scala | 12 ++++-- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/conseil-common-testkit/src/main/scala/tech/cryptonomic/conseil/common/testkit/util/RandomTestKit.scala b/conseil-common-testkit/src/main/scala/tech/cryptonomic/conseil/common/testkit/util/RandomTestKit.scala index 23f54657b..0829638da 100644 --- a/conseil-common-testkit/src/main/scala/tech/cryptonomic/conseil/common/testkit/util/RandomTestKit.scala +++ b/conseil-common-testkit/src/main/scala/tech/cryptonomic/conseil/common/testkit/util/RandomTestKit.scala @@ -17,7 +17,7 @@ trait RandomGenerationKit { //a stable timestamp reference if needed lazy val testReferenceTimestamp = - new Timestamp(testReferenceDateTime.toEpochSecond) + new Timestamp(testReferenceDateTime.toEpochSecond * 1000L) //creates pseudo-random strings of given length, based on an existing [[Random]] generator val alphaNumericGenerator = (random: Random) => random.alphanumeric.take(_: Int).mkString diff --git a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperations.scala b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperations.scala index c0b598a46..a89a14c58 100644 --- a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperations.scala +++ b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperations.scala @@ -1,7 +1,7 @@ package tech.cryptonomic.conseil.indexer.tezos import java.sql.Timestamp -import java.time.Instant +import java.time.{Instant, ZoneOffset} import cats.effect.Async import cats.implicits._ @@ -761,11 +761,41 @@ object TezosDatabaseOperations extends LazyLogging { * Given the operation kind, return range of fees and timestamp for that operation. * @param kind Operation kind * @param numberOfFeesAveraged How many values to use for statistics computations + * @param asOf When the computation is to be considered, by default uses the time of invocation * @return The average fees for a given operation kind, if it exists */ - def calculateAverageFees(kind: String, numberOfFeesAveraged: Int)( + def calculateAverageFees( + kind: String, + numberOfFeesAveraged: Int, + asOf: Instant = Instant.now() + )( implicit ec: ExecutionContext ): DBIO[Option[AverageFees]] = { + /* We need to limit the past timestamps for this computation to a reasonable value. + * Otherwise the query optimizer won't be able to efficiently use the indexing and + * will do a full table scan. + * + * This is what we know now: + * - each cycle bakes 4096 blocks + * - a cycle takes around 2-3 days to run + * - each block stores a variable number of transactions, down-to the min of 1. + * + * We can make conservative computations to figure out how far in the past we need to go, + * to guarantee a [[numberOfFeesAveraged]] values. + * + * We can assume a single transaction per block (1 trans/block), hence we need numberOfFeesAveraged blocks. + * Assuming 3 days to get 4096 blocks we have 4096/3 blocks-a-day at worst, which is ~1365 blocks. + * Therefore we want to look back to numberOfFeesAveraged/1365 days in the past to guarantee the required fees counts. + */ + val blocksPerDay = 1365 + val daysToPastHorizon = 1 + (numberOfFeesAveraged / blocksPerDay) //round-up for integer division + val secsPerDay = 60L * 60L * 24L //secs * mins * hours + val secsToPastHorizon = daysToPastHorizon.toLong * secsPerDay + + logger.info( + s"Computing fees starting from $daysToPastHorizon days before $asOf, averaging over $numberOfFeesAveraged values" + ) + type Cycle = Int type Fee = BigDecimal type FeeDetails = (Option[Fee], Timestamp, Option[Cycle], BlockLevel) @@ -785,11 +815,14 @@ object TezosDatabaseOperations extends LazyLogging { AverageFees(max(m - s, 0), m, m + s, ts, kind, cycle, level) } + val timestampLowerBound = + Timestamp.from(asOf.atOffset(ZoneOffset.UTC).minusSeconds(secsToPastHorizon).toInstant()) + val opQuery = Tables.Operations .filter(_.kind === kind) + .filter(_.timestamp >= timestampLowerBound) .map(o => (o.fee, o.timestamp, o.cycle, o.blockLevel)) - .distinct .sortBy { case (_, ts, _, _) => ts.desc } .take(numberOfFeesAveraged) .result diff --git a/conseil-lorre/src/test/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperationsTest.scala b/conseil-lorre/src/test/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperationsTest.scala index ab5906809..ebd97b4be 100644 --- a/conseil-lorre/src/test/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperationsTest.scala +++ b/conseil-lorre/src/test/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperationsTest.scala @@ -957,7 +957,9 @@ class TezosDatabaseOperationsTest ) //check - val feesCalculation = sut.calculateAverageFees(ops.head.kind, feesToConsider) + //we specify when the computation of fees needs be done, to have the test block reference time in range + val feesCalculation = + sut.calculateAverageFees(ops.head.kind, feesToConsider, asOf = testReferenceDateTime.toInstant()) dbHandler.run(feesCalculation).futureValue.value shouldEqual expected @@ -981,7 +983,9 @@ class TezosDatabaseOperationsTest dbHandler.run(populate).futureValue should have size (fees.size) //check - val feesCalculation = sut.calculateAverageFees("undefined", feesToConsider) + //we specify when the computation of fees needs be done, to have the test block reference time in range + val feesCalculation = + sut.calculateAverageFees("undefined", feesToConsider, asOf = testReferenceDateTime.toInstant()) dbHandler.run(feesCalculation).futureValue shouldBe None @@ -1027,7 +1031,9 @@ class TezosDatabaseOperationsTest level = 0 ) //check - val feesCalculation = sut.calculateAverageFees(selection.head.kind, feesToConsider) + //we specify when the computation of fees needs be done, to have the test block reference time in range + val feesCalculation = + sut.calculateAverageFees(selection.head.kind, feesToConsider, asOf = testReferenceDateTime.toInstant()) dbHandler.run(feesCalculation).futureValue.value shouldEqual expected