Add a lower bound on fees' timestamps in the computation query

conseil-common-testkit/src/main/scala/tech/cryptonomic/conseil/common/testkit/util/RandomTestKit.scala

@@ -17,7 +17,7 @@ trait RandomGenerationKit {
 
   //a stable timestamp reference if needed
   lazy val testReferenceTimestamp =
-    new Timestamp(testReferenceDateTime.toEpochSecond)
+    new Timestamp(testReferenceDateTime.toEpochSecond * 1000L)
 
   //creates pseudo-random strings of given length, based on an existing [[Random]] generator
   val alphaNumericGenerator = (random: Random) => random.alphanumeric.take(_: Int).mkString

conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperations.scala

@@ -1,7 +1,7 @@
 package tech.cryptonomic.conseil.indexer.tezos
 
 import java.sql.Timestamp
-import java.time.Instant
+import java.time.{Instant, ZoneOffset}
 
 import cats.effect.Async
 import cats.implicits._
@@ -761,11 +761,41 @@ object TezosDatabaseOperations extends LazyLogging {
     * Given the operation kind, return range of fees and timestamp for that operation.
     * @param kind                 Operation kind
     * @param numberOfFeesAveraged How many values to use for statistics computations
+    * @param asOf                 When the computation is to be considered, by default uses the time of invocation
     * @return                     The average fees for a given operation kind, if it exists
     */
-  def calculateAverageFees(kind: String, numberOfFeesAveraged: Int)(
+  def calculateAverageFees(
+      kind: String,
+      numberOfFeesAveraged: Int,
+      asOf: Instant = Instant.now()
+  )(
       implicit ec: ExecutionContext
   ): DBIO[Option[AverageFees]] = {
+    /* We need to limit the past timestamps for this computation to a reasonable value.
+     * Otherwise the query optimizer won't be able to efficiently use the indexing and
+     * will do a full table scan.
+     *
+     * This is what we know now:
+     * - each cycle bakes 4096 blocks
+     * - a cycle takes around 2-3 days to run
+     * - each block stores a variable number of transactions, down-to the min of 1.
+     *
+     * We can make conservative computations to figure out how far in the past we need to go,
+     * to guarantee a [[numberOfFeesAveraged]] values.
+     *
+     * We can assume a single transaction per block (1 trans/block), hence we need numberOfFeesAveraged blocks.
+     * Assuming 3 days to get 4096 blocks we have 4096/3 blocks-a-day at worst, which is ~1365 blocks.
+     * Therefore we want to look back to numberOfFeesAveraged/1365 days in the past to guarantee the required fees counts.
+     */
+    val blocksPerDay = 1365
+    val daysToPastHorizon = 1 + (numberOfFeesAveraged / blocksPerDay) //round-up for integer division
+    val secsPerDay = 60L * 60L * 24L //secs * mins * hours
+    val secsToPastHorizon = daysToPastHorizon.toLong * secsPerDay
+
+    logger.info(
+      s"Computing fees starting from $daysToPastHorizon days before $asOf, averaging over $numberOfFeesAveraged values"
+    )
+
     type Cycle = Int
     type Fee = BigDecimal
     type FeeDetails = (Option[Fee], Timestamp, Option[Cycle], BlockLevel)
@@ -785,11 +815,14 @@ object TezosDatabaseOperations extends LazyLogging {
       AverageFees(max(m - s, 0), m, m + s, ts, kind, cycle, level)
     }
 
+    val timestampLowerBound =
+      Timestamp.from(asOf.atOffset(ZoneOffset.UTC).minusSeconds(secsToPastHorizon).toInstant())
+
     val opQuery =
       Tables.Operations
         .filter(_.kind === kind)
+        .filter(_.timestamp >= timestampLowerBound)
         .map(o => (o.fee, o.timestamp, o.cycle, o.blockLevel))
-        .distinct
         .sortBy { case (_, ts, _, _) => ts.desc }
         .take(numberOfFeesAveraged)
         .result

conseil-lorre/src/test/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperationsTest.scala

@@ -957,7 +957,9 @@ class TezosDatabaseOperationsTest
         )
 
         //check
-        val feesCalculation = sut.calculateAverageFees(ops.head.kind, feesToConsider)
+        //we specify when the computation of fees needs be done, to have the test block reference time in range
+        val feesCalculation =
+          sut.calculateAverageFees(ops.head.kind, feesToConsider, asOf = testReferenceDateTime.toInstant())
 
         dbHandler.run(feesCalculation).futureValue.value shouldEqual expected
 
@@ -981,7 +983,9 @@ class TezosDatabaseOperationsTest
         dbHandler.run(populate).futureValue should have size (fees.size)
 
         //check
-        val feesCalculation = sut.calculateAverageFees("undefined", feesToConsider)
+        //we specify when the computation of fees needs be done, to have the test block reference time in range
+        val feesCalculation =
+          sut.calculateAverageFees("undefined", feesToConsider, asOf = testReferenceDateTime.toInstant())
 
         dbHandler.run(feesCalculation).futureValue shouldBe None
 
@@ -1027,7 +1031,9 @@ class TezosDatabaseOperationsTest
           level = 0
         )
         //check
-        val feesCalculation = sut.calculateAverageFees(selection.head.kind, feesToConsider)
+        //we specify when the computation of fees needs be done, to have the test block reference time in range
+        val feesCalculation =
+          sut.calculateAverageFees(selection.head.kind, feesToConsider, asOf = testReferenceDateTime.toInstant())
 
         dbHandler.run(feesCalculation).futureValue.value shouldEqual expected