Fix fee estimation APIs: bounded waitTime estimate, consistent with getFee (#1884)

src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala

@@ -336,20 +336,39 @@ class ErgoMemPool private[mempool](private[mempool] val pool: OrderedTxPool,
     * @return average time for this transaction to be placed in block
     */
   def getExpectedWaitTime(txFee : Long, txSize : Int): Long  = {
-    // Create dummy transaction entry
     val feePerKb = txFee * 1024 / txSize
-    val dummyModifierId = bytesToId(Array.fill(32)(0.toByte))
-    val wtx = WeightedTxId(dummyModifierId, feePerKb, feePerKb, 0)
 
-    // Find position of entry in mempool
-    val posInPool = pool.orderedTransactions.keySet.until(wtx).size
+    // First, try to estimate the wait time from the fee histogram (the same statistics
+    // `getRecommendedFee` is based on, which makes the two endpoints consistent with
+    // each other): find the earliest wait-time bin such that transactions paying on
+    // average no more than `feePerKb` were included in blocks within that time.
+    val histogramEstimateMs = stats.histogram.zipWithIndex.collectFirst {
+      case (bin, waitMinutes) if bin.nTxns > 0 && bin.totalFee / bin.nTxns <= feePerKb =>
+        waitMinutes.toLong * 60 * 1000
+    }
 
-    // Time since statistics measurement interval (needed to calculate average tx rate)
-    val elapsed = System.currentTimeMillis() - stats.startMeasurement
-    if (stats.takenTxns != 0) {
-      elapsed * posInPool / stats.takenTxns
-    } else {
-      0
+    histogramEstimateMs.getOrElse {
+      // Fallback (no suitable statistics gathered yet): estimate the wait time from the
+      // transaction's position in the pool and the average take rate. The elapsed time is
+      // bounded by the statistics measurement window, so that long periods during which no
+      // transactions are taken from the pool (e.g. stuck transactions, see #1884) cannot
+      // produce absurdly long estimates.
+      val dummyModifierId = bytesToId(Array.fill(32)(0.toByte))
+      val wtx = WeightedTxId(dummyModifierId, feePerKb, feePerKb, 0)
+
+      // Find position of entry in mempool
+      val posInPool = pool.orderedTransactions.keySet.until(wtx).size
+
+      // Time since statistics measurement start (bounded by the measurement window)
+      val elapsed = math.min(
+        System.currentTimeMillis() - stats.startMeasurement,
+        2L * MemPoolStatistics.measurementIntervalMsec
+      )
+      if (stats.takenTxns != 0) {
+        elapsed * posInPool / stats.takenTxns
+      } else {
+        0
+      }
     }
   }
 

src/main/scala/org/ergoplatform/nodeView/mempool/MemPoolStatistics.scala

@@ -51,7 +51,7 @@ case class MemPoolStatistics(startMeasurement: Long,
 object MemPoolStatistics {
   // Time parameters of mempool statistics
   val nHistogramBins: Int = 60  /* one hour */
-  val measurementIntervalMsec: Int = 60 * 1000 /* one hour */
+  val measurementIntervalMsec: Int = 60 * 60 * 1000 /* one hour */
 
   val defaultPoolHistogram: List[FeeHistogramBin] = List.fill(MemPoolStatistics.nHistogramBins)(FeeHistogramBin(0, 0))
 }

src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala

@@ -595,6 +595,63 @@ class ErgoMemPoolSpec extends AnyFlatSpec
     outcome2.isInstanceOf[ProcessingOutcome.Invalidated] shouldBe true
   }
 
+
+  it should "return minimal fee from getRecommendedFee when no statistics is collected" in {
+    val pool = ErgoMemPool.empty(settings)
+    pool.getRecommendedFee(5, 1024) shouldBe settings.nodeSettings.minimalFeeAmount
+  }
+
+  it should "base getRecommendedFee on the fee histogram when statistics is available" in {
+    val now = System.currentTimeMillis()
+    // 4 transactions paying on average 2000000 nanoErg/Kb were taken within 2..3 minutes
+    val histogram = MemPoolStatistics.defaultPoolHistogram.updated(2, FeeHistogramBin(4, 8000000))
+    val stats = MemPoolStatistics(startMeasurement = now - 100000, takenTxns = 4,
+      snapTime = now - 100000, snapTakenTxns = 0, histogram = histogram)
+    val empty = ErgoMemPool.empty(settings)
+    val pool = new ErgoMemPool(empty.pool, stats, empty.sortingOption)
+
+    // for a 1Kb transaction the recommended fee is the average fee per Kb of the first non-empty bin
+    pool.getRecommendedFee(5, 1024) shouldBe 2000000
+  }
+
+  it should "make getExpectedWaitTime consistent with getRecommendedFee" in {
+    val now = System.currentTimeMillis()
+    val histogram = MemPoolStatistics.defaultPoolHistogram.updated(2, FeeHistogramBin(4, 8000000))
+    val stats = MemPoolStatistics(startMeasurement = now - 100000, takenTxns = 4,
+      snapTime = now - 100000, snapTakenTxns = 0, histogram = histogram)
+    val empty = ErgoMemPool.empty(settings)
+    val pool = new ErgoMemPool(empty.pool, stats, empty.sortingOption)
+
+    val txSize = 1024
+    val expectedWaitTimeMinutes = 5
+    val recommendedFee = pool.getRecommendedFee(expectedWaitTimeMinutes, txSize)
+    val waitTimeMs = pool.getExpectedWaitTime(recommendedFee, txSize)
+
+    // a transaction paying the recommended fee is expected to be taken
+    // within the wait time the fee was recommended for
+    waitTimeMs shouldBe 2 * 60 * 1000
+    waitTimeMs should be <= expectedWaitTimeMinutes.toLong * 60 * 1000
+  }
+
+  it should "return bounded getExpectedWaitTime when no transactions are taken from the pool for a long time" in {
+    // no histogram data and a long period of inactivity (e.g. transactions stuck in the pool)
+    val longAgo = System.currentTimeMillis() - 365L * 24 * 60 * 60 * 1000
+    val stats = MemPoolStatistics(startMeasurement = longAgo, takenTxns = 1,
+      snapTime = longAgo, snapTakenTxns = 0)
+    val empty = ErgoMemPool.empty(settings)
+    var pool = new ErgoMemPool(empty.pool, stats, empty.sortingOption)
+
+    // fill the pool with some transactions, all with a priority higher than the queried one
+    (1 to 5).foreach { _ =>
+      pool = pool.put(UnconfirmedTransaction(invalidErgoTransactionGen.sample.get, None))
+    }
+    val posInPool = pool.size
+
+    // the estimate is bounded by the statistics measurement window:
+    // at most 2 * measurementIntervalMsec per pool position (with takenTxns = 1)
+    val waitTimeMs = pool.getExpectedWaitTime(0, 1024)
+    waitTimeMs should be <= 2L * MemPoolStatistics.measurementIntervalMsec * posInPool
+  }
   it should "return random transactions" in {
     val txs = (1 to 10).map(_ => invalidErgoTransactionGen.sample.get)
       .map(tx => UnconfirmedTransaction(tx, None))