Ignite typeahead

example/apache-ignite/README.md

@@ -2,6 +2,18 @@
 
 # Apache Ignite Sample
 
+## Running VUU and Ignite server as single process
+1. Open static main and set to run as ignite server
+```scala
+org.finos.vuu.example.ignite.IgniteVuuMain
+
+val runAsIgniteServer = ture
+
+```
+2. Run the static main
+
+## Running VUU against Separate Ignite Server
+
 How to run: 
 
 1. Run Ignite Cluster Application (run config for IntelliJ checked in): 

example/apache-ignite/src/main/scala/org/finos/vuu/example/ignite/IgniteColumnValueProvider.scala

@@ -0,0 +1,14 @@
+package org.finos.vuu.example.ignite
+
+import org.finos.vuu.core.table.ColumnValueProvider
+
+
+class IgniteColumnValueProvider(final val igniteStore: IgniteOrderStore) extends ColumnValueProvider {
+
+  private val MAX_RESULT_COUNT = 10
+  override def getUniqueValues(columnName: String): Array[String] =
+    igniteStore.getDistinct(columnName, MAX_RESULT_COUNT).toArray
+
+  override def getUniqueValuesStartingWith(columnName: String, starts: String): Array[String] =
+    igniteStore.getDistinct(columnName, starts, MAX_RESULT_COUNT).toArray
+}

example/apache-ignite/src/main/scala/org/finos/vuu/example/ignite/IgniteOrderStore.scala

@@ -2,13 +2,15 @@ package org.finos.vuu.example.ignite
 
 import com.typesafe.scalalogging.StrictLogging
 import org.apache.ignite.cache.CachePeekMode
-import org.apache.ignite.cache.query.{IndexQuery, IndexQueryCriteriaBuilder, IndexQueryCriterion, SqlFieldsQuery}
+import org.apache.ignite.cache.query._
 import org.apache.ignite.cluster.ClusterState
 import org.apache.ignite.{IgniteCache, Ignition}
 import org.finos.vuu.core.module.simul.model.{ChildOrder, OrderStore, ParentOrder}
 import org.finos.vuu.example.ignite.schema.IgniteChildOrderEntity
 
+import java.util
 import scala.collection.mutable
+import scala.collection.mutable.ListBuffer
 import scala.jdk.CollectionConverters._
 import scala.jdk.javaapi.CollectionConverters.asJava
 
@@ -59,6 +61,33 @@ class IgniteOrderStore(private val parentOrderCache: IgniteCache[Int, ParentOrde
     parentOrderCache.get(id)
   }
 
+  def getDistinct(columnName: String, rowCount: Int): Iterable[String] = {
+    val query = new SqlFieldsQuery(s"select distinct $columnName from ChildOrder limit ?")
+    query.setArgs(rowCount)
+
+    val results = childOrderCache.query(query)
+
+    val (counter, buffer) = mapToString(results)
+
+    logger.info(s"Loaded Distinct Ignite ChildOrder column $columnName for $counter rows")
+
+    buffer
+  }
+
+  def getDistinct(columnName: String, startsWith: String, rowCount: Int): Iterable[String] = {
+    val query = new SqlFieldsQuery(s"select distinct $columnName from ChildOrder where $columnName LIKE \'$startsWith%\' limit ?")
+    query.setArgs(rowCount)
+    logger.info(query.getSql)
+    val results = childOrderCache.query(query)
+
+    val (counter, buffer) = mapToString(results)
+
+    logger.info(s"Loaded Distinct Ignite ChildOrder column $columnName that starts with $startsWith for $counter rows")
+
+    buffer
+
+  }
+
   def findChildOrder(sqlFilterQueries: String, sqlSortQueries: String, rowCount: Int, startIndex: Long): Iterator[ChildOrder] = {
     val whereClause = if(sqlFilterQueries == null || sqlFilterQueries.isEmpty) "" else s" where $sqlFilterQueries"
     val orderByClause = if(sqlSortQueries == null || sqlSortQueries.isEmpty) " order by id" else s" order by $sqlSortQueries"
@@ -70,6 +99,7 @@ class IgniteOrderStore(private val parentOrderCache: IgniteCache[Int, ParentOrde
 
     results
   }
+
   def findChildOrderFilteredBy(filterQueryCriteria: List[IndexQueryCriterion]): Iterable[ChildOrder] = {
   //  val filter: IgniteBiPredicate[Int, ChildOrder]  = (key, p) => p.openQty > 0
 
@@ -96,6 +126,16 @@ class IgniteOrderStore(private val parentOrderCache: IgniteCache[Int, ParentOrde
 
   private def toChildOrder = IgniteChildOrderEntity.getListToChildOrder
 
+  private def mapToString(results: FieldsQueryCursor[util.List[_]]): (Int, ListBuffer[String]) = {
+    var counter = 0
+    val buffer = mutable.ListBuffer[String]()
+    results.forEach(row => {
+      buffer.addOne(row.get(0).asInstanceOf[String])
+      counter += 1
+    })
+    (counter, buffer)
+  }
+
   def childOrderCount(): Long = {
     childOrderCache.sizeLong(CachePeekMode.ALL)
   }

example/apache-ignite/src/main/scala/org/finos/vuu/example/ignite/IgniteVuuMain.scala

@@ -14,6 +14,7 @@ import org.finos.vuu.core.module.price.PriceModule
 import org.finos.vuu.core.module.simul.SimulationModule
 import org.finos.vuu.core.module.typeahead.TypeAheadModule
 import org.finos.vuu.core.module.vui.VuiStateModule
+import org.finos.vuu.example.ignite.loader.IgniteOrderGenerator
 import org.finos.vuu.example.ignite.module.IgniteOrderDataModule
 import org.finos.vuu.net.auth.AlwaysHappyAuthenticator
 import org.finos.vuu.net.http.VuuHttp2ServerOptions
@@ -37,6 +38,7 @@ object IgniteVuuMain extends App with StrictLogging {
   implicit val tableDefContainer: TableDefContainer = new TableDefContainer(Map())
 
   logger.info("[VUU] Starting...")
+  val runAsIgniteServer = false
 
   val store = new MemoryBackedVuiStateStore()
 
@@ -54,6 +56,11 @@ object IgniteVuuMain extends App with StrictLogging {
   val certPath = defaultConfig.getString("vuu.certPath")
   val keyPath = defaultConfig.getString("vuu.keyPath")
 
+  logger.info(s"[Ignite] Starting ignite in ${if(runAsIgniteServer) "Server" else "Client"} mode")
+  private val igniteOrderStore = IgniteOrderStore(clientMode = !runAsIgniteServer)
+  if(runAsIgniteServer)
+    SaveOrdersInIgnite()
+
   val config = VuuServerConfig(
     VuuHttp2ServerOptions()
       //only specify webroot if we want to load the source locally, we'll load it from the jar
@@ -75,9 +82,10 @@ object IgniteVuuMain extends App with StrictLogging {
     VuuThreadingOptions()
       .withViewPortThreads(4)
       .withTreeThreads(4)
-  ).withModule(MetricsModule())
-   .withModule(IgniteOrderDataModule(IgniteOrderStore()))
-   .withPlugin(VirtualizedTablePlugin)
+  ).withModule(TypeAheadModule())
+    .withModule(MetricsModule())
+    .withModule(IgniteOrderDataModule(igniteOrderStore))
+    .withPlugin(VirtualizedTablePlugin)
 
   val vuuServer = new VuuServer(config)
 
@@ -86,4 +94,10 @@ object IgniteVuuMain extends App with StrictLogging {
   logger.info("[VUU] Ready.")
 
   vuuServer.join()
+
+  private def SaveOrdersInIgnite(): Unit = {
+    val igniteOrderGenerator = new IgniteOrderGenerator(igniteOrderStore)
+    igniteOrderGenerator.save()
+  }
+
 }

example/apache-ignite/src/main/scala/org/finos/vuu/example/ignite/loader/IgniteOrderGenerator.scala

@@ -0,0 +1,40 @@
+package org.finos.vuu.example.ignite.loader
+
+import com.typesafe.scalalogging.StrictLogging
+import org.finos.toolbox.lifecycle.LifecycleContainer
+import org.finos.toolbox.time.Clock
+import org.finos.vuu.core.module.simul.model.{ChildOrder, OrderStore}
+import org.finos.vuu.core.module.simul.provider.{ParentChildOrdersModel, SeededRandomNumbers}
+
+import java.util.concurrent.Executors
+import java.util.concurrent.atomic.LongAdder
+
+class IgniteOrderGenerator(orderStore: OrderStore) (implicit clock: Clock, lifecycleContainer: LifecycleContainer) extends StrictLogging{
+
+  implicit val randomNumbers: SeededRandomNumbers = new SeededRandomNumbers(clock.now())
+  private val ordersModel = new ParentChildOrdersModel()
+  private val childOrderCounter = new LongAdder()
+  private val executor = Executors.newWorkStealingPool()
+
+  def save(): Unit = {
+
+    logger.info("[Ignite] Saving orders to ignite.")
+    (0 until 4_000).foreach(i =>
+      executor.execute { () =>
+        val parent = ordersModel.createParent()
+        val childrenToCreate = randomNumbers.seededRand(100, 250)
+
+        val children = (0 until childrenToCreate)
+          .map(_ => ordersModel.createChild(parent))
+          .foldLeft(List[ChildOrder]())((acc, child) => acc :+ child)
+
+        orderStore.storeParentOrderWithChildren(parent, children)
+        childOrderCounter.add(children.length)
+        if (i % 1000 == 0) {
+          println(s"[${Thread.currentThread().getName}] Loaded : $i parent orders and ${childOrderCounter.sum()} child orders")
+        }
+      })
+  }
+
+
+}

example/apache-ignite/src/main/scala/org/finos/vuu/example/ignite/module/IgniteOrderDataModule.scala

@@ -5,7 +5,7 @@ import org.finos.toolbox.time.Clock
 import org.finos.vuu.api.ViewPortDef
 import org.finos.vuu.core.module.{DefaultModule, ModuleFactory, TableDefContainer, ViewServerModule}
 import org.finos.vuu.core.table.{Column, Columns}
-import org.finos.vuu.example.ignite.IgniteOrderStore
+import org.finos.vuu.example.ignite.{IgniteColumnValueProvider, IgniteOrderStore}
 import org.finos.vuu.example.ignite.provider.IgniteOrderDataProvider
 import org.finos.vuu.net.rpc.RpcHandler
 import org.finos.vuu.plugin.virtualized.api.VirtualizedSessionTableDef

example/apache-ignite/src/main/scala/org/finos/vuu/example/ignite/provider/IgniteOrderDataProvider.scala

@@ -63,6 +63,13 @@ class IgniteOrderDataProvider(final val igniteStore: IgniteOrderStore)
   override def doDestroy(): Unit = {}
 
   override val lifecycleId: String = "org.finos.vuu.example.ignite.provider.IgniteOrderDataProvider"
+
+  override def getUniqueValues(columnName: String): Array[String] =
+    igniteStore.getDistinct(columnName, 10).toArray
+
+  override def getUniqueValuesStartingWith(columnName: String, starts: String): Array[String] =
+    igniteStore.getDistinct(columnName, starts, 10).toArray
+
 }
 
 object IgniteOrderDataProvider {

example/apache-ignite/src/test/scala/org/finos/vuu/example/ignite/IgniteOrderStoreTest.scala

@@ -99,17 +99,17 @@ class IgniteOrderStoreTest extends AnyFunSuiteLike with BeforeAndAfter with Matc
 
   test("Ignite Store With Custom Sql Filters Include String") {
     var parentOrder: ParentOrder = GivenParentOrder(1)
-    parentOrder = GivenParentHasChildOrder(parentOrder, 1)
-    parentOrder = GivenParentHasChildOrder(parentOrder, 2)
-    parentOrder = GivenParentHasChildOrder(parentOrder, 3)
+    parentOrder = GivenParentHasChildOrder(parentOrder, 1, ric = "VOD.L")
+    parentOrder = GivenParentHasChildOrder(parentOrder, 2, ric = "VOD.L")
+    parentOrder = GivenParentHasChildOrder(parentOrder, 3, ric = "VOD.L")
 
     var parentOrder2: ParentOrder = GivenParentOrder(2)
-    parentOrder2 = GivenParentHasChildOrder(parentOrder2, 4)
-    parentOrder2 = GivenParentHasChildOrder(parentOrder2, 5)
-    parentOrder2 = GivenParentHasChildOrder(parentOrder2, 6)
+    parentOrder2 = GivenParentHasChildOrder(parentOrder2, 4, ric = "VOD.L")
+    parentOrder2 = GivenParentHasChildOrder(parentOrder2, 5, ric = "VOD.L")
+    parentOrder2 = GivenParentHasChildOrder(parentOrder2, 6, ric = "VOD.L")
 
-    val filterQueries = "ric = \'ric\'"
-    val childOrder = orderStore.findChildOrder(filterQueries, emptySortQueries, 100, 0)
+    val filterQueries = "ric = \'VOD.L\'"
+    val childOrder = orderStore.findChildOrder(filterQueries, emptySortQueries, 100, 0).toList
 
     assert(childOrder != null)
     assert(childOrder.size == 6)
@@ -123,7 +123,7 @@ class IgniteOrderStoreTest extends AnyFunSuiteLike with BeforeAndAfter with Matc
     parentOrder2 = GivenParentHasChildOrder(parentOrder2, 4)
     parentOrder2 = GivenParentHasChildOrder(parentOrder2, 5)
 
-    val childOrder = orderStore.findChildOrder(emptyFilterQueries, emptySortQueries, 100, 0)
+    val childOrder = orderStore.findChildOrder(emptyFilterQueries, emptySortQueries, 100, 0).toList
 
     assert(childOrder != null)
     assert(childOrder.size == 3)
@@ -138,7 +138,7 @@ class IgniteOrderStoreTest extends AnyFunSuiteLike with BeforeAndAfter with Matc
     parentOrder2 = GivenParentHasChildOrder(parentOrder2, 2)
 
     val sortByValues = "parentId ASC"
-    val childOrder = orderStore.findChildOrder(emptyFilterQueries, sortByValues, 100, 0)
+    val childOrder = orderStore.findChildOrder(emptyFilterQueries, sortByValues, 100, 0).toList
 
     assert(childOrder != null)
     assert(childOrder.size == 3)
@@ -154,23 +154,74 @@ class IgniteOrderStoreTest extends AnyFunSuiteLike with BeforeAndAfter with Matc
     parentOrder2 = GivenParentHasChildOrder(parentOrder2, 2)
 
     val sortByValues = "parentId DESC, id ASC"
-    val childOrder = orderStore.findChildOrder(emptyFilterQueries, sortByValues, 100, 0)
+    val childOrder = orderStore.findChildOrder(emptyFilterQueries, sortByValues, 100, 0).toList
 
     assert(childOrder != null)
     assert(childOrder.size == 3)
     childOrder.map(c => c.id).toArray shouldBe Array(2, 1, 3)
   }
+
+  test("Ignite Store Get Distinct Rics Limited By RowCount") {
+    var parentOrder: ParentOrder = GivenParentOrder(1)
+    parentOrder = GivenParentHasChildOrder(parentOrder, 1, "A")
+    parentOrder = GivenParentHasChildOrder(parentOrder, 2, "B")
+    parentOrder = GivenParentHasChildOrder(parentOrder, 3, "A")
+    parentOrder = GivenParentHasChildOrder(parentOrder, 4, "D")
+
+    val distinctRics = orderStore.getDistinct("ric", 2)
+
+    assert(distinctRics != null)
+    assert(distinctRics.size == 2)
+    distinctRics.toArray shouldBe Array("A", "B")
+  }
+
+  //todo handle exception and report to ui
+  ignore("Ignite Store Get Distinct Rics Returns Empty When Column Does Not Exist") {
+    var parentOrder: ParentOrder = GivenParentOrder(1)
+    parentOrder = GivenParentHasChildOrder(parentOrder, 1, "A")
+
+    val distinctRics = orderStore.getDistinct("NoSuchColumn", 10)
+
+    assert(distinctRics != null)
+    assert(distinctRics.isEmpty)
+  }
+
+  test("Ignite Store Get Distinct Rics Starting With") {
+    var parentOrder: ParentOrder = GivenParentOrder(1)
+    parentOrder = GivenParentHasChildOrder(parentOrder, 1, "ABC")
+    parentOrder = GivenParentHasChildOrder(parentOrder, 2, "ABL")
+    parentOrder = GivenParentHasChildOrder(parentOrder, 3, "DAC")
+    parentOrder = GivenParentHasChildOrder(parentOrder, 4, "ACD")
+
+    val distinctRics = orderStore.getDistinct("ric", "AB", 10)
+
+    assert(distinctRics != null)
+    assert(distinctRics.size == 2)
+    distinctRics.toArray shouldBe Array("ABC", "ABL")
+  }
+
+  test("Ignite Store Get Distinct Rics Starting With Returns Empty When No Match") {
+    var parentOrder: ParentOrder = GivenParentOrder(1)
+    parentOrder = GivenParentHasChildOrder(parentOrder, 1, "ABC")
+    parentOrder = GivenParentHasChildOrder(parentOrder, 2, "ABL")
+
+    val distinctRics = orderStore.getDistinct("ric", "X", 10)
+
+    assert(distinctRics != null)
+    assert(distinctRics.isEmpty)
+  }
   private def GivenParentOrder(parentOrderId: Int): ParentOrder = {
     val parentOrder = TestUtils.createParentOrder(parentOrderId)
     orderStore.storeParentOrder(parentOrder)
     parentOrder
   }
 
-  private def GivenParentHasChildOrder(parentOrder: ParentOrder, childOrderId: Int): ParentOrder = {
+  private def GivenParentHasChildOrder(parentOrder: ParentOrder, childOrderId: Int, ric:String = "AnyRic"): ParentOrder = {
     val updatedParentOrder = parentOrder.copy(activeChildren = parentOrder.activeChildren + 1)
     orderStore.storeChildOrder(
       updatedParentOrder,
-      TestUtils.createChildOrder(childOrderId, parentId = parentOrder.id))
+      TestUtils.createChildOrder(childOrderId, parentId = parentOrder.id, ric))
+
     updatedParentOrder
   }
 

example/virtualized-table/src/main/scala/org/finos/vuu/example/virtualtable/provider/ReallyBigVirtualizedDataProvider.scala

@@ -78,4 +78,8 @@ class ReallyBigVirtualizedDataProvider(implicit clock: Clock) extends Virtualize
   override def doDestroy(): Unit = {}
 
   override val lifecycleId: String = "org.finos.vuu.example.virtualtable.provider.ReallyBigVirtualizedDataProvider"
+
+  override def getUniqueValues(columnName: String): Array[String] = ???
+
+  override def getUniqueValuesStartingWith(columnName: String, starts: String): Array[String] = ???
 }

plugin/ignite-plugin/src/main/scala/org/finos/vuu/feature/ignite/table/IgniteTableFactory.scala

@@ -1,10 +1,10 @@
 package org.finos.vuu.feature.ignite.table
 
 import org.finos.vuu.api.TableDef
-import org.finos.vuu.core.table.DataTable
+import org.finos.vuu.core.table.{DataTable, TableContainer}
 import org.finos.vuu.feature.TableFactory
 import org.finos.vuu.provider.JoinTableProvider
 
 class IgniteTableFactory extends TableFactory {
-  override def createTable(tableDef: TableDef, joinTableProvider: JoinTableProvider): DataTable = ???
+  override def createTable(tableDef: TableDef, tableContainer: TableContainer, joinTableProvider: JoinTableProvider): DataTable = ???
 }

plugin/virtualized-table-plugin/src/main/scala/org/finos/vuu/plugin/virtualized/VirtualizedTablePlugin.scala

@@ -2,7 +2,7 @@ package org.finos.vuu.plugin.virtualized
 
 import org.finos.toolbox.jmx.MetricsProvider
 import org.finos.vuu.api.TableDef
-import org.finos.vuu.core.table.InMemDataTable
+import org.finos.vuu.core.table.{InMemDataTable, TableContainer}
 import org.finos.vuu.feature.{FilterFactory, JoinTableFactory, SessionTableFactory, SortFactory, TableFactory, ViewPortCallableFactory, ViewPortFactory, ViewPortKeysCreator, ViewPortTableCreator, ViewPortTreeCallableFactory}
 import org.finos.vuu.plugin.virtualized.plugin.ViewPortVirtualizedTableCreator
 import org.finos.vuu.plugin.virtualized.viewport.VirtualizedViewPortCallableFactory
@@ -14,8 +14,10 @@ object VirtualizedTablePlugin extends DefaultPlugin {
 
   final val callableFactory = new VirtualizedViewPortCallableFactory
 
-  override def tableFactory(implicit metrics: MetricsProvider): TableFactory = (tableDef: TableDef, joinTableProvider: JoinTableProvider) => {
-    new InMemDataTable(tableDef, joinTableProvider)
+  override def tableFactory(implicit metrics: MetricsProvider): TableFactory = (tableDef: TableDef, tableContainer: TableContainer, joinTableProvider: JoinTableProvider) => {
+    val table = new InMemDataTable(tableDef, joinTableProvider)
+    tableContainer.addTable(table)
+    table
   }
 
   override def pluginType: PluginType = VirtualizedTablePluginType