Merge branch 'hotfix/4.1.7'

.gitignore

@@ -3,6 +3,7 @@ logs
 bin
 conf/application.conf
 conf/migration.conf
+/conf/cloner.conf
 graphql.config.json
 graphql.schema.json
 .graphqlconfig

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Change Log
 
+## [4.1.7](https://github.com/TheHive-Project/TheHive/milestone/76) (2021-07-05)
+
+**Implemented enhancements:**
+
+- [Enhancement] Copy the database even if the schema version doesn't match (with force flag) [\#2105](https://github.com/TheHive-Project/TheHive/issues/2105)
+
+**Fixed bugs:**
+
+- [Bug] Issue with Migration 3.5.1 -> 4.1.6 [\#2089](https://github.com/TheHive-Project/TheHive/issues/2089)
+- [Bug] Fix serialization for case number messages [\#2107](https://github.com/TheHive-Project/TheHive/issues/2107)
+- [Bug] Case is removed if the assignee is removed [\#2109](https://github.com/TheHive-Project/TheHive/issues/2109)
+
 ## [4.1.6](https://github.com/TheHive-Project/TheHive/milestone/75) (2021-06-14)
 
 **Implemented enhancements:**

build.sbt

@@ -2,7 +2,7 @@ import Dependencies._
 import com.typesafe.sbt.packager.Keys.bashScriptDefines
 import org.thp.ghcl.Milestone
 
-val thehiveVersion         = "4.1.6-1"
+val thehiveVersion         = "4.1.7-1"
 val scala212               = "2.12.13"
 val scala213               = "2.13.1"
 val supportedScalaVersions = List(scala212, scala213)

conf/application.sample.conf

@@ -21,6 +21,14 @@ db.janusgraph {
       keyspace: thehive
     }
   }
+  index.search {
+    backend: lucene
+    directory: /opt/thp/thehive/index
+    # If TheHive is in cluster ElasticSearch must be used:
+    // backend: elasticsearch
+    // hostname: ["ip1", "ip2"]
+    // index-name: thehive
+  }
 
   ## For test only !
   # Comment the two lines below before enable Cassandra database

conf/cloner.sample.conf

@@ -0,0 +1,24 @@
+# This is a sample configuration for the database cloner tool
+
+# Configuration of the source database (same format as in application.conf)
+from.db.janusgraph {
+  storage {
+    // backend: cql
+    // hostname: ["ip1", "ip2"]
+  }
+  index.search {
+    backend: lucene
+    directory: /opt/thp/thehive/index
+  }
+}
+# Configuration of the target database
+to.db.janusgraph {
+  storage {
+    // backend: cql
+    // hostname: ["ip1", "ip2"]
+  }
+  index.search {
+    backend: lucene
+    directory: /opt/thp/thehive/otherIndex
+  }
+}

frontend/bower.json

@@ -1,6 +1,6 @@
 {
   "name": "thehive",
-  "version": "4.1.6-1",
+  "version": "4.1.7-1",
   "license": "AGPL-3.0",
   "dependencies": {
     "jquery": "^3.4.1",

frontend/package.json

@@ -1,6 +1,6 @@
 {
     "name": "thehive",
-    "version": "4.1.6-1",
+    "version": "4.1.7-1",
     "license": "AGPL-3.0",
     "repository": {
         "type": "git",

migration/src/main/resources/reference.conf

@@ -117,3 +117,4 @@ to {
   }
 }
 batchSize: 100
+force: false

migration/src/main/scala/org/thp/thehive/cloner/Cloner.scala

@@ -1,7 +1,7 @@
 package org.thp.thehive.cloner
 
 import akka.actor.ActorSystem
-import com.typesafe.config.{Config, ConfigFactory}
+import com.typesafe.config.{Config, ConfigFactory, ConfigValueFactory}
 import org.apache.tinkerpop.gremlin.structure.T
 import org.thp.scalligraph.SingleInstance
 import org.thp.scalligraph.janus.JanusDatabase
@@ -31,6 +31,9 @@ object Cloner extends App with IntegrityCheckApp {
     )
   }
 
+  def addConfig(config: Config, path: String, value: Any): Config =
+    config.withValue(path, ConfigValueFactory.fromAnyRef(value))
+
   val defaultLoggerConfigFile = "/etc/thehive/logback-cloner.xml"
   if (System.getProperty("logger.file") == null && Files.exists(Paths.get(defaultLoggerConfigFile)))
     System.setProperty("logger.file", defaultLoggerConfigFile)
@@ -54,7 +57,9 @@ object Cloner extends App with IntegrityCheckApp {
         .valueName("<file>")
         .required()
         .action((f, c) => ConfigFactory.parseFileAnySyntax(f).withFallback(c))
-        .text("configuration file")
+        .text("configuration file"),
+      opt[Unit]('f', "force")
+        .action((_, c) => addConfig(c, "force", true))
     )
   }
   val defaultConfig =
@@ -78,19 +83,28 @@ object Cloner extends App with IntegrityCheckApp {
 
       val thehiveSchema = new TheHiveSchemaDefinition
       val cortexSchema  = new CortexSchemaDefinition
-      if (sourceDatabase.version(thehiveSchema.name) != thehiveSchema.operations.operations.length + 1) {
-        println(
-          "The schema of TheHive is not valid " +
-            s"(found ${sourceDatabase.version(thehiveSchema.name)}, expected ${thehiveSchema.operations.operations.length + 1})"
-        )
-        sys.exit(1)
+
+      {
+        val expectedVersion = thehiveSchema.operations.operations.length + 1
+        val foundVersion    = sourceDatabase.version(thehiveSchema.name)
+        if (foundVersion != expectedVersion) {
+          println(s"The schema of TheHive is not valid (expected: $expectedVersion, found: $foundVersion)")
+          if (config.getBoolean("force"))
+            println("Continuing ...")
+          else
+            sys.exit(1)
+        }
       }
-      if (sourceDatabase.version(cortexSchema.name) != cortexSchema.operations.operations.length + 1) {
-        println(
-          "The schema of Cortex is not valid " +
-            s"(found ${sourceDatabase.version(cortexSchema.name)}, expected ${cortexSchema.operations.operations.length + 1})"
-        )
-        sys.exit(1)
+      {
+        val expectedVersion = cortexSchema.operations.operations.length + 1
+        val foundVersion    = sourceDatabase.version(cortexSchema.name)
+        if (foundVersion != expectedVersion) {
+          println(s"The schema of Cortex is not valid (expected: $expectedVersion, found: $foundVersion)")
+          if (config.getBoolean("force"))
+            println("Continuing ...")
+          else
+            sys.exit(1)
+        }
       }
 
       val destDatabase: Database = getDatabase(
@@ -111,8 +125,8 @@ object Cloner extends App with IntegrityCheckApp {
       // don't create initial values
       val models = destDatabase.extraModels ++ thehiveSchema.modelList ++ cortexSchema.modelList
       destDatabase.createSchema(models)
-      destDatabase.setVersion(thehiveSchema.name, thehiveSchema.operations.operations.length + 1)
-      destDatabase.setVersion(cortexSchema.name, cortexSchema.operations.operations.length + 1)
+      destDatabase.setVersion(thehiveSchema.name, sourceDatabase.version(thehiveSchema.name))
+      destDatabase.setVersion(cortexSchema.name, sourceDatabase.version(cortexSchema.name))
 
       val batchSize: Int = config.getInt("batchSize")
 
@@ -167,6 +181,7 @@ object Cloner extends App with IntegrityCheckApp {
 
       println("Add indices ...")
       destDatabase.addSchemaIndexes(models)
+      println("Run checks ...")
       runChecks(destDatabase, Configuration(config))
       destDatabase.close()
     } finally {

migration/src/main/scala/org/thp/thehive/cloner/IntegrityCheckApp.scala

@@ -40,6 +40,7 @@ trait IntegrityCheckApp {
               bindActor[DummyActor]("config-actor")
               bindActor[DummyActor]("cortex-actor")
               bindActor[DummyActor]("integrity-check-actor")
+              bindTypedActor(CaseNumberActor.behavior, "case-number-actor")
 
               val integrityCheckOpsBindings = ScalaMultibinder.newSetBinder[GenIntegrityCheckOps](binder)
               integrityCheckOpsBindings.addBinding.to[ProfileIntegrityCheckOps]

migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala

@@ -52,6 +52,7 @@ object Output {
               bindActor[DummyActor]("config-actor")
               bindActor[DummyActor]("cortex-actor")
               bindActor[DummyActor]("integrity-check-actor")
+              bindTypedActor(CaseNumberActor.behavior, "case-number-actor")
 
               val schemaBindings = ScalaMultibinder.newSetBinder[UpdatableSchema](binder)
               schemaBindings.addBinding.to[TheHiveSchemaDefinition]

thehive/app/org/thp/thehive/TheHiveModule.scala

@@ -107,7 +107,7 @@ class TheHiveModule(environment: Environment, configuration: Configuration) exte
     integrityCheckOpsBindings.addBinding.to[ObservableIntegrityCheckOps]
     integrityCheckOpsBindings.addBinding.to[LogIntegrityCheckOps]
     bind[ActorRef].annotatedWithName("integrity-check-actor").toProvider[IntegrityCheckActorProvider]
-    bind[TypedActorRef[CaseNumberActor.Request]].annotatedWithName("case-number-actor").toProvider[CaseNumberActorProvider]
+    bind[TypedActorRef[CaseNumberActor.Request]].toProvider[CaseNumberActorProvider]
 
     bind[ActorRef].annotatedWithName("flow-actor").toProvider[FlowActorProvider]
 

thehive/app/org/thp/thehive/services/CaseNumber.scala

@@ -1,6 +1,6 @@
 package org.thp.thehive.services
 
-import akka.actor.typed.scaladsl.Behaviors
+import akka.actor.typed.scaladsl.{ActorContext, Behaviors}
 import akka.actor.typed.scaladsl.adapter.ClassicActorSystemOps
 import akka.actor.typed.{ActorRefResolver, Behavior, ActorRef => TypedActorRef}
 import akka.actor.{ActorSystem, ExtendedActorSystem}
@@ -12,6 +12,7 @@ import org.thp.thehive.GuiceAkkaExtension
 import org.thp.thehive.services.CaseOps._
 
 import java.io.NotSerializableException
+import java.nio.ByteBuffer
 import javax.inject.{Inject, Provider, Singleton}
 
 object CaseNumberActor {
@@ -21,15 +22,25 @@ object CaseNumberActor {
   case class GetNextNumber(replyTo: TypedActorRef[Response]) extends Request
   case class NextNumber(number: Int)                         extends Response
 
-  val behavior: Behavior[Request] = Behaviors.setup[Request] { context =>
+  val behavior: Behavior[Request] = Behaviors.setup[Request](context => waitFirstRequest(context))
+
+  def getNextCaseNumber(context: ActorContext[Request]): Int = {
     val injector = GuiceAkkaExtension(context.system).injector
     val db       = injector.getInstance(classOf[Database])
     val caseSrv  = injector.getInstance(classOf[CaseSrv])
     db.roTransaction { implicit graph =>
-      caseNumberProvider(caseSrv.startTraversal.getLast.headOption.fold(0)(_.number) + 1)
+      caseSrv.startTraversal.getLast.headOption.fold(0)(_.number) + 1
     }
   }
 
+  def waitFirstRequest(context: ActorContext[Request]): Behaviors.Receive[Request] =
+    Behaviors.receiveMessage {
+      case GetNextNumber(replyTo) =>
+        val nextNumber = getNextCaseNumber(context)
+        replyTo ! NextNumber(nextNumber)
+        caseNumberProvider(nextNumber + 1)
+    }
+
   def caseNumberProvider(nextNumber: Int): Behavior[Request] =
     Behaviors.receiveMessage {
       case GetNextNumber(replyTo) =>
@@ -55,22 +66,15 @@ class CaseNumberSerializer(system: ExtendedActorSystem) extends Serializer {
   override def toBinary(o: AnyRef): Array[Byte] =
     o match {
       case GetNextNumber(replyTo) => 0.toByte +: actorRefResolver.toSerializationFormat(replyTo).getBytes
-      case NextNumber(number) =>
-        Array(1.toByte, ((number >> 24) % 0xff).toByte, ((number >> 16) % 0xff).toByte, ((number >> 8) % 0xff).toByte, (number % 0xff).toByte)
-      case _ => throw new NotSerializableException
+      case NextNumber(number)     => ByteBuffer.allocate(5).put(1.toByte).putInt(number).array()
+      case _                      => throw new NotSerializableException
     }
 
   override def includeManifest: Boolean = false
 
   override def fromBinary(bytes: Array[Byte], manifest: Option[Class[_]]): AnyRef =
     bytes(0) match {
       case 0 => GetNextNumber(actorRefResolver.resolveActorRef(new String(bytes.tail)))
-      case 1 =>
-        NextNumber(
-          (bytes(2) << 24) +
-            (bytes(3) << 16) +
-            (bytes(4) << 8) +
-            bytes(5)
-        )
+      case 1 => NextNumber(ByteBuffer.wrap(bytes).getInt(1))
     }
 }

thehive/app/org/thp/thehive/services/CaseSrv.scala

@@ -1,8 +1,9 @@
 package org.thp.thehive.services
 
-import akka.actor.ActorRef
 import akka.actor.typed.scaladsl.AskPattern._
+import akka.actor.typed.scaladsl.adapter.ClassicSchedulerOps
 import akka.actor.typed.{Scheduler, ActorRef => TypedActorRef}
+import akka.actor.{ActorRef, ActorSystem}
 import akka.util.Timeout
 import org.apache.tinkerpop.gremlin.process.traversal.{Order, P}
 import org.apache.tinkerpop.gremlin.structure.Vertex
@@ -34,7 +35,7 @@ import java.lang.{Long => JLong}
 import java.util.{Date, List => JList, Map => JMap}
 import javax.inject.{Inject, Named, Provider, Singleton}
 import scala.concurrent.duration.DurationInt
-import scala.concurrent.{Await, ExecutionContext, Future}
+import scala.concurrent.{Await, ExecutionContextExecutor, Future}
 import scala.util.{Failure, Success, Try}
 
 @Singleton
@@ -53,10 +54,9 @@ class CaseSrv @Inject() (
     userSrv: UserSrv,
     alertSrvProvider: Provider[AlertSrv],
     @Named("integrity-check-actor") integrityCheckActor: ActorRef,
-    @Named("case-number-actor") caseNumberActor: TypedActorRef[CaseNumberActor.Request],
+    caseNumberActor: TypedActorRef[CaseNumberActor.Request],
     cache: SyncCacheApi,
-    implicit val ec: ExecutionContext,
-    implicit val scheduler: Scheduler
+    system: ActorSystem
 ) extends VertexSrv[Case] {
   lazy val alertSrv: AlertSrv = alertSrvProvider.get
 
@@ -133,7 +133,9 @@ class CaseSrv @Inject() (
   }
 
   def nextCaseNumberAsync: Future[Int] = {
-    implicit val timeout: Timeout = Timeout(1.minute)
+    implicit val timeout: Timeout             = Timeout(1.minute)
+    implicit val scheduler: Scheduler         = system.scheduler.toTyped
+    implicit val ec: ExecutionContextExecutor = system.dispatcher
     caseNumberActor.ask[CaseNumberActor.Response](replyTo => CaseNumberActor.GetNextNumber(replyTo)).map {
       case CaseNumberActor.NextNumber(caseNumber) => caseNumber
     }

thehive/test/org/thp/thehive/services/CaseNumberTest.scala

@@ -0,0 +1,34 @@
+package org.thp.thehive.services
+
+import akka.actor.ExtendedActorSystem
+import akka.actor.typed.ActorSystem
+import akka.actor.typed.scaladsl.Behaviors
+import org.thp.thehive.TestAppBuilder
+import play.api.test.PlaySpecification
+
+class CaseNumberTest extends PlaySpecification with TestAppBuilder {
+
+  "case number actor" should {
+    "serialize and deserialize messages" in withActorSystem { system =>
+      val ref = system.deadLetters[CaseNumberActor.Response]
+      val sut = new CaseNumberSerializer(system.classicSystem.asInstanceOf[ExtendedActorSystem])
+
+      val messages = Seq(
+        CaseNumberActor.GetNextNumber(ref),
+        CaseNumberActor.NextNumber(42),
+        CaseNumberActor.NextNumber(Int.MaxValue)
+      )
+      val out = messages.map(message => sut.toBinary(message))
+
+      val result = out.map(bin => sut.fromBinary(bin))
+
+      result must beEqualTo(messages)
+    }
+  }
+
+  private def withActorSystem[T](body: ActorSystem[Nothing] => T) = {
+    val system = ActorSystem(Behaviors.empty, "test")
+    try body(system)
+    finally system.terminate()
+  }
+}