From 0009f47aa377e9a308a8cb27d05620209a8493c5 Mon Sep 17 00:00:00 2001 From: Vincent Debergue Date: Wed, 30 Jun 2021 15:07:25 +0200 Subject: [PATCH 1/7] Fix serialization for case number messages --- .../org/thp/thehive/services/CaseNumber.scala | 14 +++----- .../thp/thehive/services/CaseNumberTest.scala | 34 +++++++++++++++++++ 2 files changed, 38 insertions(+), 10 deletions(-) create mode 100644 thehive/test/org/thp/thehive/services/CaseNumberTest.scala diff --git a/thehive/app/org/thp/thehive/services/CaseNumber.scala b/thehive/app/org/thp/thehive/services/CaseNumber.scala index 7366151eae..ddfe2d900a 100644 --- a/thehive/app/org/thp/thehive/services/CaseNumber.scala +++ b/thehive/app/org/thp/thehive/services/CaseNumber.scala @@ -12,6 +12,7 @@ import org.thp.thehive.GuiceAkkaExtension import org.thp.thehive.services.CaseOps._ import java.io.NotSerializableException +import java.nio.ByteBuffer import javax.inject.{Inject, Provider, Singleton} object CaseNumberActor { @@ -55,9 +56,8 @@ class CaseNumberSerializer(system: ExtendedActorSystem) extends Serializer { override def toBinary(o: AnyRef): Array[Byte] = o match { case GetNextNumber(replyTo) => 0.toByte +: actorRefResolver.toSerializationFormat(replyTo).getBytes - case NextNumber(number) => - Array(1.toByte, ((number >> 24) % 0xff).toByte, ((number >> 16) % 0xff).toByte, ((number >> 8) % 0xff).toByte, (number % 0xff).toByte) - case _ => throw new NotSerializableException + case NextNumber(number) => ByteBuffer.allocate(5).put(1.toByte).putInt(number).array() + case _ => throw new NotSerializableException } override def includeManifest: Boolean = false @@ -65,12 +65,6 @@ class CaseNumberSerializer(system: ExtendedActorSystem) extends Serializer { override def fromBinary(bytes: Array[Byte], manifest: Option[Class[_]]): AnyRef = bytes(0) match { case 0 => GetNextNumber(actorRefResolver.resolveActorRef(new String(bytes.tail))) - case 1 => - NextNumber( - (bytes(2) << 24) + - (bytes(3) << 16) + - (bytes(4) << 8) + - bytes(5) - ) + case 1 => NextNumber(ByteBuffer.wrap(bytes).getInt(1)) } } diff --git a/thehive/test/org/thp/thehive/services/CaseNumberTest.scala b/thehive/test/org/thp/thehive/services/CaseNumberTest.scala new file mode 100644 index 0000000000..c3caeb7d46 --- /dev/null +++ b/thehive/test/org/thp/thehive/services/CaseNumberTest.scala @@ -0,0 +1,34 @@ +package org.thp.thehive.services + +import akka.actor.ExtendedActorSystem +import akka.actor.typed.ActorSystem +import akka.actor.typed.scaladsl.Behaviors +import org.thp.thehive.TestAppBuilder +import play.api.test.PlaySpecification + +class CaseNumberTest extends PlaySpecification with TestAppBuilder { + + "case number actor" should { + "serialize and deserialize messages" in withActorSystem { system => + val ref = system.deadLetters[CaseNumberActor.Response] + val sut = new CaseNumberSerializer(system.classicSystem.asInstanceOf[ExtendedActorSystem]) + + val messages = Seq( + CaseNumberActor.GetNextNumber(ref), + CaseNumberActor.NextNumber(42), + CaseNumberActor.NextNumber(Int.MaxValue) + ) + val out = messages.map(message => sut.toBinary(message)) + + val result = out.map(bin => sut.fromBinary(bin)) + + result must beEqualTo(messages) + } + } + + private def withActorSystem[T](body: ActorSystem[Nothing] => T) = { + val system = ActorSystem(Behaviors.empty, "test") + try body(system) + finally system.terminate() + } +} From 75d6ff723603b5258668dd18b82607bdc9b74536 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 30 Jun 2021 19:32:06 +0200 Subject: [PATCH 2/7] #2105 Add force parameter in database cloner tool --- .gitignore | 1 + conf/application.sample.conf | 8 ++++ conf/cloner.sample.conf | 24 ++++++++++ migration/src/main/resources/reference.conf | 1 + .../scala/org/thp/thehive/cloner/Cloner.scala | 47 ++++++++++++------- 5 files changed, 65 insertions(+), 16 deletions(-) create mode 100644 conf/cloner.sample.conf diff --git a/.gitignore b/.gitignore index 2c87b915d9..90ff884861 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ logs bin conf/application.conf conf/migration.conf +/conf/cloner.conf graphql.config.json graphql.schema.json .graphqlconfig diff --git a/conf/application.sample.conf b/conf/application.sample.conf index 4de3da18e0..1045ee97b3 100644 --- a/conf/application.sample.conf +++ b/conf/application.sample.conf @@ -21,6 +21,14 @@ db.janusgraph { keyspace: thehive } } + index.search { + backend: lucene + directory: /opt/thp/thehive/index + # If TheHive is in cluster ElasticSearch must be used: + // backend: elasticsearch + // hostname: ["ip1", "ip2"] + // index-name: thehive + } ## For test only ! # Comment the two lines below before enable Cassandra database diff --git a/conf/cloner.sample.conf b/conf/cloner.sample.conf new file mode 100644 index 0000000000..990147dde4 --- /dev/null +++ b/conf/cloner.sample.conf @@ -0,0 +1,24 @@ +# This is a sample configuration for the database cloner tool + +# Configuration of the source database (same format as in application.conf) +from.db.janusgraph { + storage { + // backend: cql + // hostname: ["ip1", "ip2"] + } + index.search { + backend: lucene + directory: /opt/thp/thehive/index + } +} +# Configuration of the target database +to.db.janusgraph { + storage { + // backend: cql + // hostname: ["ip1", "ip2"] + } + index.search { + backend: lucene + directory: /opt/thp/thehive/otherIndex + } +} diff --git a/migration/src/main/resources/reference.conf b/migration/src/main/resources/reference.conf index 90a131e6a0..57b1bb0eeb 100644 --- a/migration/src/main/resources/reference.conf +++ b/migration/src/main/resources/reference.conf @@ -117,3 +117,4 @@ to { } } batchSize: 100 +force: false \ No newline at end of file diff --git a/migration/src/main/scala/org/thp/thehive/cloner/Cloner.scala b/migration/src/main/scala/org/thp/thehive/cloner/Cloner.scala index 5d15b3cadd..6b91022fc9 100644 --- a/migration/src/main/scala/org/thp/thehive/cloner/Cloner.scala +++ b/migration/src/main/scala/org/thp/thehive/cloner/Cloner.scala @@ -1,7 +1,7 @@ package org.thp.thehive.cloner import akka.actor.ActorSystem -import com.typesafe.config.{Config, ConfigFactory} +import com.typesafe.config.{Config, ConfigFactory, ConfigValueFactory} import org.apache.tinkerpop.gremlin.structure.T import org.thp.scalligraph.SingleInstance import org.thp.scalligraph.janus.JanusDatabase @@ -31,6 +31,9 @@ object Cloner extends App with IntegrityCheckApp { ) } + def addConfig(config: Config, path: String, value: Any): Config = + config.withValue(path, ConfigValueFactory.fromAnyRef(value)) + val defaultLoggerConfigFile = "/etc/thehive/logback-cloner.xml" if (System.getProperty("logger.file") == null && Files.exists(Paths.get(defaultLoggerConfigFile))) System.setProperty("logger.file", defaultLoggerConfigFile) @@ -54,7 +57,9 @@ object Cloner extends App with IntegrityCheckApp { .valueName("") .required() .action((f, c) => ConfigFactory.parseFileAnySyntax(f).withFallback(c)) - .text("configuration file") + .text("configuration file"), + opt[Unit]('f', "force") + .action((_, c) => addConfig(c, "force", true)) ) } val defaultConfig = @@ -78,19 +83,28 @@ object Cloner extends App with IntegrityCheckApp { val thehiveSchema = new TheHiveSchemaDefinition val cortexSchema = new CortexSchemaDefinition - if (sourceDatabase.version(thehiveSchema.name) != thehiveSchema.operations.operations.length + 1) { - println( - "The schema of TheHive is not valid " + - s"(found ${sourceDatabase.version(thehiveSchema.name)}, expected ${thehiveSchema.operations.operations.length + 1})" - ) - sys.exit(1) + + { + val expectedVersion = thehiveSchema.operations.operations.length + 1 + val foundVersion = sourceDatabase.version(thehiveSchema.name) + if (foundVersion != expectedVersion) { + println(s"The schema of TheHive is not valid (expected: $expectedVersion, found: $foundVersion)") + if (config.getBoolean("force")) + println("Continuing ...") + else + sys.exit(1) + } } - if (sourceDatabase.version(cortexSchema.name) != cortexSchema.operations.operations.length + 1) { - println( - "The schema of Cortex is not valid " + - s"(found ${sourceDatabase.version(cortexSchema.name)}, expected ${cortexSchema.operations.operations.length + 1})" - ) - sys.exit(1) + { + val expectedVersion = cortexSchema.operations.operations.length + 1 + val foundVersion = sourceDatabase.version(cortexSchema.name) + if (foundVersion != expectedVersion) { + println(s"The schema of Cortex is not valid (expected: $expectedVersion, found: $foundVersion)") + if (config.getBoolean("force")) + println("Continuing ...") + else + sys.exit(1) + } } val destDatabase: Database = getDatabase( @@ -111,8 +125,8 @@ object Cloner extends App with IntegrityCheckApp { // don't create initial values val models = destDatabase.extraModels ++ thehiveSchema.modelList ++ cortexSchema.modelList destDatabase.createSchema(models) - destDatabase.setVersion(thehiveSchema.name, thehiveSchema.operations.operations.length + 1) - destDatabase.setVersion(cortexSchema.name, cortexSchema.operations.operations.length + 1) + destDatabase.setVersion(thehiveSchema.name, sourceDatabase.version(thehiveSchema.name)) + destDatabase.setVersion(cortexSchema.name, sourceDatabase.version(cortexSchema.name)) val batchSize: Int = config.getInt("batchSize") @@ -167,6 +181,7 @@ object Cloner extends App with IntegrityCheckApp { println("Add indices ...") destDatabase.addSchemaIndexes(models) + println("Run checks ...") runChecks(destDatabase, Configuration(config)) destDatabase.close() } finally { From eb1bee3193a27a1fca4f18599312996c99307d81 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 1 Jul 2021 12:06:07 +0200 Subject: [PATCH 3/7] #2105 Fix injection of case number actor --- .../thp/thehive/cloner/IntegrityCheckApp.scala | 1 + thehive/app/org/thp/thehive/TheHiveModule.scala | 2 +- .../org/thp/thehive/services/CaseNumber.scala | 16 +++++++++++++--- .../app/org/thp/thehive/services/CaseSrv.scala | 14 ++++++++------ 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/migration/src/main/scala/org/thp/thehive/cloner/IntegrityCheckApp.scala b/migration/src/main/scala/org/thp/thehive/cloner/IntegrityCheckApp.scala index 1224be9325..61b4bcfca1 100644 --- a/migration/src/main/scala/org/thp/thehive/cloner/IntegrityCheckApp.scala +++ b/migration/src/main/scala/org/thp/thehive/cloner/IntegrityCheckApp.scala @@ -40,6 +40,7 @@ trait IntegrityCheckApp { bindActor[DummyActor]("config-actor") bindActor[DummyActor]("cortex-actor") bindActor[DummyActor]("integrity-check-actor") + bindTypedActor(CaseNumberActor.behavior, "case-number-actor") val integrityCheckOpsBindings = ScalaMultibinder.newSetBinder[GenIntegrityCheckOps](binder) integrityCheckOpsBindings.addBinding.to[ProfileIntegrityCheckOps] diff --git a/thehive/app/org/thp/thehive/TheHiveModule.scala b/thehive/app/org/thp/thehive/TheHiveModule.scala index 6667d840b2..988e8101b2 100644 --- a/thehive/app/org/thp/thehive/TheHiveModule.scala +++ b/thehive/app/org/thp/thehive/TheHiveModule.scala @@ -107,7 +107,7 @@ class TheHiveModule(environment: Environment, configuration: Configuration) exte integrityCheckOpsBindings.addBinding.to[ObservableIntegrityCheckOps] integrityCheckOpsBindings.addBinding.to[LogIntegrityCheckOps] bind[ActorRef].annotatedWithName("integrity-check-actor").toProvider[IntegrityCheckActorProvider] - bind[TypedActorRef[CaseNumberActor.Request]].annotatedWithName("case-number-actor").toProvider[CaseNumberActorProvider] + bind[TypedActorRef[CaseNumberActor.Request]].toProvider[CaseNumberActorProvider] bind[ActorRef].annotatedWithName("flow-actor").toProvider[FlowActorProvider] diff --git a/thehive/app/org/thp/thehive/services/CaseNumber.scala b/thehive/app/org/thp/thehive/services/CaseNumber.scala index ddfe2d900a..f25e242bb6 100644 --- a/thehive/app/org/thp/thehive/services/CaseNumber.scala +++ b/thehive/app/org/thp/thehive/services/CaseNumber.scala @@ -1,6 +1,6 @@ package org.thp.thehive.services -import akka.actor.typed.scaladsl.Behaviors +import akka.actor.typed.scaladsl.{ActorContext, Behaviors} import akka.actor.typed.scaladsl.adapter.ClassicActorSystemOps import akka.actor.typed.{ActorRefResolver, Behavior, ActorRef => TypedActorRef} import akka.actor.{ActorSystem, ExtendedActorSystem} @@ -22,15 +22,25 @@ object CaseNumberActor { case class GetNextNumber(replyTo: TypedActorRef[Response]) extends Request case class NextNumber(number: Int) extends Response - val behavior: Behavior[Request] = Behaviors.setup[Request] { context => + val behavior: Behavior[Request] = Behaviors.setup[Request](context => waitFirstRequest(context)) + + def getNextCaseNumber(context: ActorContext[Request]): Int = { val injector = GuiceAkkaExtension(context.system).injector val db = injector.getInstance(classOf[Database]) val caseSrv = injector.getInstance(classOf[CaseSrv]) db.roTransaction { implicit graph => - caseNumberProvider(caseSrv.startTraversal.getLast.headOption.fold(0)(_.number) + 1) + caseSrv.startTraversal.getLast.headOption.fold(0)(_.number) + 1 } } + def waitFirstRequest(context: ActorContext[Request]): Behaviors.Receive[Request] = + Behaviors.receiveMessage { + case GetNextNumber(replyTo) => + val nextNumber = getNextCaseNumber(context) + replyTo ! NextNumber(nextNumber) + caseNumberProvider(nextNumber + 1) + } + def caseNumberProvider(nextNumber: Int): Behavior[Request] = Behaviors.receiveMessage { case GetNextNumber(replyTo) => diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index a57cef3a73..683157ff50 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -1,8 +1,9 @@ package org.thp.thehive.services -import akka.actor.ActorRef import akka.actor.typed.scaladsl.AskPattern._ +import akka.actor.typed.scaladsl.adapter.ClassicSchedulerOps import akka.actor.typed.{Scheduler, ActorRef => TypedActorRef} +import akka.actor.{ActorRef, ActorSystem} import akka.util.Timeout import org.apache.tinkerpop.gremlin.process.traversal.{Order, P} import org.apache.tinkerpop.gremlin.structure.Vertex @@ -34,7 +35,7 @@ import java.lang.{Long => JLong} import java.util.{Date, List => JList, Map => JMap} import javax.inject.{Inject, Named, Provider, Singleton} import scala.concurrent.duration.DurationInt -import scala.concurrent.{Await, ExecutionContext, Future} +import scala.concurrent.{Await, ExecutionContextExecutor, Future} import scala.util.{Failure, Success, Try} @Singleton @@ -53,10 +54,9 @@ class CaseSrv @Inject() ( userSrv: UserSrv, alertSrvProvider: Provider[AlertSrv], @Named("integrity-check-actor") integrityCheckActor: ActorRef, - @Named("case-number-actor") caseNumberActor: TypedActorRef[CaseNumberActor.Request], + caseNumberActor: TypedActorRef[CaseNumberActor.Request], cache: SyncCacheApi, - implicit val ec: ExecutionContext, - implicit val scheduler: Scheduler + system: ActorSystem ) extends VertexSrv[Case] { lazy val alertSrv: AlertSrv = alertSrvProvider.get @@ -133,7 +133,9 @@ class CaseSrv @Inject() ( } def nextCaseNumberAsync: Future[Int] = { - implicit val timeout: Timeout = Timeout(1.minute) + implicit val timeout: Timeout = Timeout(1.minute) + implicit val scheduler: Scheduler = system.scheduler.toTyped + implicit val ec: ExecutionContextExecutor = system.dispatcher caseNumberActor.ask[CaseNumberActor.Response](replyTo => CaseNumberActor.GetNextNumber(replyTo)).map { case CaseNumberActor.NextNumber(caseNumber) => caseNumber } From 0cc0b912a8cb9bf7e9ad046e5b01ac69aa0274d7 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 1 Jul 2021 18:22:07 +0200 Subject: [PATCH 4/7] #2089 Fix injection of case number actor --- .../src/main/scala/org/thp/thehive/migration/th4/Output.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala index 12a8137888..2585c8e22c 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala @@ -52,6 +52,7 @@ object Output { bindActor[DummyActor]("config-actor") bindActor[DummyActor]("cortex-actor") bindActor[DummyActor]("integrity-check-actor") + bindTypedActor(CaseNumberActor.behavior, "case-number-actor") val schemaBindings = ScalaMultibinder.newSetBinder[UpdatableSchema](binder) schemaBindings.addBinding.to[TheHiveSchemaDefinition] From 25962bfaa7ff9d378c2f0c24e79db7ccaf530120 Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 5 Jul 2021 09:25:45 +0200 Subject: [PATCH 5/7] Prepare release --- CHANGELOG.md | 11 +++++++++++ build.sbt | 2 +- frontend/bower.json | 2 +- frontend/package.json | 2 +- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44b9983ee8..37e32a524a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Change Log +## [4.1.7](https://github.com/TheHive-Project/TheHive/milestone/76) (2021-07-05) + +**Implemented enhancements:** + +- [Enhancement] Copy the database even if the schema version doesn't match (with force flag) [\#2105](https://github.com/TheHive-Project/TheHive/issues/2105) + +**Fixed bugs:** + +- [Bug] Issue with Migration 3.5.1 -> 4.1.6 [\#2089](https://github.com/TheHive-Project/TheHive/issues/2089) +- [Bug] Fix serialization for case number messages [\#2107](https://github.com/TheHive-Project/TheHive/issues/2107) + ## [4.1.6](https://github.com/TheHive-Project/TheHive/milestone/75) (2021-06-14) **Implemented enhancements:** diff --git a/build.sbt b/build.sbt index 8db30d29ea..aeeceb03dc 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ import Dependencies._ import com.typesafe.sbt.packager.Keys.bashScriptDefines import org.thp.ghcl.Milestone -val thehiveVersion = "4.1.6-1" +val thehiveVersion = "4.1.7-1" val scala212 = "2.12.13" val scala213 = "2.13.1" val supportedScalaVersions = List(scala212, scala213) diff --git a/frontend/bower.json b/frontend/bower.json index 31b21b6795..8eda9b376d 100644 --- a/frontend/bower.json +++ b/frontend/bower.json @@ -1,6 +1,6 @@ { "name": "thehive", - "version": "4.1.6-1", + "version": "4.1.7-1", "license": "AGPL-3.0", "dependencies": { "jquery": "^3.4.1", diff --git a/frontend/package.json b/frontend/package.json index 1f6fa7f507..d653140327 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "thehive", - "version": "4.1.6-1", + "version": "4.1.7-1", "license": "AGPL-3.0", "repository": { "type": "git", From a95d6f77b6bab3533baa6441c4ae42a7389725c7 Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 5 Jul 2021 16:21:38 +0200 Subject: [PATCH 6/7] #2109 Fix case where entities were deleted in case of missing optional field --- ScalliGraph | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScalliGraph b/ScalliGraph index f1a647d4dd..ff5911920a 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit f1a647d4dd61f538d0d390f30d357e37d6a3aebc +Subproject commit ff5911920aca12e491878970e1a1061fc9998453 From 52cd6572b9247acb3ee5d039d0159f3b2f47e987 Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 5 Jul 2021 16:24:13 +0200 Subject: [PATCH 7/7] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37e32a524a..45f298a135 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - [Bug] Issue with Migration 3.5.1 -> 4.1.6 [\#2089](https://github.com/TheHive-Project/TheHive/issues/2089) - [Bug] Fix serialization for case number messages [\#2107](https://github.com/TheHive-Project/TheHive/issues/2107) +- [Bug] Case is removed if the assignee is removed [\#2109](https://github.com/TheHive-Project/TheHive/issues/2109) ## [4.1.6](https://github.com/TheHive-Project/TheHive/milestone/75) (2021-06-14)