From 2e02312039a4f328e6b21f0d97461d98ab69917f Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 14 Dec 2021 16:46:59 +0100 Subject: [PATCH 1/8] #2288 Use hash for big data --- ScalliGraph | 2 +- .../thp/thehive/migration/th4/Output.scala | 11 ++-- .../app/org/thp/thehive/TheHiveModule.scala | 6 ++- .../controllers/v0/ObservableCtrl.scala | 14 ++++- .../thehive/controllers/v1/Properties.scala | 14 ++++- .../org/thp/thehive/models/Observable.scala | 51 +++++++++++++++++-- .../thp/thehive/services/ObservableSrv.scala | 37 ++++++++------ .../org/thp/thehive/DatabaseBuilder.scala | 5 +- 8 files changed, 108 insertions(+), 32 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index f24ff5ed42..c1a5fb613b 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit f24ff5ed42f6c0b8ae7f9548af008bbc66fff337 +Subproject commit c1a5fb613ba4152455de77fa594e6e604aa0546b diff --git a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala index 5fdfdfd814..0eafba74e5 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala @@ -630,9 +630,11 @@ class Output @Inject() ( } yield IdMapping(inputLog.metaData.id, log._id) } - private def getData(value: String)(implicit graph: Graph, authContext: AuthContext): Try[Data with Entity] = - if (observableDataIsIndexed) dataSrv.create(Data(value)) - else dataSrv.createEntity(Data(value)) + private def getData(value: String)(implicit graph: Graph, authContext: AuthContext): Try[Data with Entity] = { + val (dataOrHash, fullData) = UseHashToIndex.hashToIndex(value).fold[(String, Option[String])](value -> None)(_ -> Some(value)) + if (observableDataIsIndexed) dataSrv.create(Data(dataOrHash, fullData)) + else dataSrv.createEntity(Data(dataOrHash, fullData)) + } private def createSimpleObservable(observable: Observable, observableType: ObservableType with Entity, dataValue: String)(implicit graph: Graph, @@ -700,7 +702,8 @@ class Output @Inject() ( richObservable <- createObservable(caseId, inputObservable, organisations.map(_._id).toSet) _ <- reportTagSrv.updateTags(richObservable, inputObservable.reportTags) case0 <- getCase(caseId) - _ <- organisations.toTry(o => shareSrv.shareObservable(RichObservable(richObservable, None, None, Nil), case0, o._id)) + // the data in richObservable is not set because it is not used in shareSrv + _ <- organisations.toTry(o => shareSrv.shareObservable(RichObservable(richObservable, None, None, None, Nil), case0, o._id)) } yield IdMapping(inputObservable.metaData.id, richObservable._id) } diff --git a/thehive/app/org/thp/thehive/TheHiveModule.scala b/thehive/app/org/thp/thehive/TheHiveModule.scala index 988e8101b2..e2ab245244 100644 --- a/thehive/app/org/thp/thehive/TheHiveModule.scala +++ b/thehive/app/org/thp/thehive/TheHiveModule.scala @@ -6,11 +6,11 @@ import com.google.inject.AbstractModule import net.codingwell.scalaguice.{ScalaModule, ScalaMultibinder} import org.thp.scalligraph.SingleInstance import org.thp.scalligraph.auth._ -import org.thp.scalligraph.janus.JanusDatabaseProvider +import org.thp.scalligraph.janus.{ImmenseTermProcessor, JanusDatabaseProvider} import org.thp.scalligraph.models.{Database, UpdatableSchema} import org.thp.scalligraph.services.{GenIntegrityCheckOps, HadoopStorageSrv, S3StorageSrv} import org.thp.thehive.controllers.v0.QueryExecutorVersion0Provider -import org.thp.thehive.models.TheHiveSchemaDefinition +import org.thp.thehive.models.{TheHiveSchemaDefinition, UseHashToIndex} import org.thp.thehive.services.notification.notifiers._ import org.thp.thehive.services.notification.triggers._ import org.thp.thehive.services.{UserSrv => _, _} @@ -112,6 +112,8 @@ class TheHiveModule(environment: Environment, configuration: Configuration) exte bind[ActorRef].annotatedWithName("flow-actor").toProvider[FlowActorProvider] bind[SingleInstance].to[ClusterSetup].asEagerSingleton() + + ImmenseTermProcessor.registerStrategy("observableHashToIndex", _ => UseHashToIndex) () } } diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala index 0ef5aceb1d..8853f0c9cc 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala @@ -2,13 +2,16 @@ package org.thp.thehive.controllers.v0 import net.lingala.zip4j.ZipFile import net.lingala.zip4j.model.FileHeader +import org.apache.tinkerpop.gremlin.process.traversal.Compare import org.thp.scalligraph._ import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers._ import org.thp.scalligraph.models.{Database, Entity, UMapping} +import org.thp.scalligraph.query.PredicateOps.PredicateOpsDefs import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} +import org.thp.scalligraph.utils.Hasher import org.thp.thehive.controllers.v0.Conversion._ import org.thp.thehive.dto.v0.{InputAttachment, InputObservable} import org.thp.thehive.models._ @@ -417,6 +420,7 @@ class PublicObservable @Inject() ( Query[Traversal.V[Observable], Traversal.V[Case]]("case", (observableSteps, _) => observableSteps.`case`), Query[Traversal.V[Observable], Traversal.V[Alert]]("alert", (observableSteps, _) => observableSteps.alert) ) + lazy val hasher: Hasher = Hasher("SHA-256") override val publicProperties: PublicProperties = PublicPropertyListBuilder[Observable] .property("status", UMapping.string)(_.select(_.constant("Ok")).readonly) .property("startDate", UMapping.date)(_.select(_._createdAt).readonly) @@ -445,7 +449,15 @@ class PublicObservable @Inject() ( _ <- observableSrv.updateType(observable, newDataType)(graph, authContext) } yield Json.obj("dataType" -> value) }) - .property("data", UMapping.string.optional)(_.field.readonly) + .property("data", UMapping.string.optional)( + _.select(_.value(_.data)) + .filter[String] { + case (_, observables, _, Right(predicate)) => observables.has(_.data, predicate.mapValue(v => UseHashToIndex.hashToIndex(v).getOrElse(v))) + case (_, observables, _, Left(true)) => observables.has(_.data) + case (_, observables, _, Left(false)) => observables.hasNot(_.data) + } + .readonly + ) .property("attachment.name", UMapping.string.optional)(_.select(_.attachments.value(_.name)).readonly) .property("attachment.hashes", UMapping.hash.sequence)(_.select(_.attachments.value(_.hashes)).readonly) .property("attachment.size", UMapping.long.optional)(_.select(_.attachments.value(_.size)).readonly) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index 588c83efc7..f07b9cbf7f 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -1,11 +1,13 @@ package org.thp.thehive.controllers.v1 +import org.apache.tinkerpop.gremlin.process.traversal.Compare import org.apache.tinkerpop.gremlin.structure.T import org.thp.scalligraph.controllers.{FPathElem, FPathEmpty, FString} import org.thp.scalligraph.models.{Database, UMapping} import org.thp.scalligraph.query.PredicateOps._ import org.thp.scalligraph.query.{PublicProperties, PublicPropertyListBuilder} import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.utils.Hasher import org.thp.scalligraph.{BadRequestError, EntityId, EntityIdOrName, InvalidFormatAttributeError, RichSeq} import org.thp.thehive.dto.v1.InputCustomFieldValue import org.thp.thehive.models._ @@ -28,7 +30,7 @@ import org.thp.thehive.services._ import play.api.libs.json.{JsObject, JsValue, Json} import javax.inject.{Inject, Singleton} -import scala.util.{Failure, Success, Try} +import scala.util.{Failure, Success} @Singleton class Properties @Inject() ( @@ -452,7 +454,15 @@ class Properties @Inject() ( _ <- observableSrv.updateType(observable, newDataType)(graph, authContext) } yield Json.obj("dataType" -> value) }) - .property("data", UMapping.string.optional)(_.field.readonly) + .property("data", UMapping.string.optional)( + _.select(_.value(_.data)) + .filter[String] { + case (_, observables, _, Right(predicate)) => observables.has(_.data, predicate.mapValue(v => UseHashToIndex.hashToIndex(v).getOrElse(v))) + case (_, observables, _, Left(true)) => observables.has(_.data) + case (_, observables, _, Left(false)) => observables.hasNot(_.data) + } + .readonly + ) .property("attachment.name", UMapping.string.optional)(_.select(_.attachments.value(_.name)).readonly) .property("attachment.hashes", UMapping.hash.sequence)(_.select(_.attachments.value(_.hashes)).readonly) .property("attachment.size", UMapping.long.optional)(_.select(_.attachments.value(_.size)).readonly) diff --git a/thehive/app/org/thp/thehive/models/Observable.scala b/thehive/app/org/thp/thehive/models/Observable.scala index b0a92a85e8..63d6b21c1c 100644 --- a/thehive/app/org/thp/thehive/models/Observable.scala +++ b/thehive/app/org/thp/thehive/models/Observable.scala @@ -1,9 +1,13 @@ package org.thp.thehive.models -import org.thp.scalligraph.models.{DefineIndex, Entity, IndexType} +import org.apache.tinkerpop.gremlin.structure.{Vertex, VertexProperty} +import org.thp.scalligraph.janus.{ImmenseStringTermFilter, ImmenseTermProcessor} +import org.thp.scalligraph.models.{DefineIndex, Entity, IndexType, UMapping} +import org.thp.scalligraph.utils.Hasher import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity, EntityId} import java.util.Date +import scala.util.Try @BuildEdgeEntity[Observable, KeyValue] case class ObservableKeyValue() @@ -46,6 +50,7 @@ case class Observable( case class RichObservable( observable: Observable with Entity, + fullData: Option[Data with Entity], attachment: Option[Attachment with Entity], seen: Option[Boolean], reportTags: Seq[ReportTag with Entity] @@ -60,12 +65,50 @@ case class RichObservable( def ioc: Boolean = observable.ioc def sighted: Boolean = observable.sighted def ignoreSimilarity: Option[Boolean] = observable.ignoreSimilarity - def dataOrAttachment: Either[String, Attachment with Entity] = observable.data.toLeft(attachment.get) + def dataOrAttachment: Either[String, Attachment with Entity] = data.toLeft(attachment.get) def dataType: String = observable.dataType - def data: Option[String] = observable.data + def data: Option[String] = fullData.map(d => d.fullData.getOrElse(d.data)) def tags: Seq[String] = observable.tags } @DefineIndex(IndexType.standard, "data") @BuildVertexEntity -case class Data(data: String) +case class Data(data: String, fullData: Option[String]) + +object UseHashToIndex extends ImmenseTermProcessor with ImmenseStringTermFilter { + override val termSizeLimit: Int = 8191 + private val hasher: Hasher = Hasher("SHA-256") + + def hashToIndex(value: String): Option[String] = + if (value.length > termSizeLimit) Some(hasher.fromString(value).head.toString) + else None + + override def apply[V](vertex: Vertex, property: VertexProperty[V]): Boolean = { + if (property.key() == "data") + vertex.label() match { + case "Observable" => + collect(vertex, property).foreach { strProp => + val currentValue = strProp.value() + logger.info(s"""Use hash for observable ~${vertex.id()}: + | dataType=${UMapping.string.getProperty(vertex, "dataType")} + | data=$currentValue + | message=${UMapping.string.optional.getProperty(vertex, "message").getOrElse("")} + | tags=${UMapping.string.sequence.getProperty(vertex, "message").mkString(", ")}""".stripMargin) + strProp.remove() + vertex.property(strProp.key(), hasher.fromString(currentValue).head.toString) + } + + case "Data" => + collect(vertex, property).foreach { strProp => + val currentValue = strProp.value() + logger.info(s"Use hash and move data for $vertex/${strProp.key()}: $currentValue") + strProp.remove() + vertex.property(strProp.key(), hasher.fromString(currentValue).head.toString) + vertex.property("fullData", currentValue) + } + + case _ => + } + false + } +} diff --git a/thehive/app/org/thp/thehive/services/ObservableSrv.scala b/thehive/app/org/thp/thehive/services/ObservableSrv.scala index 985b7a2302..91ffe952fb 100644 --- a/thehive/app/org/thp/thehive/services/ObservableSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableSrv.scala @@ -10,7 +10,7 @@ import org.thp.scalligraph.services._ import org.thp.scalligraph.traversal.Converter.Identity import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, Graph, StepLabel, Traversal} -import org.thp.scalligraph.utils.Hash +import org.thp.scalligraph.utils.{Hash, Hasher} import org.thp.scalligraph.{BadRequestError, CreateError, EntityId, EntityIdOrName, EntityName, RichSeq} import org.thp.thehive.models._ import org.thp.thehive.services.AlertOps._ @@ -76,7 +76,7 @@ class ObservableSrv @Inject() ( _ <- observableObservableTypeSrv.create(ObservableObservableType(), createdObservable, observableType) _ <- observableAttachmentSrv.create(ObservableAttachment(), createdObservable, attachment) _ <- tags.toTry(observableTagSrv.create(ObservableTag(), createdObservable, _)) - } yield RichObservable(createdObservable, Some(attachment), None, Nil) + } yield RichObservable(createdObservable, None, Some(attachment), None, Nil) } def create( @@ -86,10 +86,11 @@ class ObservableSrv @Inject() ( graph: Graph, authContext: AuthContext ): Try[RichObservable] = { + val (dataOrHash, fullData) = UseHashToIndex.hashToIndex(dataValue).fold[(String, Option[String])](dataValue -> None)(_ -> Some(dataValue)) val alreadyExists = startTraversal .has(_.organisationIds, organisationSrv.currentId) .has(_.relatedId, observable.relatedId) - .has(_.data, dataValue) + .has(_.data, dataOrHash) .has(_.dataType, observable.dataType) .exists if (alreadyExists) Failure(CreateError("Observable already exists")) @@ -100,12 +101,12 @@ class ObservableSrv @Inject() ( if (observableType.isAttachment) Failure(BadRequestError("A attachment observable doesn't accept string value")) else Success(()) tags <- observable.tags.toTry(tagSrv.getOrCreate) - data <- dataSrv.create(Data(dataValue)) - createdObservable <- createEntity(observable.copy(data = Some(dataValue))) + data <- dataSrv.create(Data(dataOrHash, fullData)) + createdObservable <- createEntity(observable.copy(data = Some(dataOrHash))) _ <- observableObservableTypeSrv.create(ObservableObservableType(), createdObservable, observableType) _ <- observableDataSrv.create(ObservableData(), createdObservable, data) _ <- tags.toTry(observableTagSrv.create(ObservableTag(), createdObservable, _)) - } yield RichObservable(createdObservable, None, None, Nil) + } yield RichObservable(createdObservable, Some(data), None, None, Nil) } def addTags(observable: Observable with Entity, tags: Set[String])(implicit graph: Graph, authContext: AuthContext): Try[Seq[Tag with Entity]] = { @@ -289,14 +290,16 @@ object ObservableOps { traversal .project( _.by - .by(_.attachments.fold) + .by(_.data.option) + .by(_.attachments.option) .by(_.reportTags.fold) ) .domainMap { - case (observable, attachment, reportTags) => + case (observable, data, attachment, reportTags) => RichObservable( observable, - attachment.headOption, + data, + attachment, None, reportTags ) @@ -308,15 +311,17 @@ object ObservableOps { traversal .project( _.by - .by(_.attachments.fold) + .by(_.data.option) + .by(_.attachments.option) .by(_.filteredSimilar.visible(organisationSrv).limit(1).count) .by(_.reportTags.fold) ) .domainMap { - case (observable, attachment, count, reportTags) => + case (observable, data, attachment, count, reportTags) => RichObservable( observable, - attachment.headOption, + data, + attachment, Some(count != 0), reportTags ) @@ -329,16 +334,18 @@ object ObservableOps { traversal .project( _.by - .by(_.attachments.fold) + .by(_.data.option) + .by(_.attachments.option) .by(_.filteredSimilar.visible(organisationSrv).limit(1).count) .by(_.reportTags.fold) .by(entityRenderer) ) .domainMap { - case (observable, attachment, count, reportTags, renderedEntity) => + case (observable, data, attachment, count, reportTags, renderedEntity) => RichObservable( observable, - attachment.headOption, + data, + attachment, Some(count != 0), reportTags ) -> renderedEntity diff --git a/thehive/test/org/thp/thehive/DatabaseBuilder.scala b/thehive/test/org/thp/thehive/DatabaseBuilder.scala index eb3190e31e..84303cbbcc 100644 --- a/thehive/test/org/thp/thehive/DatabaseBuilder.scala +++ b/thehive/test/org/thp/thehive/DatabaseBuilder.scala @@ -3,7 +3,7 @@ package org.thp.thehive import org.scalactic.Or import org.thp.scalligraph.auth.{AuthContext, AuthContextImpl} import org.thp.scalligraph.controllers._ -import org.thp.scalligraph.models.{Database, Entity, Schema} +import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.services.{EdgeSrv, GenIntegrityCheckOps, VertexSrv} import org.thp.scalligraph.traversal.Graph import org.thp.scalligraph.traversal.TraversalOps._ @@ -29,7 +29,6 @@ import scala.util.{Failure, Success, Try} @Singleton class DatabaseBuilder @Inject() ( - schema: Schema, alertSrv: AlertSrv, attachmentSrv: AttachmentSrv, caseSrv: CaseSrv, @@ -233,7 +232,7 @@ class DatabaseBuilder @Inject() ( dataSrv .getByName(data) .getOrFail("data") - .orElse(dataSrv.create(Data(data))) + .orElse(dataSrv.create(Data(data, None))) .flatMap(observableSrv.observableDataSrv.create(ObservableData(), observable, _)) .get ) From e20a23187d4ed5f16ca6ae10e27bd58153ff17bf Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 14 Dec 2021 18:20:36 +0100 Subject: [PATCH 2/8] #2288 Prefix hash with algorithm --- ScalliGraph | 2 +- .../app/org/thp/thehive/controllers/v0/ObservableCtrl.scala | 1 - thehive/app/org/thp/thehive/models/Observable.scala | 5 ++--- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index c1a5fb613b..3891a432b6 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit c1a5fb613ba4152455de77fa594e6e604aa0546b +Subproject commit 3891a432b6508da46a2db6d4595c6ed992c4f5da diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala index 8853f0c9cc..4ca40c5ddd 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala @@ -420,7 +420,6 @@ class PublicObservable @Inject() ( Query[Traversal.V[Observable], Traversal.V[Case]]("case", (observableSteps, _) => observableSteps.`case`), Query[Traversal.V[Observable], Traversal.V[Alert]]("alert", (observableSteps, _) => observableSteps.alert) ) - lazy val hasher: Hasher = Hasher("SHA-256") override val publicProperties: PublicProperties = PublicPropertyListBuilder[Observable] .property("status", UMapping.string)(_.select(_.constant("Ok")).readonly) .property("startDate", UMapping.date)(_.select(_._createdAt).readonly) diff --git a/thehive/app/org/thp/thehive/models/Observable.scala b/thehive/app/org/thp/thehive/models/Observable.scala index 63d6b21c1c..62902a69f0 100644 --- a/thehive/app/org/thp/thehive/models/Observable.scala +++ b/thehive/app/org/thp/thehive/models/Observable.scala @@ -7,7 +7,6 @@ import org.thp.scalligraph.utils.Hasher import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity, EntityId} import java.util.Date -import scala.util.Try @BuildEdgeEntity[Observable, KeyValue] case class ObservableKeyValue() @@ -80,7 +79,7 @@ object UseHashToIndex extends ImmenseTermProcessor with ImmenseStringTermFilter private val hasher: Hasher = Hasher("SHA-256") def hashToIndex(value: String): Option[String] = - if (value.length > termSizeLimit) Some(hasher.fromString(value).head.toString) + if (value.length > termSizeLimit) Some("sha256/" + hasher.fromString(value).head.toString) else None override def apply[V](vertex: Vertex, property: VertexProperty[V]): Boolean = { @@ -95,7 +94,7 @@ object UseHashToIndex extends ImmenseTermProcessor with ImmenseStringTermFilter | message=${UMapping.string.optional.getProperty(vertex, "message").getOrElse("")} | tags=${UMapping.string.sequence.getProperty(vertex, "message").mkString(", ")}""".stripMargin) strProp.remove() - vertex.property(strProp.key(), hasher.fromString(currentValue).head.toString) + vertex.property(strProp.key(), "sha256/" + hasher.fromString(currentValue).head.toString) } case "Data" => From 84cbcedc991149d32e46313bb45c220d4ccbbafb Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 16 Dec 2021 10:41:21 +0100 Subject: [PATCH 3/8] #2291 Remove unnecessary log4j dependency --- ScalliGraph | 2 +- project/Dependencies.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index 3891a432b6..ff0c21df8f 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 3891a432b6508da46a2db6d4595c6ed992c4f5da +Subproject commit ff0c21df8f84e779b88e4005971781ff4ebe94c6 diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 9c904e468a..0485f64973 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -35,7 +35,7 @@ object Dependencies { lazy val elastic4sHttpStreams = "com.sksamuel.elastic4s" %% "elastic4s-http-streams" % elastic4sVersion lazy val elastic4sClient = "com.sksamuel.elastic4s" %% "elastic4s-client-esjava" % elastic4sVersion lazy val reflections = "org.reflections" % "reflections" % "0.9.12" - lazy val hadoopClient = "org.apache.hadoop" % "hadoop-client" % "3.3.0" + lazy val hadoopClient = "org.apache.hadoop" % "hadoop-client" % "3.3.0" exclude ("log4j", "log4j") lazy val zip4j = "net.lingala.zip4j" % "zip4j" % "2.6.4" lazy val alpakka = "com.lightbend.akka" %% "akka-stream-alpakka-json-streaming" % "2.0.2" lazy val handlebars = "com.github.jknack" % "handlebars" % "4.2.0" From 8f3608c4c2f1349a00038a7480edd9d8c5784b0a Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 16 Dec 2021 11:37:07 +0100 Subject: [PATCH 4/8] #2291 Update logback --- ScalliGraph | 2 +- build.sbt | 5 +++-- project/Dependencies.scala | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index ff0c21df8f..e3d3fce06b 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit ff0c21df8f84e779b88e4005971781ff4ebe94c6 +Subproject commit e3d3fce06baec550c9597df4d9f2ced50bc527a2 diff --git a/build.sbt b/build.sbt index 61003d9d43..5752739f11 100644 --- a/build.sbt +++ b/build.sbt @@ -63,8 +63,9 @@ libraryDependencies in ThisBuild ++= { } dependencyOverrides in ThisBuild ++= Seq( // "org.locationtech.spatial4j" % "spatial4j" % "0.6", -// "org.elasticsearch.client" % "elasticsearch-rest-client" % "6.7.2" - akkaActor +// "org.elasticsearch.client" % "elasticsearch-rest-client" % "6.7.2 + akkaActor, + logbackClassic ) PlayKeys.includeDocumentationInBinary := false milestoneFilter := ((milestone: Milestone) => milestone.title.startsWith("4")) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 0485f64973..bfa3e79cf4 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -9,6 +9,7 @@ object Dependencies { lazy val playLogback = "com.typesafe.play" %% "play-logback" % play.core.PlayVersion.current lazy val playGuice = "com.typesafe.play" %% "play-guice" % play.core.PlayVersion.current lazy val playFilters = "com.typesafe.play" %% "filters-helpers" % play.core.PlayVersion.current + lazy val logbackClassic = "ch.qos.logback" % "logback-classic" % "1.2.8" lazy val playMockws = "de.leanovate.play-mockws" %% "play-mockws" % "2.8.0" lazy val akkaActor = "com.typesafe.akka" %% "akka-actor" % akkaVersion lazy val akkaCluster = "com.typesafe.akka" %% "akka-cluster" % akkaVersion From f0b14a4a7f652ecb588bbbd8bcdfcac5e30998a9 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Thu, 16 Dec 2021 15:09:45 +0100 Subject: [PATCH 5/8] #2282 Don't store filters of alert simlar cases sestion in localstorage --- .../app/scripts/components/alert/AlertSimilarCaseListCmp.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js b/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js index 4ed19defc1..9faa729ca3 100644 --- a/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js +++ b/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js @@ -39,7 +39,7 @@ this.filtering = new FilteringSrv('case', 'alert.dialog.similar-cases', { version: 'v1', defaults: { - showFilters: true, + showFilters: false, showStats: false, pageSize: 2, sort: ['-startDate'] @@ -47,7 +47,8 @@ defaultFilter: [] }); - self.filtering.initContext('alert.dialog.similar-cases') + //self.filtering.initContext('alert.dialog.similar-cases') + self.filtering.initContext() .then(function () { var defaultFilter = AlertingSrv.getSimilarityFilter(self.state.defaultAlertSimilarCaseFilter); From 949bf367b6b876694c46ddce86cba3bdef078964 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 16 Dec 2021 16:02:48 +0100 Subject: [PATCH 6/8] #2292 Set updateAt and updateBy in v0 of Alert --- thehive/app/org/thp/thehive/controllers/v0/Conversion.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala index babf09ca4f..934a544409 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala @@ -50,6 +50,8 @@ object Conversion { .withFieldComputed(_.customFields, rc => JsObject(rc.customFields.map(cf => cf.name -> Json.obj(cf.typeName -> cf.toJson)))) .withFieldRenamed(_._createdAt, _.createdAt) .withFieldRenamed(_._createdBy, _.createdBy) + .withFieldRenamed(_._updatedAt, _.updatedAt) + .withFieldRenamed(_._updatedBy, _.updatedBy) .withFieldComputed(_._id, _._id.toString) .withFieldComputed(_.id, _._id.toString) .withFieldComputed(_.id, _._id.toString) @@ -81,6 +83,8 @@ object Conversion { .withFieldComputed(_.id, _._id.toString) .withFieldRenamed(_._createdAt, _.createdAt) .withFieldRenamed(_._createdBy, _.createdBy) + .withFieldRenamed(_._updatedAt, _.updatedAt) + .withFieldRenamed(_._updatedBy, _.updatedBy) .withFieldConst(_._type, "alert") .withFieldComputed(_.tags, _.tags.toSet) .withFieldComputed(_.`case`, _.caseId.map(_.toString)) From ae991f5131b45d5952481f229d6a2c267e828b8c Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Thu, 16 Dec 2021 22:46:26 +0100 Subject: [PATCH 7/8] Update the index name used for drop and restore operation --- frontend/app/views/partials/admin/platform/status.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/views/partials/admin/platform/status.html b/frontend/app/views/partials/admin/platform/status.html index 6a25e82efd..fbc776d1eb 100644 --- a/frontend/app/views/partials/admin/platform/status.html +++ b/frontend/app/views/partials/admin/platform/status.html @@ -61,7 +61,7 @@

Data index status Reindex the data - + Drop and rebuild the index From c14fdaaff8f55af9f6b13c95efdc407a2c54f0ef Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 17 Dec 2021 08:34:26 +0100 Subject: [PATCH 8/8] Release 4.1.16 --- CHANGELOG.md | 13 +++++++++++++ build.sbt | 2 +- frontend/bower.json | 2 +- frontend/package.json | 2 +- 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60b3c26377..4936324ce4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Change Log +## [4.1.16](https://github.com/TheHive-Project/TheHive/milestone/86) (2021-12-17) + +**Implemented enhancements:** + +- [Feature Request] Remove persistent filters on "Similar Cases" tab [\#2282](https://github.com/TheHive-Project/TheHive/issues/2282) +- [Enhancement] When observable data is too big, use hash [\#2288](https://github.com/TheHive-Project/TheHive/issues/2288) +- Remove unnecessary log4j dependency [\#2291](https://github.com/TheHive-Project/TheHive/issues/2291) + +**Fixed bugs:** + +- [Bug] Index fails with immense terms [\#2289](https://github.com/TheHive-Project/TheHive/issues/2289) +- [Bug] Marking an alert as read do not update it's "updatedAt" nor "updatedBy" field [\#2292](https://github.com/TheHive-Project/TheHive/issues/2292) + ## [4.1.15](https://github.com/TheHive-Project/TheHive/milestone/85) (2021-12-06) **Implemented enhancements:** diff --git a/build.sbt b/build.sbt index 5752739f11..87487d86f2 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ import Dependencies._ import com.typesafe.sbt.packager.Keys.bashScriptDefines import org.thp.ghcl.Milestone -val thehiveVersion = "4.1.15-1" +val thehiveVersion = "4.1.16-1" val scala212 = "2.12.13" val scala213 = "2.13.1" val supportedScalaVersions = List(scala212, scala213) diff --git a/frontend/bower.json b/frontend/bower.json index d4252f1517..eb85ae583c 100644 --- a/frontend/bower.json +++ b/frontend/bower.json @@ -1,6 +1,6 @@ { "name": "thehive", - "version": "4.1.15-1", + "version": "4.1.16-1", "license": "AGPL-3.0", "dependencies": { "jquery": "^3.4.1", diff --git a/frontend/package.json b/frontend/package.json index 2cdc8bcad6..b0369ef0f4 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "thehive", - "version": "4.1.15-1", + "version": "4.1.16-1", "license": "AGPL-3.0", "repository": { "type": "git",