Merge branch 'release/3.4.0'

CHANGELOG.md

@@ -1,7 +1,25 @@
 # Change Log
 
-## [3.4.0-4C1](https://github.com/TheHive-Project/TheHive/tree/3.4.0-RC1) (2019-07-09)
+## [3.4.0](https://github.com/TheHive-Project/TheHive/tree/HEAD) (2019-09-05)
 
+[Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.4.0-RC2...3.4.0)
+
+**Implemented enhancements:**
+
+- Removing custom fields [\#954](https://github.com/TheHive-Project/TheHive/issues/954)
+
+**Fixed bugs:**
+
+- Cosmetic Bug: wrong number of exported observables displayed [\#1071](https://github.com/TheHive-Project/TheHive/issues/1071)
+- Update Database button does not appear in training appliance [\#1067](https://github.com/TheHive-Project/TheHive/issues/1067)
+- bulk merge alerts into case lose description's alert [\#1065](https://github.com/TheHive-Project/TheHive/issues/1065)
+- Incorrect number of related observables returned [\#1062](https://github.com/TheHive-Project/TheHive/issues/1062)
+- Incorrect tag filter results when observables with tags are added then deleted [\#1061](https://github.com/TheHive-Project/TheHive/issues/1061)
+- Cannot setup TheHive 3.4.0-RC2 using Docker [\#1051](https://github.com/TheHive-Project/TheHive/issues/1051)
+- Case statistics dashboard loads with an error message and the case over time panel fails to display any data [\#1050](https://github.com/TheHive-Project/TheHive/issues/1050)
+- Can't secure ElasticSearch connection [\#1046](https://github.com/TheHive-Project/TheHive/issues/1046)
+
+## [3.4.0-RC2](https://github.com/TheHive-Project/TheHive/tree/3.4.0-RC2) (2019-07-10)
 [Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.4.0-RC1...3.4.0-RC2)
 
 **Implemented enhancements:**
@@ -25,7 +43,7 @@
 - Cannot add custom fields to case template [\#1042](https://github.com/TheHive-Project/TheHive/issues/1042)
 - sample hive does not connect to cortex and prints no helpful error message [\#1028](https://github.com/TheHive-Project/TheHive/issues/1028)
 
-## [3.4.0-4C1](https://github.com/TheHive-Project/TheHive/tree/HEAD) (2019-06-05)
+## [3.4.0-RC1](https://github.com/TheHive-Project/TheHive/tree/HEAD) (2019-06-05)
 
 [Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.3.1...3.4.0-4C1)
 

project/Dependencies.scala

@@ -20,8 +20,8 @@ object Dependencies {
 
     val reflections = "org.reflections" % "reflections" % "0.9.11"
     val zip4j = "net.lingala.zip4j" % "zip4j" % "1.3.2"
-    val elastic4play = "org.thehive-project" %% "elastic4play" % "1.11.4"
-    val akkaCluster = "com.typesafe.akka" %% "akka-cluster" % "2.5.19"
-    val akkaClusterTools = "com.typesafe.akka" %% "akka-cluster-tools" % "2.5.19"
+    val elastic4play = "org.thehive-project" %% "elastic4play" % "1.11.5"
+    val akkaCluster = "com.typesafe.akka" %% "akka-cluster" % "2.5.21"
+    val akkaClusterTools = "com.typesafe.akka" %% "akka-cluster-tools" % "2.5.21"
   }
 }

project/plugins.sbt

@@ -1,6 +1,6 @@
 // Comment to get more information during initialization
 logLevel := Level.Info
 
-addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.6.22")
+addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.6.23")
 addSbtPlugin("org.foundweekends" % "sbt-bintray" % "0.5.1")
 addSbtPlugin("org.scalameta"     % "sbt-scalafmt" % "2.0.0")

thehive-backend/app/controllers/AlertCtrl.scala

@@ -128,8 +128,9 @@ class AlertCtrl @Inject()(
   @Timed
   def bulkDelete(): Action[Fields] = authenticated(Roles.admin).async(fieldsBodyParser) { implicit request ⇒
     request.body.getStrings("ids").fold(Future.successful(NoContent)) { ids ⇒
-      Future.traverse(ids)(alertSrv.delete(_, request.body.getBoolean("force").getOrElse(false)))
-        .map(_ => NoContent)
+      Future
+        .traverse(ids)(alertSrv.delete(_, request.body.getBoolean("force").getOrElse(false)))
+        .map(_ ⇒ NoContent)
     }
   }
 
@@ -180,7 +181,9 @@ class AlertCtrl @Inject()(
   def createCase(id: String): Action[Fields] = authenticated(Roles.write).async(fieldsBodyParser) { implicit request ⇒
     for {
       alert ← alertSrv.get(id)
-      customCaseTemplate = request.body.getString("caseTemplate")
+      customCaseTemplate = request
+        .body
+        .getString("caseTemplate")
         .orElse(alert.caseTemplate())
       caze ← alertSrv.createCase(alert, customCaseTemplate)
     } yield renderer.toOutput(CREATED, caze)

thehive-backend/app/controllers/CustomFieldsCtrl.scala

@@ -0,0 +1,56 @@
+package controllers
+
+import scala.concurrent.{ExecutionContext, Future}
+
+import play.api.http.Status
+import play.api.libs.json.{JsNumber, JsObject, Json}
+import play.api.mvc.{AbstractController, Action, AnyContent, ControllerComponents}
+
+import akka.stream.Materializer
+import akka.stream.scaladsl.Sink
+import com.sksamuel.elastic4s.http.ElasticDsl.{search, termsAggregation}
+import javax.inject.{Inject, Singleton}
+import models.Roles
+
+import org.elastic4play.NotFoundError
+import org.elastic4play.controllers.Authenticated
+import org.elastic4play.database.DBFind
+import org.elastic4play.services.DBLists
+import org.elastic4play.services.QueryDSL._
+
+@Singleton
+class CustomFieldsCtrl @Inject()(
+    authenticated: Authenticated,
+    dbfind: DBFind,
+    dblists: DBLists,
+    components: ControllerComponents,
+    implicit val ec: ExecutionContext,
+    implicit val mat: Materializer
+) extends AbstractController(components)
+    with Status {
+
+  def useCount(customField: String): Action[AnyContent] =
+    authenticated(Roles.read)
+      .async {
+        dblists("custom_fields")
+          .getItems[JsObject]
+          ._1
+          .collect {
+            case (_, value) if (value \ "reference").asOpt[String].contains(customField) ⇒ (value \ "type").as[String]
+          }
+          .runWith(Sink.head)
+          .recoverWith { case _ ⇒ Future.failed(NotFoundError(s"CustomField $customField not found")) }
+          .flatMap { customFieldType ⇒
+            val filter = and("relations" in ("case", "alert", "caseTemplate"), contains(s"customFields.$customField.$customFieldType"))
+            dbfind(
+              indexName ⇒ search(indexName).query(filter.query).aggregations(termsAggregation("t").field("relations"))
+            ).map { searchResponse ⇒
+                val buckets = searchResponse.aggregations.terms("t").buckets
+                val total   = buckets.map(_.docCount).sum
+                val result  = buckets.map(b ⇒ b.key → JsNumber(b.docCount)) :+ ("total" → JsNumber(total))
+                Ok(JsObject(result))
+              }
+              .recover { case _ ⇒ Ok(Json.obj("total" → 0)) }
+          }
+      }
+}

thehive-backend/app/controllers/DBListCtrl.scala

@@ -1,14 +1,14 @@
-package org.elastic4play.controllers
-
-import javax.inject.{Inject, Singleton}
+package controllers
 
 import scala.concurrent.{ExecutionContext, Future}
 
 import play.api.libs.json.{JsValue, Json}
 import play.api.mvc._
 
+import javax.inject.{Inject, Singleton}
 import models.Roles
 
+import org.elastic4play.controllers.{Authenticated, Fields, FieldsBodyParser, Renderer}
 import org.elastic4play.services.DBLists
 import org.elastic4play.{MissingAttributeError, Timed}
 

thehive-backend/app/controllers/StatusCtrl.scala

@@ -1,40 +1,48 @@
 package controllers
 
+import akka.actor.ActorSystem
+
 import scala.collection.immutable
 import scala.concurrent.ExecutionContext
 import scala.util.Try
-
 import play.api.Configuration
 import play.api.libs.json.Json.toJsFieldJsValueWrapper
 import play.api.libs.json.{JsBoolean, JsObject, JsString, Json}
 import play.api.mvc.{AbstractController, Action, AnyContent, ControllerComponents}
-
 import com.sksamuel.elastic4s.http.ElasticDsl
 import connectors.Connector
 import javax.inject.{Inject, Singleton}
 import models.HealthStatus
 import org.elasticsearch.client.Node
-
 import org.elastic4play.Timed
 import org.elastic4play.database.DBIndex
 import org.elastic4play.services.AuthSrv
 import org.elastic4play.services.auth.MultiAuthSrv
 
+import scala.concurrent.duration.{DurationInt, FiniteDuration}
+
 @Singleton
 class StatusCtrl @Inject()(
     connectors: immutable.Set[Connector],
     configuration: Configuration,
     dbIndex: DBIndex,
     authSrv: AuthSrv,
+    system: ActorSystem,
     components: ControllerComponents,
     implicit val ec: ExecutionContext
 ) extends AbstractController(components) {
 
   private[controllers] def getVersion(c: Class[_]) = Option(c.getPackage.getImplementationVersion).getOrElse("SNAPSHOT")
+  private var clusterStatusName: String            = "Init"
+  val checkStatusInterval: FiniteDuration          = configuration.getOptional[FiniteDuration]("statusCheckInterval").getOrElse(1.minute)
+  private def updateStatus(): Unit = {
+    clusterStatusName = Try(dbIndex.clusterStatusName).getOrElse("ERROR")
+    system.scheduler.scheduleOnce(checkStatusInterval)(updateStatus())
+  }
+  updateStatus()
 
   @Timed("controllers.StatusCtrl.get")
   def get: Action[AnyContent] = Action {
-    val clusterStatusName = Try(dbIndex.clusterStatusName).getOrElse("ERROR")
     Ok(
       Json.obj(
         "versions" → Json.obj(

thehive-backend/app/controllers/StreamCtrl.scala

@@ -23,7 +23,7 @@ import services.StreamActor
 import services.StreamActor.StreamMessages
 
 import org.elastic4play.controllers._
-import org.elastic4play.services.{AuxSrv, EventSrv, MigrationSrv}
+import org.elastic4play.services.{AuxSrv, EventSrv, MigrationSrv, UserSrv}
 import org.elastic4play.Timed
 
 @Singleton
@@ -33,6 +33,7 @@ class StreamCtrl(
     authenticated: Authenticated,
     renderer: Renderer,
     eventSrv: EventSrv,
+    userSrv: UserSrv,
     auxSrv: AuxSrv,
     migrationSrv: MigrationSrv,
     components: ControllerComponents,
@@ -46,6 +47,7 @@ class StreamCtrl(
       authenticated: Authenticated,
       renderer: Renderer,
       eventSrv: EventSrv,
+      userSrv: UserSrv,
       auxSrv: AuxSrv,
       migrationSrv: MigrationSrv,
       components: ControllerComponents,
@@ -58,6 +60,7 @@ class StreamCtrl(
       authenticated,
       renderer,
       eventSrv,
+      userSrv,
       auxSrv,
       migrationSrv,
       components,
@@ -96,9 +99,10 @@ class StreamCtrl(
       Future.successful(BadRequest("Invalid stream id"))
     } else {
       val futureStatus = authenticated.expirationStatus(request) match {
-        case ExpirationError if !migrationSrv.isMigrating ⇒ authenticated.getFromApiKey(request).map(_ ⇒ OK)
-        case _: ExpirationWarning                         ⇒ Future.successful(220)
-        case _                                            ⇒ Future.successful(OK)
+        case ExpirationError if !migrationSrv.isMigrating ⇒
+          userSrv.getInitialUser(request).recoverWith { case _ ⇒ authenticated.getFromApiKey(request) }.map(_ ⇒ OK)
+        case _: ExpirationWarning ⇒ Future.successful(220)
+        case _                    ⇒ Future.successful(OK)
       }
 
       // Check if stream actor exists

thehive-backend/app/services/AlertSrv.scala

@@ -3,27 +3,27 @@ package services
 import java.nio.file.Files
 
 import scala.collection.immutable
-import scala.concurrent.{ ExecutionContext, Future }
+import scala.concurrent.{ExecutionContext, Future}
 import scala.util.matching.Regex
-import scala.util.{ Failure, Success, Try }
+import scala.util.{Failure, Success, Try}
 
 import play.api.libs.json._
-import play.api.{ Configuration, Logger }
+import play.api.{Configuration, Logger}
 
 import akka.NotUsed
 import akka.stream.Materializer
-import akka.stream.scaladsl.{ Sink, Source }
+import akka.stream.scaladsl.{Sink, Source}
 import connectors.ConnectorRouter
-import javax.inject.{ Inject, Singleton }
+import javax.inject.{Inject, Singleton}
 import models._
 
-import org.elastic4play.controllers.{ Fields, FileInputValue }
+import org.elastic4play.controllers.{Fields, FileInputValue}
 import org.elastic4play.database.ModifyConfig
 import org.elastic4play.services.JsonFormat.attachmentFormat
-import org.elastic4play.services.QueryDSL.{ groupByField, parent, selectCount, withId }
+import org.elastic4play.services.QueryDSL.{groupByField, parent, selectCount, withId}
 import org.elastic4play.services._
 import org.elastic4play.utils.Collection
-import org.elastic4play.{ ConflictError, InternalError }
+import org.elastic4play.{ConflictError, InternalError}
 
 trait AlertTransformer {
   def createCase(alert: Alert, customCaseTemplate: Option[String])(implicit authContext: AuthContext): Future[Case]
@@ -312,7 +312,7 @@ class AlertSrv(
       .create(caze, artifactsFields)
       .flatMap { artifacts ⇒
         Future.traverse(artifacts) {
-          case Success(_) => Future.successful(())
+          case Success(_) ⇒ Future.successful(())
           case Failure(ConflictError(_, attributes)) ⇒ // if it already exists, add tags from alert
             import org.elastic4play.services.QueryDSL._
             (for {
@@ -340,7 +340,7 @@ class AlertSrv(
             Future.successful(())
         }
       }
-        .map(_ => caze)
+      .map(_ ⇒ caze)
     updatedCase.onComplete { _ ⇒
       // remove temporary files
       artifactsFields