Merge branch 'release/3.4.0-RC2'

TheHive-Project · Jul 10, 2019 · 8b09be6 · 8b09be6
2 parents d3c15bb + 0f7fcbd
commit 8b09be6
Show file tree

Hide file tree

Showing 23 changed files with 218 additions and 72 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,30 @@
 # Change Log
 
+## [3.4.0-4C1](https://github.com/TheHive-Project/TheHive/tree/3.4.0-RC1) (2019-07-09)
+
+[Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.4.0-RC1...3.4.0-RC2)
+
+**Implemented enhancements:**
+
+- Display ioc and sighted attributes in Alert artifact list [\#1035](https://github.com/TheHive-Project/TheHive/issues/1035)
+- Merge Observable tags with existing observables during importing alerts into case [\#1014](https://github.com/TheHive-Project/TheHive/issues/1014)
+- API not recognizing the attribute 'sighted' of artifacts on alert creation [\#1003](https://github.com/TheHive-Project/TheHive/issues/1003)
+- Alerts are not getting deleted as expected [\#974](https://github.com/TheHive-Project/TheHive/issues/974)
+
+**Fixed bugs:**
+
+- Update case owner field validation to handle null value [\#1036](https://github.com/TheHive-Project/TheHive/issues/1036)
+- thehive prints error messages on first run \("Authentication failure" / "user init not found"\) [\#1027](https://github.com/TheHive-Project/TheHive/issues/1027)
+- TLP:WHITE for observable not shown, not editable [\#1025](https://github.com/TheHive-Project/TheHive/issues/1025)
+- Dashboard based on observables not refreshing correctly [\#996](https://github.com/TheHive-Project/TheHive/issues/996)
+- javascript error in tasks [\#979](https://github.com/TheHive-Project/TheHive/issues/979)
+- /api/alert/{}/createCase does not use caseTemplate [\#929](https://github.com/TheHive-Project/TheHive/issues/929)
+
+**Closed issues:**
+
+- Cannot add custom fields to case template [\#1042](https://github.com/TheHive-Project/TheHive/issues/1042)
+- sample hive does not connect to cortex and prints no helpful error message [\#1028](https://github.com/TheHive-Project/TheHive/issues/1028)
+
 ## [3.4.0-4C1](https://github.com/TheHive-Project/TheHive/tree/HEAD) (2019-06-05)
 
 [Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.3.1...3.4.0-4C1)

diff --git a/conf/application.sample b/conf/application.sample
@@ -8,10 +8,8 @@ search {
   ## Basic configuration
   # Index name.
   index = the_hive
-  # ElasticSearch cluster name.
-  cluster = hive
   # ElasticSearch instance address.
-  host = ["127.0.0.1:9300"]
+  uri = "http://127.0.0.1:9200/"
 
   ## Advanced configuration
   # Scroll keepalive.
@@ -28,33 +26,21 @@ search {
   #  mapping.nested_fields.limit = 100
   #}
 
-  ### XPack SSL configuration
-  # Username for XPack authentication
+  ## Authentication configuration
   #search.username = ""
-  # Password for XPack authentication
   #search.password = ""
-  # Enable SSL to connect to ElasticSearch
-  search.ssl.enabled = false
-  # Path to certificate authority file
-  #search.ssl.ca = ""
-  # Path to certificate file
-  #search.ssl.certificate = ""
-  # Path to key file
-  #search.ssl.key = ""
-
-  ### SearchGuard configuration
-  # Path to JKS file containing client certificate
-  #search.guard.keyStore.path = ""
-  # Password of the keystore
-  #search.guard.keyStore.password = ""
-  # Path to JKS file containing certificate authorities
-  #search.guard.trustStore.path = ""
-  ## Password of the truststore
-  #search.guard.trustStore.password = ""
-  # Enforce hostname verification
-  #search.guard.hostVerification = false
-  # If hostname verification is enabled specify if hostname should be resolved
-  #search.guard.hostVerificationResolveHostname = false
+
+  ## SSL configuration
+  #search.keyStore {
+  #  path = "/path/to/keystore"
+  #  type = "JKS" # or PKCS12
+  #  password = "keystore-password"
+  #}
+  #search.trustStore {
+  #  path = "/path/to/trustStore"
+  #  type = "JKS" # or PKCS12
+  #  password = "trustStore-password"
+  #}
 }
 
 # Authentication

diff --git a/docker/thehive/docker-compose.yml b/docker/thehive/docker-compose.yml
@@ -13,13 +13,13 @@ services:
         soft: 65536
         hard: 65536
   cortex:
-    image: thehiveproject/cortex:latest
+    image: thehiveproject/cortex:3.0.0-RC4
     depends_on:
       - elasticsearch
     ports:
       - "0.0.0.0:9001:9001"
   thehive:
-    image: thehiveproject/thehive:latest
+    image: thehiveproject/thehive:3.4.0-RC2
     depends_on:
       - elasticsearch
       - cortex

diff --git a/migration/12/dashboards/Observable_statistics .json b/migration/12/dashboards/Observable_statistics .json
@@ -1 +1 @@
-{"definition":{"period":"last3Months","items":[{"type":"container","items":[{"type":"donut","options":{"title":"Observables by type","entity":"case_artifact","field":"dataType","query":{},"names":{"fqdn":"fqdn","url":"url","regexp":"regexp","mail":"mail","hash":"hash","registry":"registry","uri_path":"uri_path","truc":"truc","ip":"ip","user-agent":"user-agent","autonomous-system":"autonomous-system","file":"file","mail_subject":"mail_subject","filename":"filename","other":"other","domain":"domain"}},"id":"6ee86a99-3f40-1960-fd4d-398a1da5b76e"},{"type":"donut","options":{"title":"Observables by attachment content type","entity":"case_artifact","field":"attachment.contentType","query":{"_field":"dataType","_value":"file"},"names":{},"filters":[{"field":"dataType","type":"enumeration","value":{"list":[{"text":"file","label":"file"}]}}]},"id":"b6110238-3074-4e85-674f-4bc56829e68a"}]},{"type":"container","items":[{"type":"donut","options":{"title":"Observable tags","entity":"case_artifact","field":"tags","query":{},"names":{}},"id":"70bbc0a5-1692-4e46-ebac-8769952ad9c0"},{"type":"donut","options":{"title":"Observables by TLP","entity":"case_artifact","field":"tlp","query":{},"names":{"0":"white","1":"green","2":"amber","3":"red"},"colors":{"0":"#bdf0ea","1":"#48e80f","2":"#e0a91a","3":"#f02626"}},"id":"633fbe97-805e-6123-3330-29f5c8f45f13"}]},{"type":"container","items":[{"type":"donut","options":{"title":"Observables by IOC flag","entity":"case_artifact","field":"ioc","query":{},"names":{}},"id":"771a3bdf-e437-ac3a-384d-23be91a25b07"},{"type":"line","options":{"title":"Observables over time","entity":"case_artifact","field":"createdAt","interval":"1w","series":[{"agg":"count","field":null,"type":"area-spline","filters":[{"field":"ioc","type":"boolean","value":true}],"label":"IOC","query":{"_field":"ioc","_value":true}},{"agg":"count","field":null,"type":"area-spline","label":"non-IOC","filters":[{"field":"ioc","type":"boolean","value":false}],"query":{"_field":"ioc","_value":false}}],"stacked":true,"query":{}},"id":"e5ed24a6-51ed-ecc4-9db0-ce837fd84214"}]}],"customPeriod":{"fromDate":null,"toDate":null}},"status":"Shared","title":"Observable statistics","description":"Observable statistics"}
+{"_routing":"AWu4YZXHg8tFuebkSwcG","description":"Observable statistics","title":"Observable statistics","_parent":null,"definition":{"period":"last3Months","items":[{"type":"container","items":[{"type":"donut","options":{"title":"Observables by type","entity":"case_artifact","field":"dataType","query":{"_not":{"_field":"status","_value":"Deleted"}},"names":{"fqdn":"fqdn","url":"url","regexp":"regexp","mail":"mail","hash":"hash","registry":"registry","uri_path":"uri_path","truc":"truc","ip":"ip","user-agent":"user-agent","autonomous-system":"autonomous-system","file":"file","mail_subject":"mail_subject","filename":"filename","other":"other","domain":"domain"},"filters":[{"field":"status","type":"enumeration","value":{"operator":"none","list":[{"text":"Deleted","label":"Deleted"}]}}]},"id":"6ee86a99-3f40-1960-fd4d-398a1da5b76e"},{"type":"donut","options":{"title":"Observables by attachment content type","entity":"case_artifact","field":"attachment.contentType","query":{"_and":[{"_field":"dataType","_value":"file"},{"_not":{"_field":"status","_value":"Deleted"}}]},"names":{},"filters":[{"field":"dataType","type":"enumeration","value":{"list":[{"text":"file","label":"file"}]}},{"field":"status","type":"enumeration","value":{"operator":"none","list":[{"text":"Deleted","label":"Deleted"}]}}]},"id":"b6110238-3074-4e85-674f-4bc56829e68a"}]},{"type":"container","items":[{"type":"donut","options":{"title":"Observable tags","entity":"case_artifact","field":"tags","query":{"_not":{"_field":"status","_value":"Deleted"}},"names":{},"filters":[{"field":"status","type":"enumeration","value":{"operator":"none","list":[{"text":"Deleted","label":"Deleted"}]}}]},"id":"70bbc0a5-1692-4e46-ebac-8769952ad9c0"},{"type":"donut","options":{"title":"Observables by TLP","entity":"case_artifact","field":"tlp","query":{"_not":{"_field":"status","_value":"Deleted"}},"names":{"0":"white","1":"green","2":"amber","3":"red"},"colors":{"0":"#bdf0ea","1":"#48e80f","2":"#e0a91a","3":"#f02626"},"filters":[{"field":"status","type":"enumeration","value":{"operator":"none","list":[{"text":"Deleted","label":"Deleted"}]}}]},"id":"633fbe97-805e-6123-3330-29f5c8f45f13"}]},{"type":"container","items":[{"type":"donut","options":{"title":"Observables by IOC flag","entity":"case_artifact","field":"ioc","query":{"_not":{"_field":"status","_value":"Deleted"}},"names":{},"filters":[{"field":"status","type":"enumeration","value":{"operator":"none","list":[{"text":"Deleted","label":"Deleted"}]}}]},"id":"771a3bdf-e437-ac3a-384d-23be91a25b07"},{"type":"line","options":{"title":"Observables over time","entity":"case_artifact","field":"createdAt","interval":"1w","series":[{"agg":"count","field":null,"type":"area-spline","filters":[{"field":"ioc","type":"boolean","value":true}],"label":"IOC","query":{"_field":"ioc","_value":true}},{"agg":"count","field":null,"type":"area-spline","label":"non-IOC","filters":[{"field":"ioc","type":"boolean","value":false}],"query":{"_field":"ioc","_value":false}}],"stacked":true,"query":{"_not":{"_field":"status","_value":"Deleted"}},"filters":[{"field":"status","type":"enumeration","value":{"operator":"none","list":[{"text":"Deleted","label":"Deleted"}]}}]},"id":"e5ed24a6-51ed-ecc4-9db0-ce837fd84214"}]}],"customPeriod":{"fromDate":null,"toDate":null}},"_id":"AWu4YZXHg8tFuebkSwcG","_version":3,"status":"Shared"}
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -20,7 +20,7 @@ object Dependencies {
 
     val reflections = "org.reflections" % "reflections" % "0.9.11"
     val zip4j = "net.lingala.zip4j" % "zip4j" % "1.3.2"
-    val elastic4play = "org.thehive-project" %% "elastic4play" % "1.11.3"
+    val elastic4play = "org.thehive-project" %% "elastic4play" % "1.11.4"
     val akkaCluster = "com.typesafe.akka" %% "akka-cluster" % "2.5.19"
     val akkaClusterTools = "com.typesafe.akka" %% "akka-cluster-tools" % "2.5.19"
   }

diff --git a/thehive-backend/app/controllers/AlertCtrl.scala b/thehive-backend/app/controllers/AlertCtrl.scala
@@ -119,12 +119,20 @@ class AlertCtrl @Inject()(
   }
 
   @Timed
-  def delete(id: String): Action[AnyContent] = authenticated(Roles.write).async { implicit request ⇒
+  def delete(id: String, force: Option[Boolean]): Action[AnyContent] = authenticated(Roles.write).async { implicit request ⇒
     alertSrv
-      .delete(id)
+      .delete(id, force.getOrElse(false))
       .map(_ ⇒ NoContent)
   }
 
+  @Timed
+  def bulkDelete(): Action[Fields] = authenticated(Roles.admin).async(fieldsBodyParser) { implicit request ⇒
+    request.body.getStrings("ids").fold(Future.successful(NoContent)) { ids ⇒
+      Future.traverse(ids)(alertSrv.delete(_, request.body.getBoolean("force").getOrElse(false)))
+        .map(_ => NoContent)
+    }
+  }
+
   @Timed
   def find(): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒
     val query     = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef])
@@ -173,6 +181,7 @@ class AlertCtrl @Inject()(
     for {
       alert ← alertSrv.get(id)
       customCaseTemplate = request.body.getString("caseTemplate")
+        .orElse(alert.caseTemplate())
       caze ← alertSrv.createCase(alert, customCaseTemplate)
     } yield renderer.toOutput(CREATED, caze)
   }

diff --git a/thehive-backend/app/models/Alert.scala b/thehive-backend/app/models/Alert.scala
@@ -54,6 +54,7 @@ trait AlertAttributes {
       Attribute("alert", "startDate", OptionalAttributeFormat(F.dateFmt), Nil, None, ""),
       Attribute("alert", "attachment", OptionalAttributeFormat(F.attachmentFmt), Nil, None, ""),
       Attribute("alert", "remoteAttachment", OptionalAttributeFormat(F.objectFmt(remoteAttachmentAttributes)), Nil, None, ""),
+      Attribute("alert", "sighted", OptionalAttributeFormat(F.booleanFmt), Nil, None, ""),
       Attribute("alert", "tlp", OptionalAttributeFormat(TlpAttributeFormat), Nil, None, ""),
       Attribute("alert", "tags", MultiAttributeFormat(F.stringFmt), Nil, None, ""),
       Attribute("alert", "ioc", OptionalAttributeFormat(F.booleanFmt), Nil, None, "")

diff --git a/thehive-backend/app/services/AlertSrv.scala b/thehive-backend/app/services/AlertSrv.scala
@@ -3,27 +3,27 @@ package services
 import java.nio.file.Files
 
 import scala.collection.immutable
-import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.{ ExecutionContext, Future }
 import scala.util.matching.Regex
-import scala.util.{Failure, Try}
+import scala.util.{ Failure, Success, Try }
 
 import play.api.libs.json._
-import play.api.{Configuration, Logger}
+import play.api.{ Configuration, Logger }
 
 import akka.NotUsed
 import akka.stream.Materializer
-import akka.stream.scaladsl.{Sink, Source}
+import akka.stream.scaladsl.{ Sink, Source }
 import connectors.ConnectorRouter
-import javax.inject.{Inject, Singleton}
+import javax.inject.{ Inject, Singleton }
 import models._
 
-import org.elastic4play.InternalError
-import org.elastic4play.controllers.{Fields, FileInputValue}
+import org.elastic4play.controllers.{ Fields, FileInputValue }
 import org.elastic4play.database.ModifyConfig
 import org.elastic4play.services.JsonFormat.attachmentFormat
-import org.elastic4play.services.QueryDSL.{groupByField, parent, selectCount, withId}
+import org.elastic4play.services.QueryDSL.{ groupByField, parent, selectCount, withId }
 import org.elastic4play.services._
 import org.elastic4play.utils.Collection
+import org.elastic4play.{ ConflictError, InternalError }
 
 trait AlertTransformer {
   def createCase(alert: Alert, customCaseTemplate: Option[String])(implicit authContext: AuthContext): Future[Case]
@@ -240,9 +240,17 @@ class AlertSrv(
           case _ ⇒
             for {
               _ ← importArtifacts(alert, caze)
-              description = caze.description() + s"\n  \n#### Merged with alert #${alert.sourceRef()} ${alert.title()}\n\n${alert.description().trim}"
-              updatedCase ← caseSrv.update(caze, Fields.empty.set("description", description))
-              _           ← setCase(alert, caze)
+              newDescription = caze
+                .description() + s"\n  \n#### Merged with alert #${alert.sourceRef()} ${alert.title()}\n\n${alert.description().trim}"
+              newTags = (caze.tags() ++ alert.tags()).distinct.map(JsString.apply)
+              updatedCase ← caseSrv.update(
+                caze,
+                Fields
+                  .empty
+                  .set("description", newDescription)
+                  .set("tags", JsArray(newTags))
+              )
+              _ ← setCase(alert, caze)
             } yield updatedCase
         }
     }
@@ -257,7 +265,16 @@ class AlertSrv(
       }
       .flatMap { _ ⇒ // then merge all tags
         val newTags = (caze.tags() ++ alerts.flatMap(_.tags())).distinct.map(JsString.apply)
-        caseSrv.update(caze, Fields.empty.set("tags", JsArray(newTags)))
+        val newDescription = caze.description() + alerts
+          .map(alert ⇒ s"\n  \n#### Merged with alert #${alert.sourceRef()} ${alert.title()}\n\n${alert.description().trim}")
+          .mkString("")
+        caseSrv.update(
+          caze,
+          Fields
+            .empty
+            .set("description", newDescription)
+            .set("tags", JsArray(newTags))
+        )
       }
 
   def importArtifacts(alert: Alert, caze: Case)(implicit authContext: AuthContext): Future[Case] = {
@@ -293,12 +310,37 @@ class AlertSrv(
 
     val updatedCase = artifactSrv
       .create(caze, artifactsFields)
-      .map { artifacts ⇒
-        artifacts.collect {
-          case Failure(e) ⇒ logger.warn("Create artifact error", e)
+      .flatMap { artifacts ⇒
+        Future.traverse(artifacts) {
+          case Success(_) => Future.successful(())
+          case Failure(ConflictError(_, attributes)) ⇒ // if it already exists, add tags from alert
+            import org.elastic4play.services.QueryDSL._
+            (for {
+              dataType ← (attributes \ "dataType").asOpt[String]
+              data       = (attributes \ "data").asOpt[String]
+              attachment = (attributes \ "attachment").asOpt[Attachment]
+              tags ← (attributes \ "tags").asOpt[Seq[String]]
+              _    ← data orElse attachment
+              dataOrAttachment = data.toLeft(attachment.get)
+            } yield artifactSrv
+              .find(artifactSrv.similarArtifactFilter(dataType, dataOrAttachment, withParent(caze)), None, Nil)
+              ._1
+              .mapAsyncUnordered(1) { artifact ⇒
+                artifactSrv.update(artifact.id, Fields.empty.set("tags", JsArray((artifact.tags() ++ tags).distinct.map(JsString.apply))))
+              }
+              .map(_ ⇒ caze)
+              .runWith(Sink.ignore)
+              .map(_ ⇒ caze))
+              .getOrElse {
+                logger.warn(s"A conflict error occurs when creating the artifact $attributes but it doesn't exist")
+                Future.successful(())
+              }
+          case Failure(e) ⇒
+            logger.warn("Create artifact error", e)
+            Future.successful(())
         }
-        caze
       }
+        .map(_ => caze)
     updatedCase.onComplete { _ ⇒
       // remove temporary files
       artifactsFields
@@ -325,8 +367,9 @@ class AlertSrv(
     updateSrv(alert, Fields(Json.obj("case" → JsNull, "status" → status)), modifyConfig)
   }
 
-  def delete(id: String)(implicit authContext: AuthContext): Future[Alert] =
-    deleteSrv[AlertModel, Alert](alertModel, id)
+  def delete(id: String, force: Boolean)(implicit authContext: AuthContext): Future[Unit] =
+    if (force) deleteSrv.realDelete[AlertModel, Alert](alertModel, id)
+    else get(id).flatMap(alert ⇒ markAsUnread(alert)).map(_ ⇒ ())
 
   def find(queryDef: QueryDef, range: Option[String], sortBy: Seq[String]): (Source[Alert, NotUsed], Future[Long]) =
     findSrv[AlertModel, Alert](alertModel, queryDef, range, sortBy)

diff --git a/thehive-backend/app/services/CaseSrv.scala b/thehive-backend/app/services/CaseSrv.scala
@@ -60,7 +60,7 @@ class CaseSrv @Inject()(
   }
 
   def create(fields: Fields, template: Option[CaseTemplate] = None)(implicit authContext: AuthContext): Future[Case] = {
-    val fieldsWithOwner = fields.get("owner") match {
+    val fieldsWithOwner = fields.getString("owner") match {
       case None    ⇒ fields.set("owner", authContext.userId)
       case Some(_) ⇒ fields
     }

diff --git a/thehive-backend/conf/routes b/thehive-backend/conf/routes
@@ -64,14 +64,15 @@ GET      /api/alert/_fixStatus                    controllers.AlertCtrl.fixStatu
 POST     /api/alert                               controllers.AlertCtrl.create()
 GET      /api/alert/:alertId                      controllers.AlertCtrl.get(alertId)
 PATCH    /api/alert/:alertId                      controllers.AlertCtrl.update(alertId)
-DELETE   /api/alert/:alertId                      controllers.AlertCtrl.delete(alertId)
+DELETE   /api/alert/:alertId                      controllers.AlertCtrl.delete(alertId, force: Option[Boolean])
 POST     /api/alert/:alertId/markAsRead           controllers.AlertCtrl.markAsRead(alertId)
 POST     /api/alert/:alertId/markAsUnread         controllers.AlertCtrl.markAsUnread(alertId)
 POST     /api/alert/:alertId/createCase           controllers.AlertCtrl.createCase(alertId)
 POST     /api/alert/:alertId/follow               controllers.AlertCtrl.followAlert(alertId)
 POST     /api/alert/:alertId/unfollow             controllers.AlertCtrl.unfollowAlert(alertId)
 POST     /api/alert/:alertId/merge/:caseId        controllers.AlertCtrl.mergeWithCase(alertId, caseId)
 POST     /api/alert/merge/_bulk                   controllers.AlertCtrl.bulkMergeWithCase()
+POST     /api/alert/delete/_bulk                  controllers.AlertCtrl.bulkDelete()
 
 GET      /api/flow                                controllers.AuditCtrl.flow(rootId: Option[String], count: Option[Int])
 GET      /api/audit                               controllers.AuditCtrl.find()