Merge branch 'hotfix/4.1.18'

Author: To-om

CHANGELOG.md

@@ -1,5 +1,20 @@
 # Change Log
 
+## [4.1.18](https://github.com/TheHive-Project/TheHive/milestone/88) (2022-02-07)
+
+**Implemented enhancements:**
+
+- [Enhancement] Integrity check improvement [\#2334](https://github.com/TheHive-Project/TheHive/issues/2334)
+- [Enhancement] Improve migration tool [\#2335](https://github.com/TheHive-Project/TheHive/issues/2335)
+
+**Fixed bugs:**
+
+- [Bug] "Character 8211 cannot match AsciiSet because it is out of range" error when downloading a report [\#1534](https://github.com/TheHive-Project/TheHive/issues/1534)
+- [Bug] Can add a "space" as observable [\#2324](https://github.com/TheHive-Project/TheHive/issues/2324)
+- [Bug]- Migration from Hive 3.4.4 to Hive 4.1.17 not working [\#2331](https://github.com/TheHive-Project/TheHive/issues/2331)
+- [Bug] Duplicated entities after "db.janusgraph.forceDropAndRebuildIndex: true" with Elasticsearch index [\#2333](https://github.com/TheHive-Project/TheHive/issues/2333)
+- [Bug] Query with parendId filter doesn't work (v0) [\#2336](https://github.com/TheHive-Project/TheHive/issues/2336)
+
 ## [4.1.17](https://github.com/TheHive-Project/TheHive/milestone/87) (2022-01-24)
 
 **Implemented enhancements:**

ScalliGraph

@@ -2,7 +2,7 @@ import Dependencies._
 import com.typesafe.sbt.packager.Keys.bashScriptDefines
 import org.thp.ghcl.Milestone
 
-val thehiveVersion         = "4.1.17-1"
+val thehiveVersion         = "4.1.18-1"
 val scala212               = "2.12.13"
 val scala213               = "2.13.1"
 val supportedScalaVersions = List(scala212, scala213)
@@ -165,7 +165,8 @@ lazy val thehiveCore = (project in file("thehive"))
       pbkdf2,
       commonCodec,
       scalaGuice,
-      reflections
+      reflections,
+      quartzScheduler
     )
   )
 

build.sbt

@@ -50,7 +50,7 @@ class CortexQueryExecutor @Inject() (
 
   override val customFilterQuery: FilterQuery = FilterQuery(publicProperties) { (tpe, globalParser) =>
     FieldsParser("parentChildFilter") {
-      case (_, FObjOne("_parent", ParentIdFilter(_, parentId))) if parentTypes.isDefinedAt(tpe) =>
+      case (_, FObjOne("_parent", ParentIdFilter(parentId, _))) if parentTypes.isDefinedAt(tpe) =>
         Good(new CortexParentIdInputFilter(parentId))
       case (path, FObjOne("_parent", ParentQueryFilter(_, parentFilterField))) if parentTypes.isDefinedAt(tpe) =>
         globalParser(parentTypes(tpe)).apply(path, parentFilterField).map(query => new CortexParentQueryInputFilter(query))

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/CortexQueryExecutor.scala

@@ -1,6 +1,6 @@
 {
   "name": "thehive",
-  "version": "4.1.17-1",
+  "version": "4.1.18-1",
   "license": "AGPL-3.0",
   "dependencies": {
     "jquery": "^3.4.1",

frontend/bower.json

@@ -1,6 +1,6 @@
 {
     "name": "thehive",
-    "version": "4.1.17-1",
+    "version": "4.1.18-1",
     "license": "AGPL-3.0",
     "repository": {
         "type": "git",

frontend/package.json

@@ -43,6 +43,7 @@ output {
   caseNumberShift: 0
   resume: false
   removeData: false
+  integrityCheck.enabled: false
   db {
     provider: janusgraph
     janusgraph {

migration/src/main/resources/reference.conf

@@ -20,6 +20,7 @@ import javax.inject.Inject
 import scala.collection.JavaConverters._
 import scala.collection.immutable
 import scala.concurrent.ExecutionContext
+import scala.concurrent.duration.DurationInt
 import scala.util.Success
 
 trait IntegrityCheckApp {
@@ -43,22 +44,22 @@ trait IntegrityCheckApp {
               bindActor[DummyActor]("integrity-check-actor")
               bind[ActorRef[CaseNumberActor.Request]].toProvider[CaseNumberActorProvider]
 
-              val integrityCheckOpsBindings = ScalaMultibinder.newSetBinder[GenIntegrityCheckOps](binder)
-              integrityCheckOpsBindings.addBinding.to[AlertIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[CaseIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[CaseTemplateIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[CustomFieldIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[DataIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ImpactStatusIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[LogIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ObservableIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ObservableTypeIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[OrganisationIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ProfileIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ResolutionStatusIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[TagIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[TaskIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[UserIntegrityCheckOps]
+              val integrityCheckOpsBindings = ScalaMultibinder.newSetBinder[IntegrityCheck](binder)
+              integrityCheckOpsBindings.addBinding.to[AlertIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[CaseIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[CaseTemplateIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[CustomFieldIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[DataIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[ImpactStatusIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[LogIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[ObservableIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[ObservableTypeIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[OrganisationIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[ProfileIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[ResolutionStatusIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[TagIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[TaskIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[UserIntegrityCheck]
 
               bind[Environment].toInstance(Environment.simple())
               bind[ApplicationLifecycle].to[DefaultApplicationLifecycle]
@@ -77,25 +78,29 @@ trait IntegrityCheckApp {
     buildApp(configuration, db).getInstance(classOf[IntegrityChecks]).runChecks()
 }
 
-class IntegrityChecks @Inject() (db: Database, checks: immutable.Set[GenIntegrityCheckOps], userSrv: UserDB) extends MapMerger {
+class IntegrityChecks @Inject() (db: Database, checks: immutable.Set[IntegrityCheck], userSrv: UserDB) extends MapMerger {
   def runChecks(): Unit = {
     implicit val authContext: AuthContext = userSrv.getSystemAuthContext
     checks.foreach { c =>
-      db.tryTransaction { implicit graph =>
-        println(s"Running check on ${c.name} ...")
-        c.initialCheck()
-        val stats = c.duplicationCheck() <+> c.globalCheck()
-        val statsStr = stats
-          .collect {
-            case (k, v) if v != 0 => s"$k:$v"
-          }
-          .mkString(" ")
-        if (statsStr.isEmpty)
-          println("  no change needed")
-        else
-          println(s"  $statsStr")
-        Success(())
+      println(s"Running check on ${c.name} ...")
+      val desupStats = c match {
+        case dc: DedupCheck[_] => dc.dedup(KillSwitch.alwaysOn)
+        case _                 => Map.empty[String, Long]
       }
+      val globalStats = c match {
+        case gc: GlobalCheck[_] => gc.runGlobalCheck(24.hours, KillSwitch.alwaysOn)
+        case _                  => Map.empty[String, Long]
+      }
+      val statsStr = (desupStats <+> globalStats)
+        .collect {
+          case (k, v) if v != 0 => s"$k:$v"
+        }
+        .mkString(" ")
+      if (statsStr.isEmpty)
+        println("  no change needed")
+      else
+        println(s"  $statsStr")
+
     }
   }
 }

migration/src/main/scala/org/thp/thehive/cloner/IntegrityCheckApp.scala

@@ -120,6 +120,8 @@ trait Input {
   def listJobObservables(caseId: String): Source[Try[(String, InputObservable)], NotUsed]
   def countAction(filter: Filter): Future[Long]
   def listActions(entityIds: Seq[String]): Source[Try[(String, InputAction)], NotUsed]
-  def countAudit(filter: Filter): Future[Long]
+  def countAudits(filter: Filter): Future[Long]
   def listAudits(entityIds: Seq[String], filter: Filter): Source[Try[(String, InputAudit)], NotUsed]
+  def countDashboards(filter: Filter): Future[Long]
+  def listDashboards(filter: Filter): Source[Try[InputDashboard], NotUsed]
 }

migration/src/main/scala/org/thp/thehive/migration/Input.scala

@@ -91,6 +91,9 @@ object Migrate extends App with MigrationOps {
       opt[Int]('t', "thread-count")
         .text("number of threads")
         .action((t, c) => addConfig(c, "threadCount", t)),
+      opt[Unit]('k', "integrity-checks")
+        .text("run integrity checks after the migration")
+        .action((_, c) => addConfig(c, "output.integrityCheck.enabled", true)),
       /* case age */
       opt[String]("max-case-age")
         .valueName("<duration>")

migration/src/main/scala/org/thp/thehive/migration/Migrate.scala

@@ -68,7 +68,7 @@ class MigrationStats() {
     def setTotal(v: Long): Unit = total = v
 
     override def toString: String = {
-      val totalTxt = if (total < 0) s"/${nSuccess + nFailure}" else s"/${total / 1000}"
+      val totalTxt = if (total < 0) s"/${nSuccess + nFailure}" else s"/$total"
       val avg      = if (global.isEmpty) "" else s" avg:${global}µs"
       val failureAndExistTxt = if (nFailure > 0 || nExist > 0) {
         val failureTxt = if (nFailure > 0) s"$nFailure failures" else ""
@@ -454,7 +454,7 @@ trait MigrationOps {
     input.countJobs(filter).foreach(count => migrationStats.setTotal("Job", count))
     input.countJobObservables(filter).foreach(count => migrationStats.setTotal("Job/Observable", count))
     input.countAction(filter).foreach(count => migrationStats.setTotal("Action", count))
-    input.countAudit(filter).foreach(count => migrationStats.setTotal("Audit", count))
+    input.countAudits(filter).foreach(count => migrationStats.setTotal("Audit", count))
 
     migrationStats.stage = "Prepare database"
     output.startMigration().flatMap { _ =>
@@ -474,6 +474,8 @@ trait MigrationOps {
       migrate(output)("ObservableType", input.listObservableTypes(filter), output.createObservableTypes, output.observableTypeExists)
       migrationStats.stage = "Migrate case templates"
       migrateWholeCaseTemplates(input, output, filter)
+      migrationStats.stage = "Migrate dashboards"
+      migrate(output)("Dashboard", input.listDashboards(filter), output.createDashboard, output.dashboardExists)
       migrationStats.stage = "Migrate cases and alerts"
       migrateCasesAndAlerts(input, output, filter)
       migrationStats.stage = "Finalisation"

migration/src/main/scala/org/thp/thehive/migration/MigrationOps.scala

@@ -39,4 +39,6 @@ trait Output[TX] {
   def createAlertObservable(tx: TX, alertId: EntityId, inputObservable: InputObservable): Try[IdMapping]
   def createAction(tx: TX, objectId: EntityId, inputAction: InputAction): Try[IdMapping]
   def createAudit(tx: TX, contextId: EntityId, inputAudit: InputAudit): Try[Unit]
+  def dashboardExists(tx: TX, inputDashboard: InputDashboard): Boolean
+  def createDashboard(tx: TX, inputDashboard: InputDashboard): Try[IdMapping]
 }

migration/src/main/scala/org/thp/thehive/migration/Output.scala

@@ -0,0 +1,5 @@
+package org.thp.thehive.migration.dto
+
+import org.thp.thehive.models.Dashboard
+
+case class InputDashboard(metaData: MetaData, organisation: Option[(String, Boolean)], dashboard: Dashboard)

migration/src/main/scala/org/thp/thehive/migration/dto/InputDashboard.scala

@@ -347,7 +347,7 @@ trait Conversion {
   implicit val customFieldReads: Reads[InputCustomField] = Reads[InputCustomField] { json =>
     for {
       //      metaData    <- json.validate[MetaData]
-      valueJson <- (json \ "value").validate[String].map(truncateString)
+      valueJson <- (json \ "value").validate[String]
       value = Json.parse(valueJson)
       displayName <- (value \ "name").validate[String].map(truncateString)
       name        <- (value \ "reference").validate[String].map(truncateString)
@@ -584,4 +584,14 @@ trait Conversion {
       )
     )
   }
+  implicit val dashboardReads: Reads[InputDashboard] = Reads[InputDashboard] { json =>
+    for {
+      metaData         <- json.validate[MetaData]
+      title            <- (json \ "title").validate[String]
+      description      <- (json \ "description").validate[String]
+      definitionString <- (json \ "definition").validate[String]
+      definition       <- Json.parse(definitionString).validate[JsObject]
+      status           <- (json \ "status").validate[String]
+    } yield InputDashboard(metaData, if (status == "Shared") Some(mainOrganisation -> true) else None, Dashboard(title, description, definition))
+  }
 }

migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala

@@ -19,6 +19,7 @@ import java.net.{URI, URLEncoder}
 import javax.inject.{Inject, Provider, Singleton}
 import scala.concurrent.duration.{Duration, DurationInt, DurationLong, FiniteDuration}
 import scala.concurrent.{Await, ExecutionContext, Future}
+import scala.util.Try
 
 @Singleton
 class ElasticClientProvider @Inject() (
@@ -196,17 +197,22 @@ class ElasticConfig(
       )
       .status == 200
 
-  def isSingleType(indexName: String): Boolean = {
-    val response = Await
-      .result(
-        authentication(ws.url(stripUrl(s"$esUri/$indexName")))
-          .get(),
-        10.seconds
-      )
-    if (response.status != 200)
-      throw InternalError(s"Unexpected response from Elasticsearch: ${response.status} ${response.statusText}\n${response.body}")
-    (response.json \ indexName \ "settings" \ "index" \ "mapping" \ "single_type").asOpt[String].fold(version.head > '6')(_.toBoolean)
-  }
+  def isSingleType(indexName: String): Boolean =
+    indexName
+      .split('_')
+      .lastOption
+      .flatMap(version => Try(version.toInt).toOption)
+      .fold {
+        val response = Await
+          .result(
+            authentication(ws.url(stripUrl(s"$esUri/$indexName")))
+              .get(),
+            10.seconds
+          )
+        if (response.status != 200)
+          throw InternalError(s"Unexpected response from Elasticsearch: ${response.status} ${response.statusText}\n${response.body}")
+        (response.json \ indexName \ "settings" \ "index" \ "mapping" \ "single_type").asOpt[String].fold(version.head > '6')(_.toBoolean)
+      }(version => version >= 15)
 
   def version: String = {
     val response = Await.result(authentication(ws.url(stripUrl(esUri))).get(), 10.seconds)