Skip to content

Commit b7b196c

Browse files
To-omTo-om
committed
Merge branch 'hotfix/4.1.17'
2 parents f216ed3 + 78631ed commit b7b196c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+2014
-2094
lines changed

‎CHANGELOG.md

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,18 @@
11
# Change Log
22

3+
## [4.1.17](https://github.com/TheHive-Project/TheHive/milestone/87) (2022-01-24)
4+
5+
**Implemented enhancements:**
6+
7+
- [Enhancement] Improve migration tool by accepting old versions of TheHive [\#2305](https://github.com/TheHive-Project/TheHive/issues/2305)
8+
- Security concern [\#2309](https://github.com/TheHive-Project/TheHive/issues/2309)
9+
10+
**Fixed bugs:**
11+
12+
- [Bug] Action 'mergeCase' not mapped in v0 [\#2304](https://github.com/TheHive-Project/TheHive/issues/2304)
13+
- Can't start after upgrade thehive4 (4.1.16-1) over (4.0.0-1) [Bug] [\#2308](https://github.com/TheHive-Project/TheHive/issues/2308)
14+
- [Bug] Notifications are executed several times [\#2317](https://github.com/TheHive-Project/TheHive/issues/2317)
15+
316
## [4.1.16](https://github.com/TheHive-Project/TheHive/milestone/86) (2021-12-17)
417

518
**Implemented enhancements:**

‎ScalliGraph

‎build.sbt

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import Dependencies._
22
import com.typesafe.sbt.packager.Keys.bashScriptDefines
33
import org.thp.ghcl.Milestone
44

5-
val thehiveVersion = "4.1.16-1"
5+
val thehiveVersion = "4.1.17-1"
66
val scala212 = "2.12.13"
77
val scala213 = "2.13.1"
88
val supportedScalaVersions = List(scala212, scala213)
@@ -342,10 +342,7 @@ lazy val thehiveMigration = (project in file("migration"))
342342
resolvers += "elasticsearch-releases" at "https://artifacts.elastic.co/maven",
343343
crossScalaVersions := Seq(scala212),
344344
libraryDependencies ++= Seq(
345-
elastic4sCore,
346-
elastic4sHttpStreams,
347-
elastic4sClient,
348-
// jts,
345+
alpakka,
349346
ehcache,
350347
scopt,
351348
specs % Test

‎conf/migration-logback.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
<logger name="org.thp.thehive.services.LocalAuthSrv" level="TRACE" />
4545
<logger name="org.thp.scalligraph.graphql" level="TRACE" />
4646
-->
47+
<logger name="org.thp.scalligraph.traversal" level="INFO"/>
4748
<logger name="org.janusgraph.graphdb.transaction.StandardJanusGraphTx" level="ERROR"/>
4849
<logger name="org.thp.thehive" level="INFO"/>
4950

‎dto/src/main/scala/org/thp/thehive/dto/v1/User.scala

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package org.thp.thehive.dto.v1
22

33
import org.thp.scalligraph.controllers.FFile
4-
import play.api.libs.json.{Json, OFormat, Writes}
4+
import play.api.libs.json.{JsObject, Json, OFormat, Writes}
55

66
import java.util.Date
77

@@ -32,7 +32,8 @@ case class OutputUser(
3232
permissions: Set[String],
3333
organisation: String,
3434
avatar: Option[String],
35-
organisations: Seq[OutputOrganisationProfile]
35+
organisations: Seq[OutputOrganisationProfile],
36+
extraData: JsObject
3637
)
3738

3839
object OutputUser {

‎frontend/bower.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "thehive",
3-
"version": "4.1.16-1",
3+
"version": "4.1.17-1",
44
"license": "AGPL-3.0",
55
"dependencies": {
66
"jquery": "^3.4.1",

‎frontend/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "thehive",
3-
"version": "4.1.16-1",
3+
"version": "4.1.17-1",
44
"license": "AGPL-3.0",
55
"repository": {
66
"type": "git",

‎migration/src/main/resources/reference.conf

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,13 @@ input {
1010
keepalive: 10h
1111
# Size of the page for scroll
1212
pagesize: 10
13+
14+
maxAttempts = 5
15+
minBackoff = 10 milliseconds
16+
maxBackoff = 10 seconds
17+
randomFactor = 0.2
1318
}
1419
filter {
15-
maxCaseAge: 0
16-
maxAlertAge: 0
17-
maxAuditAge: 0
1820
includeAlertTypes: []
1921
excludeAlertTypes: []
2022
includeAlertSources: []
@@ -39,6 +41,7 @@ input {
3941

4042
output {
4143
caseNumberShift: 0
44+
resume: false
4245
removeData: false
4346
db {
4447
provider: janusgraph
@@ -77,6 +80,8 @@ output {
7780
}
7881
}
7982

83+
threadCount: 4
84+
transactionPageSize: 50
8085

8186
from {
8287
db {

‎migration/src/main/scala/org/thp/thehive/cloner/IntegrityCheckApp.scala

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -44,21 +44,21 @@ trait IntegrityCheckApp {
4444
bind[ActorRef[CaseNumberActor.Request]].toProvider[CaseNumberActorProvider]
4545

4646
val integrityCheckOpsBindings = ScalaMultibinder.newSetBinder[GenIntegrityCheckOps](binder)
47-
integrityCheckOpsBindings.addBinding.to[ProfileIntegrityCheckOps]
48-
integrityCheckOpsBindings.addBinding.to[OrganisationIntegrityCheckOps]
49-
integrityCheckOpsBindings.addBinding.to[TagIntegrityCheckOps]
50-
integrityCheckOpsBindings.addBinding.to[UserIntegrityCheckOps]
51-
integrityCheckOpsBindings.addBinding.to[ImpactStatusIntegrityCheckOps]
52-
integrityCheckOpsBindings.addBinding.to[ResolutionStatusIntegrityCheckOps]
53-
integrityCheckOpsBindings.addBinding.to[ObservableTypeIntegrityCheckOps]
54-
integrityCheckOpsBindings.addBinding.to[CustomFieldIntegrityCheckOps]
47+
integrityCheckOpsBindings.addBinding.to[AlertIntegrityCheckOps]
48+
integrityCheckOpsBindings.addBinding.to[CaseIntegrityCheckOps]
5549
integrityCheckOpsBindings.addBinding.to[CaseTemplateIntegrityCheckOps]
50+
integrityCheckOpsBindings.addBinding.to[CustomFieldIntegrityCheckOps]
5651
integrityCheckOpsBindings.addBinding.to[DataIntegrityCheckOps]
57-
integrityCheckOpsBindings.addBinding.to[CaseIntegrityCheckOps]
58-
integrityCheckOpsBindings.addBinding.to[AlertIntegrityCheckOps]
59-
integrityCheckOpsBindings.addBinding.to[TaskIntegrityCheckOps]
60-
integrityCheckOpsBindings.addBinding.to[ObservableIntegrityCheckOps]
52+
integrityCheckOpsBindings.addBinding.to[ImpactStatusIntegrityCheckOps]
6153
integrityCheckOpsBindings.addBinding.to[LogIntegrityCheckOps]
54+
integrityCheckOpsBindings.addBinding.to[ObservableIntegrityCheckOps]
55+
integrityCheckOpsBindings.addBinding.to[ObservableTypeIntegrityCheckOps]
56+
integrityCheckOpsBindings.addBinding.to[OrganisationIntegrityCheckOps]
57+
integrityCheckOpsBindings.addBinding.to[ProfileIntegrityCheckOps]
58+
integrityCheckOpsBindings.addBinding.to[ResolutionStatusIntegrityCheckOps]
59+
integrityCheckOpsBindings.addBinding.to[TagIntegrityCheckOps]
60+
integrityCheckOpsBindings.addBinding.to[TaskIntegrityCheckOps]
61+
integrityCheckOpsBindings.addBinding.to[UserIntegrityCheckOps]
6262

6363
bind[Environment].toInstance(Environment.simple())
6464
bind[ApplicationLifecycle].to[DefaultApplicationLifecycle]

‎migration/src/main/scala/org/thp/thehive/migration/Input.scala

Lines changed: 4 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -48,17 +48,15 @@ object Filter {
4848
new ParseException(s"Unparseable date: $s\nExpected format is ${dateFormats.map(_.toPattern).mkString("\"", "\" or \"", "\"")}", 0)
4949
)
5050
}
51-
def readDate(dateConfigName: String, ageConfigName: String) =
51+
def readDate(dateConfigName: String, ageConfigName: String): Option[Long] =
5252
Try(config.getString(dateConfigName))
5353
.flatMap(parseDate)
5454
.map(d => d.getTime)
55-
.toOption
5655
.orElse {
57-
Try {
58-
val age = config.getDuration(ageConfigName)
59-
if (age.isZero) None else Some(now - age.getSeconds * 1000)
60-
}.toOption.flatten
56+
Try(config.getDuration(ageConfigName))
57+
.map(d => now - d.getSeconds * 1000)
6158
}
59+
.toOption
6260
val caseFromDate = readDate("caseFromDate", "maxCaseAge")
6361
val caseUntilDate = readDate("caseUntilDate", "minCaseAge")
6462
val caseFromNumber = Try(config.getInt("caseFromNumber")).toOption
@@ -90,24 +88,16 @@ trait Input {
9088
def countOrganisations(filter: Filter): Future[Long]
9189
def listCases(filter: Filter): Source[Try[InputCase], NotUsed]
9290
def countCases(filter: Filter): Future[Long]
93-
def listCaseObservables(filter: Filter): Source[Try[(String, InputObservable)], NotUsed]
9491
def countCaseObservables(filter: Filter): Future[Long]
9592
def listCaseObservables(caseId: String): Source[Try[(String, InputObservable)], NotUsed]
96-
def countCaseObservables(caseId: String): Future[Long]
97-
def listCaseTasks(filter: Filter): Source[Try[(String, InputTask)], NotUsed]
9893
def countCaseTasks(filter: Filter): Future[Long]
9994
def listCaseTasks(caseId: String): Source[Try[(String, InputTask)], NotUsed]
100-
def countCaseTasks(caseId: String): Future[Long]
101-
def listCaseTaskLogs(filter: Filter): Source[Try[(String, InputLog)], NotUsed]
10295
def countCaseTaskLogs(filter: Filter): Future[Long]
10396
def listCaseTaskLogs(caseId: String): Source[Try[(String, InputLog)], NotUsed]
104-
def countCaseTaskLogs(caseId: String): Future[Long]
10597
def listAlerts(filter: Filter): Source[Try[InputAlert], NotUsed]
10698
def countAlerts(filter: Filter): Future[Long]
107-
def listAlertObservables(filter: Filter): Source[Try[(String, InputObservable)], NotUsed]
10899
def countAlertObservables(filter: Filter): Future[Long]
109100
def listAlertObservables(alertId: String): Source[Try[(String, InputObservable)], NotUsed]
110-
def countAlertObservables(alertId: String): Future[Long]
111101
def listUsers(filter: Filter): Source[Try[InputUser], NotUsed]
112102
def countUsers(filter: Filter): Future[Long]
113103
def listCustomFields(filter: Filter): Source[Try[InputCustomField], NotUsed]
@@ -123,25 +113,13 @@ trait Input {
123113
def listCaseTemplate(filter: Filter): Source[Try[InputCaseTemplate], NotUsed]
124114
def countCaseTemplate(filter: Filter): Future[Long]
125115
def listCaseTemplateTask(caseTemplateId: String): Source[Try[(String, InputTask)], NotUsed]
126-
def countCaseTemplateTask(caseTemplateId: String): Future[Long]
127-
def listCaseTemplateTask(filter: Filter): Source[Try[(String, InputTask)], NotUsed]
128116
def countCaseTemplateTask(filter: Filter): Future[Long]
129117
def listJobs(caseId: String): Source[Try[(String, InputJob)], NotUsed]
130-
def countJobs(caseId: String): Future[Long]
131-
def listJobs(filter: Filter): Source[Try[(String, InputJob)], NotUsed]
132118
def countJobs(filter: Filter): Future[Long]
133-
def listJobObservables(filter: Filter): Source[Try[(String, InputObservable)], NotUsed]
134119
def countJobObservables(filter: Filter): Future[Long]
135120
def listJobObservables(caseId: String): Source[Try[(String, InputObservable)], NotUsed]
136-
def countJobObservables(caseId: String): Future[Long]
137-
def listAction(filter: Filter): Source[Try[(String, InputAction)], NotUsed]
138121
def countAction(filter: Filter): Future[Long]
139-
def listAction(entityId: String): Source[Try[(String, InputAction)], NotUsed]
140122
def listActions(entityIds: Seq[String]): Source[Try[(String, InputAction)], NotUsed]
141-
def countAction(entityId: String): Future[Long]
142-
def listAudit(filter: Filter): Source[Try[(String, InputAudit)], NotUsed]
143123
def countAudit(filter: Filter): Future[Long]
144-
def listAudit(entityId: String, filter: Filter): Source[Try[(String, InputAudit)], NotUsed]
145124
def listAudits(entityIds: Seq[String], filter: Filter): Source[Try[(String, InputAudit)], NotUsed]
146-
def countAudit(entityId: String, filter: Filter): Future[Long]
147125
}

‎migration/src/main/scala/org/thp/thehive/migration/Migrate.scala

Lines changed: 37 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,16 @@ import scopt.OParser
1010
import java.io.File
1111
import java.nio.file.{Files, Paths}
1212
import scala.collection.JavaConverters._
13-
import scala.concurrent.duration.{Duration, DurationInt}
14-
import scala.concurrent.{Await, ExecutionContext}
13+
import scala.concurrent.duration.DurationInt
14+
import scala.concurrent.{blocking, Await, ExecutionContext, Future}
1515

1616
object Migrate extends App with MigrationOps {
1717
val defaultLoggerConfigFile = "/etc/thehive/logback-migration.xml"
1818
if (System.getProperty("logger.file") == null && Files.exists(Paths.get(defaultLoggerConfigFile)))
1919
System.setProperty("logger.file", defaultLoggerConfigFile)
20+
(new LogbackLoggerConfigurator).configure(Environment.simple(), Configuration.empty, Map.empty)
21+
var transactionPageSize: Int = 100
22+
var threadCount: Int = 3
2023

2124
def getVersion: String = Option(getClass.getPackage.getImplementationVersion).getOrElse("SNAPSHOT")
2225

@@ -53,6 +56,9 @@ object Migrate extends App with MigrationOps {
5356
opt[Unit]('d', "drop-database")
5457
.action((_, c) => addConfig(c, "output.dropDatabase", true))
5558
.text("Drop TheHive4 database before migration"),
59+
opt[Unit]('r', "resume")
60+
.action((_, c) => addConfig(c, "output.resume", true))
61+
.text("Resume migration (or migrate on existing database)"),
5662
opt[String]('m', "main-organisation")
5763
.valueName("<organisation>")
5864
.action((o, c) => addConfig(c, "input.mainOrganisation", o)),
@@ -64,13 +70,27 @@ object Migrate extends App with MigrationOps {
6470
.valueName("<index>")
6571
.text("TheHive3 ElasticSearch index name")
6672
.action((i, c) => addConfig(c, "input.search.index", i)),
73+
opt[String]('x', "es-index-version")
74+
.valueName("<index>")
75+
.text("TheHive3 ElasticSearch index name version number (default: autodetect)")
76+
.action((i, c) => addConfig(c, "input.search.indexVersion", i)),
6777
opt[String]('a', "es-keepalive")
6878
.valueName("<duration>")
6979
.text("TheHive3 ElasticSearch keepalive")
7080
.action((a, c) => addConfig(c, "input.search.keepalive", a)),
7181
opt[Int]('p', "es-pagesize")
7282
.text("TheHive3 ElasticSearch page size")
7383
.action((p, c) => addConfig(c, "input.search.pagesize", p)),
84+
opt[Boolean]('s', "es-single-type")
85+
.valueName("<bool>")
86+
.text("Elasticsearch single type")
87+
.action((s, c) => addConfig(c, "input.search.singleType", s)),
88+
opt[Int]('y', "transaction-pagesize")
89+
.text("page size for each transaction")
90+
.action((t, c) => addConfig(c, "transactionPageSize", t)),
91+
opt[Int]('t', "thread-count")
92+
.text("number of threads")
93+
.action((t, c) => addConfig(c, "threadCount", t)),
7494
/* case age */
7595
opt[String]("max-case-age")
7696
.valueName("<duration>")
@@ -134,11 +154,11 @@ object Migrate extends App with MigrationOps {
134154
opt[String]("max-audit-age")
135155
.valueName("<duration>")
136156
.text("migrate only audits whose age is less than <duration>")
137-
.action((v, c) => addConfig(c, "input.filter.minAuditAge", v)),
157+
.action((v, c) => addConfig(c, "input.filter.maxAuditAge", v)),
138158
opt[String]("min-audit-age")
139159
.valueName("<duration>")
140160
.text("migrate only audits whose age is greater than <duration>")
141-
.action((v, c) => addConfig(c, "input.filter.maxAuditAge", v)),
161+
.action((v, c) => addConfig(c, "input.filter.minAuditAge", v)),
142162
opt[String]("audit-from-date")
143163
.valueName("<date>")
144164
.text("migrate only audits created from <date>")
@@ -183,13 +203,19 @@ object Migrate extends App with MigrationOps {
183203
implicit val actorSystem: ActorSystem = ActorSystem("TheHiveMigration", config)
184204
implicit val ec: ExecutionContext = actorSystem.dispatcher
185205
implicit val mat: Materializer = Materializer(actorSystem)
206+
transactionPageSize = config.getInt("transactionPageSize")
207+
threadCount = config.getInt("threadCount")
208+
var stop = false
186209

187210
try {
188-
(new LogbackLoggerConfigurator).configure(Environment.simple(), Configuration.empty, Map.empty)
189-
190-
val timer = actorSystem.scheduler.scheduleAtFixedRate(10.seconds, 10.seconds) { () =>
191-
logger.info(migrationStats.showStats())
192-
migrationStats.flush()
211+
Future {
212+
blocking {
213+
while (!stop) {
214+
logger.info(migrationStats.showStats())
215+
migrationStats.flush()
216+
Thread.sleep(10000) // 10 seconds
217+
}
218+
}
193219
}
194220

195221
val returnStatus =
@@ -198,17 +224,15 @@ object Migrate extends App with MigrationOps {
198224
val output = th4.Output(Configuration(config.getConfig("output").withFallback(config)))
199225
val filter = Filter.fromConfig(config.getConfig("input.filter"))
200226

201-
val process = migrate(input, output, filter)
202-
203-
Await.result(process, Duration.Inf)
227+
migrate(input, output, filter).get
204228
logger.info("Migration finished")
205229
0
206230
} catch {
207231
case e: Throwable =>
208232
logger.error(s"Migration failed", e)
209233
1
210234
} finally {
211-
timer.cancel()
235+
stop = true
212236
Await.ready(actorSystem.terminate(), 1.minute)
213237
()
214238
}

0 commit comments

Comments
 (0)