Setup elasticsearch sync in docker environment

Setup elasticsearch sync in docker environment.
Initial change in rules for test execution aggregation to create duration for activities and tests.

Author: vasile-baluta

.travis.yml

@@ -25,7 +25,7 @@ before_install:
 before_script:
   - cd src/main/docker/
   - source env.bash
-  - docker-compose up -d
+  - docker-compose up -d mongodb rabbitmq eiffel-er jenkins mail-server ei-backend
   - sleep 60
   - cd ../../../
 

src/main/docker/docker-compose.yml

@@ -146,7 +146,6 @@ services:
         aliases:
           - jenkins
 
-
   mail-server:
     image: ${MAILSERVER_IMAGE}
     expose:
@@ -212,6 +211,72 @@ services:
       - logging.level.org.springframework.web=DEBUG
       - logging.level.com.ericsson.ei=DEBUG
 
+  elasticsearch:
+    restart: always
+    image: ${ELASTICSEARCH_IMAGE}
+    ports:
+      - "${ELASTICSEARCH_PORT1}:9200"
+      - "${ELASTICSEARCH_PORT2}:9300"
+    depends_on:
+      - mongodb
+    environment:
+      - ES_JAVA_OPTS=-Xms2048m -Xmx2048m
+      - discovery.type=single-node
+    healthcheck:
+      test: "wget -q -O - http://localhost:9200/_cat/health"
+      interval: 1s
+      timeout: 30s
+      retries: 300
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+
+    networks:
+          eiffel_2.0_1:
+            aliases:
+              - elasticsearch
+
+  kibana:
+    restart: always
+    image: ${KIBANA_IMAGE}
+    expose:
+      - "8080"
+    ports:
+      - "${KIBANA_PORT}:8080"
+    depends_on:
+      - elasticsearch
+    environment:
+      - SERVER_PORT=8080
+      - SERVER_NAME=kibana.example.org
+      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
+
+    networks:
+          eiffel_2.0_1:
+            aliases:
+              - kibana
+
+## monstache is used to sync from mongodb to elasticsearch using oplog in mongodb
+  monstache:
+    restart: always
+    image: ${MONSTACHE_IMAGE}
+    expose:
+      - "8080"
+    ports:
+      - "${MONSTACHE_PORT}:8080"
+    depends_on:
+      - mongodb
+      - elasticsearch
+    environment:
+      - MONSTACHE_MONGO_URL=mongodb:${MONGODB_PORT}
+      - MONSTACHE_ES_URLS=http://elasticsearch:9200
+    links:
+      - elasticsearch
+
+    networks:
+          eiffel_2.0_1:
+            aliases:
+              - monstache
 
 networks:
   eiffel_2.0_1:

src/main/docker/env.bash

@@ -8,6 +8,10 @@ export JENKINS_PORT=8082
 export MAIL_SMTP_PORT=1025
 export MAIL_WEB_PORT=8025
 export EI_BACKEND_PORT=8090
+export ELASTICSEARCH_PORT1=8026
+export ELASTICSEARCH_PORT2=8027
+export KIBANA_PORT=8085
+export MONSTACHE_PORT=8086
 
 
 export MONGODB_IMAGE="mongo:latest"
@@ -16,3 +20,6 @@ export EIFFEL_ER_IMAGE="eiffelericsson/eiffel-er:0.0.67"
 export JENKINS_IMAGE="bitnami/jenkins:2.138.3"
 export MAILSERVER_IMAGE="mailhog/mailhog"
 export EI_BACKEND_IMAGE="eiffelericsson/eiffel-intelligence-backend:1.0.1"
+export ELASTICSEARCH_IMAGE="docker.elastic.co/elasticsearch/elasticsearch:6.2.4"
+export KIBANA_IMAGE="docker.elastic.co/kibana/kibana:6.2.4"
+export MONSTACHE_IMAGE="rwynn/monstache:4.15.1"

src/main/resources/SourceChangeObjectRules.json

@@ -58,7 +58,7 @@
     "ArrayMergeOptions": "",
     "HistoryIdentifyRules": "",
     "HistoryExtractionRules": "",
-    "ProcessRules": null
+    "ProcessRules": "{activityDuration : diff(causedActivities | [?triggeredEventId=='%IdentifyRules%'].canceledTime | [0], causedActivities | [?triggeredEventId=='%IdentifyRules%'].startedTime | [0])}"
   },
   {
     "TemplateName": "SC_1",
@@ -73,7 +73,7 @@
     "ArrayMergeOptions": "",
     "HistoryIdentifyRules": "",
     "HistoryExtractionRules": "",
-    "ProcessRules": null
+    "ProcessRules": "{activityDuration : diff(causedActivities | [?triggeredEventId=='%IdentifyRules%'].finishedTime | [0], causedActivities | [?triggeredEventId=='%IdentifyRules%'].startedTime | [0])}"
   },
   {
     "TemplateName": "SC_1",

src/main/resources/TestExecutionObjectRules.json

@@ -7,13 +7,12 @@
     "StartEvent": "YES",
     "IdentifyRules": "[meta.id]",
     "MatchIdRules": {"_id": "%IdentifyRules_objid%"},
-    "ExtractionRules": "{ activity_triggered_event_id : meta.id, type : meta.type, time : meta.time, version : meta.version, ongoing : `\"false\"` }",
+    "ExtractionRules": "{ activity_triggered_event_id : meta.id, type : meta.type, activity_triggered_time : meta.time, version : meta.version, ongoing : `\"false\"` }",
     "MergeResolverRules": null,
     "ArrayMergeOptions": "",
     "HistoryIdentifyRules": "",
     "HistoryExtractionRules": "",
-    "ProcessRules": null,
-    "ProcessFunction": null
+    "ProcessRules": null
   },
   {
     "TemplateName": "TEST_EXECUTION_1",
@@ -27,8 +26,7 @@
     "ArrayMergeOptions": "",
     "HistoryIdentifyRules": "",
     "HistoryExtractionRules": "",
-    "ProcessRules": null,
-    "ProcessFunction": null
+    "ProcessRules": null
   },
   {
     "TemplateName": "TEST_EXECUTION_1",
@@ -43,8 +41,7 @@
     "ArrayMergeOptions": "",
     "HistoryIdentifyRules": "",
     "HistoryExtractionRules": "",
-    "ProcessRules": null,
-    "ProcessFunction": null
+    "ProcessRules": null
   },
   {
     "TemplateName": "TEST_EXECUTION_1",
@@ -59,8 +56,7 @@
     "ArrayMergeOptions": "",
     "HistoryIdentifyRules": "",
     "HistoryExtractionRules": "",
-    "ProcessRules": null,
-    "ProcessFunction": null
+    "ProcessRules": null
   },
   {
     "TemplateName": "TEST_EXECUTION_1",
@@ -75,8 +71,7 @@
     "ArrayMergeOptions": "",
     "HistoryIdentifyRules": "",
     "HistoryExtractionRules": "",
-    "ProcessRules": null,
-    "ProcessFunction": null
+    "ProcessRules": null
   },
   {
     "TemplateName": "TEST_EXECUTION_1",
@@ -91,8 +86,7 @@
     "ArrayMergeOptions": "",
     "HistoryIdentifyRules": "",
     "HistoryExtractionRules": "",
-    "ProcessRules": null,
-    "ProcessFunction": null
+    "ProcessRules": null
   },
   {
     "TemplateName": "TEST_EXECUTION_1",
@@ -107,8 +101,7 @@
     "ArrayMergeOptions": "",
     "HistoryIdentifyRules": "",
     "HistoryExtractionRules": "",
-    "ProcessRules": null,
-    "ProcessFunction": null
+    "ProcessRules": null
   },
   {
     "TemplateName": "TEST_EXECUTION_1",
@@ -123,7 +116,6 @@
     "ArrayMergeOptions": "",
     "HistoryIdentifyRules": "",
     "HistoryExtractionRules": "",
-    "ProcessRules": null,
-    "ProcessFunction": null
+    "ProcessRules": null
   }
 ]