Escape Character issues in Aggregated objects in MongoDb solved (#34)

tobiasake · web-flow · commit c6b4317e4641 · 2017-11-14T10:50:11.000+01:00
* Added some debug logging.

* Solved some potential Escape character bugs around handling of aggregated objects.

* Added missing quotation in getEventToObjectList() for eventId in condition variable.

* Solved more issue extra quotation in some cases when merging, extracting and macthing objects id.

* Cleaned up more debug and test prinout code. Improved debug logs.

* Solved some failing unit tests.

* Solved some issues related to flowtest.

* add back template name in object

* add back put properties

* Fixed Flowtest tests.

* Solved escape character issues in FlowTest2 and in AggregatedDocument2.json.
diff --git a/src/main/java/com/ericsson/ei/handlers/DownstreamExtractionHandler.java b/src/main/java/com/ericsson/ei/handlers/DownstreamExtractionHandler.java
@@ -39,6 +39,8 @@ public class DownstreamExtractionHandler {
     public void runExtraction(RulesObject rulesObject, String mergeId, String event, String aggregatedDbObject) {
         try {
             ObjectMapper mapper = new ObjectMapper();
+            log.debug("Start extraction of Aggregated Object:\n" + aggregatedDbObject + 
+            		"\nwith Event:\n" + event);
             JsonNode aggregatedJsonObject = mapper.readValue(aggregatedDbObject, JsonNode.class);
             runExtraction(rulesObject, mergeId, event, aggregatedJsonObject);
         } catch (Exception e) {
@@ -49,6 +51,9 @@ public void runExtraction(RulesObject rulesObject, String mergeId, String event,
     public void runExtraction(RulesObject rulesObject, String mergeId, String event, JsonNode aggregatedDbObject) {
         JsonNode extractedContent;
         extractedContent = extractContent(rulesObject, event);
+        log.debug("Start extraction of Aggregated Object:\n" + aggregatedDbObject.toString() + 
+        		"\nwith Event:\n" + event);
+
 
         if(aggregatedDbObject != null) {
             String objectId = objectHandler.extractObjectId(aggregatedDbObject);
diff --git a/src/main/java/com/ericsson/ei/handlers/EventToObjectMapHandler.java b/src/main/java/com/ericsson/ei/handlers/EventToObjectMapHandler.java
@@ -100,8 +100,10 @@ public void updateEventToObjectMapInMemoryDB(RulesObject rulesObject, String eve
         try {
             entry = new ObjectMapper().readValue(condition, JsonNode.class);
             ArrayNode jsonNode = mapper.convertValue(list, ArrayNode.class);
-            ((ObjectNode) entry).put(listPropertyName, jsonNode);
+            ((ObjectNode) entry).set(listPropertyName, mapper.readTree(jsonNode.toString()));
             String mapStr = entry.toString();
+            log.debug("MongoDbHandler Insert/Update Event: " + mapStr +
+            		"\nto database: " + databaseName + " and to Collection: " + collectionName);
             if (firstTime) {
                 mongodbhandler.insertDocument(databaseName, collectionName, mapStr);
             } else {
diff --git a/src/main/java/com/ericsson/ei/handlers/ExtractionHandler.java b/src/main/java/com/ericsson/ei/handlers/ExtractionHandler.java
@@ -62,7 +62,7 @@ public void setObjectHandler(ObjectHandler objectHandler) {
     public void runExtraction(RulesObject rulesObject, String id, String event, String aggregatedDbObject) {
         try {
             ObjectMapper mapper = new ObjectMapper();
-            JsonNode aggregatedJsonObject = mapper.readValue(aggregatedDbObject, JsonNode.class);
+            JsonNode aggregatedJsonObject = mapper.readTree(aggregatedDbObject);
             runExtraction(rulesObject, id, event, aggregatedJsonObject);
         } catch (Exception e) {
             log.info(e.getMessage(),e);
@@ -74,11 +74,15 @@ public void runExtraction(RulesObject rulesObject, String mergeId, String event,
         extractedContent = extractContent(rulesObject, event);
 
         if(aggregatedDbObject != null) {
+            log.debug("ExtractionHandler: Merging Aggregated Object:\n" + aggregatedDbObject.toString() +
+            		"\nwith extracted content:\n" + extractedContent.toString() +
+            		"\nfrom event:\n" + event);
             String objectId = objectHandler.extractObjectId(aggregatedDbObject);
             String mergedContent = mergeHandler.mergeObject(objectId, mergeId, rulesObject, event, extractedContent);
             mergedContent = processRulesHandler.runProcessRules(event, rulesObject, mergedContent, objectId, mergeId);
             //historyIdRulesHandler.runHistoryIdRules(aggregationObject, rulesObject, event);
         } else {
+        	ObjectMapper mapper = new ObjectMapper();
             ObjectNode objectNode = (ObjectNode) extractedContent;
             objectNode.put("TemplateName", rulesObject.getTemplateName());
             mergeHandler.addNewObject(event, extractedContent, rulesObject);
diff --git a/src/main/java/com/ericsson/ei/handlers/ObjectHandler.java b/src/main/java/com/ericsson/ei/handlers/ObjectHandler.java
@@ -86,8 +86,8 @@ public boolean insertObject(String aggregatedObject, RulesObject rulesObject, St
             id = idNode.textValue();
         }
         JsonNode document = prepareDocumentForInsertion(id, aggregatedObject);
-        String documentStr = document.toString();
-        boolean result = mongoDbHandler.insertDocument(databaseName, collectionName, documentStr);
+        log.debug("ObjectHandler: Aggregated Object document to be inserted: " + document.toString());
+        boolean result = mongoDbHandler.insertDocument(databaseName, collectionName, document.toString());
         if (result)
             eventToObjectMap.updateEventToObjectMapInMemoryDB(rulesObject, event, id);
             subscriptionHandler.checkSubscriptionForObject(aggregatedObject);
@@ -113,6 +113,8 @@ public boolean updateObject(String aggregatedObject, RulesObject rulesObject, St
             JsonNode idNode = jmespathInterface.runRuleOnEvent(idRules, event);
             id = idNode.textValue();
         }
+        log.debug("ObjectHandler: Updating Aggregated Object:\n" + aggregatedObject +
+        		"\nEvent:\n" + event);
         JsonNode document = prepareDocumentForInsertion(id, aggregatedObject);
         String condition = "{\"_id\" : \"" + id + "\"}";
         String documentStr = document.toString();
@@ -148,18 +150,22 @@ public ArrayList<String> findObjectsByIds(ArrayList<String> ids) {
         return objects;
     }
 
-    private JsonNode prepareDocumentForInsertion(String id, String object) {
-        ObjectMapper mapper = new ObjectMapper();
-        try {
-            String docStr = "{\"_id\":\"" + id +"\"}";
-            JsonNode document = mapper.readValue(docStr, JsonNode.class);
-            ((ObjectNode) document).put("aggregatedObject", object);
-            return document;
-        } catch (Exception e) {
-            log.info(e.getMessage(),e);
-        }
-        return null;
-    }
+        public JsonNode prepareDocumentForInsertion(String id, String object) {
+	        ObjectMapper mapper = new ObjectMapper();
+	        try {
+	            String docStr = "{\"_id\": \"" + id + "\"}";
+	            JsonNode jsonNodeNew = mapper.readValue(docStr, JsonNode.class);
+	            
+	            JsonNode jsonNode = mapper.readValue(jsonNodeNew.toString(), JsonNode.class); 
+	            ObjectNode objNode = (ObjectNode) jsonNode;  
+	            objNode.set("aggregatedObject", mapper.readTree(object));
+
+	            return jsonNode;
+	        } catch (Exception e) {
+	            log.info(e.getMessage(),e);
+	        }
+	        return null;
+	    }
 
     public JsonNode getAggregatedObject(String dbDocument) {
          ObjectMapper mapper = new ObjectMapper();
@@ -174,7 +180,7 @@ public JsonNode getAggregatedObject(String dbDocument) {
     }
 
     public String extractObjectId(JsonNode aggregatedDbObject) {
-        return aggregatedDbObject.get("_id").asText();
+        return aggregatedDbObject.get("_id").textValue();
     }
 
     /**
diff --git a/src/main/java/com/ericsson/ei/jsonmerge/MergeHandler.java b/src/main/java/com/ericsson/ei/jsonmerge/MergeHandler.java
@@ -84,6 +84,7 @@ public String mergeObject(String id, String mergeId, RulesObject rules, String e
         }catch (Exception e){
             log.info(e.getMessage(),e);
         }
+        log.debug("Merged Aggregated Object:\n" + mergedObject);
             // unlocking of document will be performed, when mergedObject will be inserted to database
             objectHandler.updateObject(mergedObject, rules, event, id);
             return mergedObject;
@@ -94,9 +95,9 @@ protected String getMergeRules(RulesObject rules) {
     }
 
     public String replaceIdMarkerInRules(String rule, String id){
-        String literal = "\"" + id + "\"";
+
         if (rule.contains(mergeIdMarker)) {
-            String updatedRule = rule.replaceAll(mergeIdMarker, literal);
+            String updatedRule = rule.replaceAll(mergeIdMarker, "\"" + id + "\"");
             updatedRule = "`" + updatedRule + "`";
             return updatedRule;
         }
@@ -172,7 +173,7 @@ public String getAggregatedObject(String id){
             String document = objectHandler.lockDocument(id);
             JsonNode result = objectHandler.getAggregatedObject(document);
             if (result != null)
-                return result.asText();
+                return result.toString();
         }catch (Exception e){
             log.info(e.getMessage(),e);
         }
diff --git a/src/main/java/com/ericsson/ei/mongodbhandler/MongoDBHandler.java b/src/main/java/com/ericsson/ei/mongodbhandler/MongoDBHandler.java
@@ -107,6 +107,8 @@ public ArrayList<String> getAllDocuments(String dataBaseName, String collectionN
     //Retrieve data from the collection based on condition
     public  ArrayList<String> find(String dataBaseName, String collectionName, String condition){
         ArrayList<String> result = new ArrayList<>();
+        log.debug("Find and retrieve data from database: " + dataBaseName + " Collection: " + collectionName +
+        		"\nwith Condition: " + condition);
         try{
             DB db = mongoClient.getDB(dataBaseName);
             DBCollection table = db.getCollection(collectionName);
diff --git a/src/main/java/com/ericsson/ei/subscriptionhandler/SubscriptionValidator.java b/src/main/java/com/ericsson/ei/subscriptionhandler/SubscriptionValidator.java
@@ -39,7 +39,6 @@ public class SubscriptionValidator {
 	public void validateSubscription(Subscription subscription) throws SubscriptionValidationException {
 		
 		LOG.info("Validation of subscription " + subscription.getSubscriptionName() + " Started.");
-		
 		this.validateSubscriptionName(subscription.getSubscriptionName());
 		this.validateNotificationMessage(subscription.getNotificationMessage());
 		this.validateNotificationMeta(subscription.getNotificationMeta());
diff --git a/src/main/java/com/ericsson/ei/waitlist/WaitListWorker.java b/src/main/java/com/ericsson/ei/waitlist/WaitListWorker.java
@@ -22,7 +22,10 @@
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.scheduling.TaskScheduler;
 import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.scheduling.concurrent.ConcurrentTaskScheduler;
 import org.springframework.stereotype.Component;
 
 import com.ericsson.ei.handlers.MatchIdRulesHandler;
@@ -54,6 +57,11 @@ public class WaitListWorker {
     private MatchIdRulesHandler matchIdRulesHandler;
 
     static Logger log = (Logger) LoggerFactory.getLogger(WaitListWorker.class);
+    
+    @Bean
+    public TaskScheduler taskScheduler() {
+        return new ConcurrentTaskScheduler();
+    }
 
     @Scheduled(initialDelayString = "${waitlist.initialDelayResend}", fixedRateString = "${waitlist.fixedRateResend}")
     public void run() {
diff --git a/src/test/java/com/ericsson/ei/flowtests/FlowTestBase.java b/src/test/java/com/ericsson/ei/flowtests/FlowTestBase.java
@@ -235,7 +235,7 @@ protected void checkResult() {
             JsonNode expectedJson = objectmapper.readTree(expectedDocument);
             JsonNode actualJson = objectmapper.readTree(document);
             String breakString = "breakHere";
-            assertEquals(expectedJson.toString().length(), actualJson.toString().length());
+            assertEquals("\nExpectedJson:\n" + expectedJson.toString() + "\nActual:\n" + actualJson.toString() + "\n", expectedJson.toString().length(), actualJson.toString().length());
         } catch (IOException e) {
             log.info(e.getMessage(),e);
         }
diff --git a/src/test/java/com/ericsson/ei/flowtests/TrafficGeneratedTest.java b/src/test/java/com/ericsson/ei/flowtests/TrafficGeneratedTest.java
@@ -233,17 +233,18 @@ public void trafficGeneratedTest() {
      * @return list of ready to send events.
      */
     private List<String> getPreparedEventsToSend(List<String> eventNames) throws IOException {
+    	ObjectMapper mapper = new ObjectMapper();
         List<String> events = new ArrayList<>();
         String newID;
         for (int i = 0; i < EVENT_PACKAGES; i++) {
             for(String eventName : eventNames) {
                 JsonNode eventJson = parsedJason.get(eventName);
                 newID = eventJson.at("/meta/id").textValue().substring(0, 30).concat(String.format("%06d", i));;
-                ((ObjectNode) eventJson.path("meta")).put("id", newID);
+                ((ObjectNode) eventJson.path("meta")).set("id", mapper.readValue(newID, JsonNode.class));
                 for (JsonNode link : eventJson.path("links")) {
                     if (link.has("target")) {
                         newID = link.path("target").textValue().substring(0, 30).concat(String.format("%06d", i));
-                        ((ObjectNode) link).put("target", newID);
+                        ((ObjectNode) link).set("target", mapper.readValue(newID, JsonNode.class));
                     }
                 }
                 events.add(eventJson.toString());
diff --git a/src/test/java/com/ericsson/ei/handlers/test/ObjectHandlerTest.java b/src/test/java/com/ericsson/ei/handlers/test/ObjectHandlerTest.java
@@ -21,6 +21,7 @@
 import static org.junit.Assert.assertEquals;
 
 import java.io.File;
+import java.io.IOException;
 
 import javax.annotation.PostConstruct;
 
@@ -44,8 +45,11 @@
 import com.ericsson.ei.mongodbhandler.MongoDBHandler;
 import com.ericsson.ei.rules.RulesObject;
 import com.ericsson.ei.rules.test.TestRulesObject;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.mongodb.MongoClient;
 
 import de.flapdoodle.embed.mongo.MongodExecutable;
@@ -80,7 +84,7 @@ public class ObjectHandlerTest {
 
     static String dataBaseName = "EventStorageDBbbb";
     static String collectionName = "SampleEvents";
-    static String input = "{\"TemplateName\":\"ARTIFACT_1\",\"id\":\"eventId\",\"type\":\"eventType11\",\"test_cases\" : [{\"event_id\" : \"testcaseid1\", \"test_data\" : \"testcase1data\"},{\"event_id\" : \"testcaseid2\", \"test_data\" : \"testcase2data\"}]}";
+    static String input = "{\"TemplateName\":\"ARTIFACT_1\",\"id\":\"eventId\",\"type\":\"eventType11\",\"test_cases\":[{\"event_id\":\"testcaseid1\",\"test_data\":\"testcase1data\"},{\"event_id\":\"testcaseid2\",\"test_data\":\"testcase2data\"}]}";
     String updateInput = "{\"TemplateName\":\"ARTIFACT_1\",\"id\":\"eventId\",\"type\":\"eventType11\",\"test_cases\" : [{\"event_id\" : \"testcaseid1\", \"test_data\" : \"testcase2data\"},{\"event_id\" : \"testcaseid3\", \"test_data\" : \"testcase3data\"}]}";
     static String condition = "{\"_id\" : \"eventId\"}";
     static String event = "{\"meta\":{\"id\":\"eventId\"}}";
@@ -102,6 +106,7 @@ public static void init() throws Exception
         objHandler.setCollectionName(collectionName);
         objHandler.setDatabaseName(dataBaseName);
         objHandler.setSubscriptionHandler(subscriptionHandler);
+        
 
         try {
             String rulesString = FileUtils.readFileToString(new File(inputFilePath));
@@ -118,7 +123,8 @@ public static void init() throws Exception
     public void test() {
         String document = objHandler.findObjectById("eventId");
         JsonNode result = objHandler.getAggregatedObject(document);
-        assertEquals(input, result.asText());
+
+        assertEquals(input, result.toString());
     }
 
     @AfterClass
diff --git a/src/test/resources/AggregatedDocument.json b/src/test/resources/AggregatedDocument.json
@@ -1 +1 @@
-{ "TemplateName":"\"ARTIFACT_1\"", "_id" : "6acc3c87-75e0-4b6d-88f5-b1a5d4e62b43" , "aggregatedObject" : "{\"fileInformation\":[{\"extension\":\"jar\",\"classifier\":\"\"}],\"buildCommand\":null,\"confidenceLevels\":[{\"eventId\":\"f37d59a3-069e-4f4c-8cc5-a52e73501a75\",\"name\":\"readyForDelivery\",\"time\":1481875944272,\"value\":\"SUCCESS\"}],\"testCaseExecutions\":[{\"testCaseFinishEventId\":\"11109351-41e0-474a-bc1c-f6e81e58a1c9\",\"testCaseStartedTime\":1481875925916,\"testCaseStartedEventId\":\"cb9d64b0-a6e9-4419-8b5d-a650c27c59ca\",\"testCaseFinishedTime\":1481875935919,\"testCaseDuration\":10003,\"testCase\":{\"conclusion\":\"SUCCESSFUL\",\"verdict\":\"PASSED\",\"tracker\":\"My Other Test Management System\",\"id\":\"TC5\",\"uri\":\"https:\\/\\/other-tm.company.com\\/testCase\\/TC5\"}}],\"id\":\"6acc3c87-75e0-4b6d-88f5-b1a5d4e62b43\",\"time\":1481875891763,\"type\":\"EiffelArtifactCreatedEvent\",\"gav\":{\"groupId\":\"com.mycompany.myproduct\",\"artifactId\":\"sub-system\",\"version\":\"1.1.0\"},\"publications\":[{\"eventId\":\"33d05e6f-9bd9-4138-83b6-e20cc74680a3\",\"locations\":[{\"type\":\"PLAIN\",\"uri\":\"https:\\/\\/myrepository.com\\/mySubSystemArtifact\"}],\"time\":1481875921763}]}"}
+{ "TemplateName":"ARTIFACT_1", "_id" : "6acc3c87-75e0-4b6d-88f5-b1a5d4e62b43" , "aggregatedObject" : {"fileInformation":[{"extension":"jar","classifier":""}],"buildCommand":null,"confidenceLevels":[{"eventId":"f37d59a3-069e-4f4c-8cc5-a52e73501a75","name":"readyForDelivery","time":1481875944272,"value":"SUCCESS"}],"testCaseExecutions":[{"testCaseFinishEventId":"11109351-41e0-474a-bc1c-f6e81e58a1c9","testCaseStartedTime":1481875925916,"testCaseStartedEventId":"cb9d64b0-a6e9-4419-8b5d-a650c27c59ca","testCaseFinishedTime":1481875935919,"testCaseDuration":10003,"testCase":{"conclusion":"SUCCESSFUL","verdict":"PASSED","tracker":"My Other Test Management System","id":"TC5","uri":"https://other-tm.company.com/testCase/TC5"}}],"id":"6acc3c87-75e0-4b6d-88f5-b1a5d4e62b43","time":1481875891763,"type":"EiffelArtifactCreatedEvent","gav":{"groupId":"com.mycompany.myproduct","artifactId":"sub-system","version":"1.1.0"},"publications":[{"eventId":"33d05e6f-9bd9-4138-83b6-e20cc74680a3","locations":[{"type":"PLAIN","uri":"https://myrepository.com/mySubSystemArtifact"}],"time":1481875921763}]}}
diff --git a/src/test/resources/AggregatedDocument2.json b/src/test/resources/AggregatedDocument2.json
@@ -1 +1 @@
-{ "TemplateName":"\"ARTIFACT_1\"", "_id":"ccce572c-c364-441e-abc9-b62fed080ca2","aggregatedObject":"{\"fileInformation\":[{\"extension\":\"jar\",\"classifier\":\"debug\"},{\"extension\":\"\",\"classifier\":\"test\"},{\"extension\":\"exe\",\"classifier\":\"\"}],\"buildCommand\":null,\"testCaseExecutions\":[{\"testCaseFinishEventId\":\"d7de0b67-fbc3-4581-93fd-c2454737d71d\",\"testCaseStartedTime\":1473177170586,\"testCaseStartedEventId\":\"bcc98cef-707c-46be-ad89-392047f05a51\",\"testCaseFinishedTime\":1473177180589,\"testCaseDuration\":10003,\"testCase\":{\"verdict\":\"PASSED\"}}],\"confidenceLevels\":[{\"eventId\":\"f463feef-afd2-4cf1-a42b-ce144eb97024\",\"name\":\"readyForSystemIntegration\",\"time\":1473177188942,\"value\":\"SUCCESS\"}],\"id\":\"ccce572c-c364-441e-abc9-b62fed080ca2\",\"time\":1473177136433,\"type\":\"EiffelArtifactCreatedEvent\",\"gav\":{\"groupId\":\"com.mycompany.myproduct\",\"artifactId\":\"sub-system\",\"version\":\"1.53.0\"},\"publications\":[{\"eventId\":\"0d93f0ce-1eab-46d1-80e3-207393c899df\",\"locations\":[{\"type\":\"PLAIN\",\"uri\":\"https:\\/\\/myrepository.com\\/mySubSystemArtifact\"},{\"type\":\"ARTTIFACTORY\",\"uri\":\"https:\\/\\/another.com\"}],\"time\":1473177166433}]}"}
+{ "TemplateName":"ARTIFACT_1", "_id":"ccce572c-c364-441e-abc9-b62fed080ca2","aggregatedObject": {"fileInformation":[{"extension":"jar","classifier":"debug"},{"extension":"","classifier":"test"},{"extension":"exe","classifier":""}],"buildCommand":null,"testCaseExecutions":[{"testCaseFinishEventId":"d7de0b67-fbc3-4581-93fd-c2454737d71d","testCaseStartedTime":1473177170586,"testCaseStartedEventId":"bcc98cef-707c-46be-ad89-392047f05a51","testCaseFinishedTime":1473177180589,"testCaseDuration":10003,"testCase":{"verdict":"PASSED"}}],"confidenceLevels":[{"eventId":"f463feef-afd2-4cf1-a42b-ce144eb97024","name":"readyForSystemIntegration","time":1473177188942,"value":"SUCCESS"}],"id":"ccce572c-c364-441e-abc9-b62fed080ca2","time":1473177136433,"type":"EiffelArtifactCreatedEvent","gav":{"groupId":"com.mycompany.myproduct","artifactId":"sub-system","version":"1.53.0"},"publications":[{"eventId":"0d93f0ce-1eab-46d1-80e3-207393c899df","locations":[{"type":"PLAIN","uri":"https://myrepository.com/mySubSystemArtifact"},{"type":"ARTTIFACTORY","uri":"https://another.com"}],"time":1473177166433}]}}

Original file line number	Diff line number	Diff line change
`@@ -235,7 +235,7 @@ protected void checkResult() {`
`235`	`235`	`JsonNode expectedJson = objectmapper.readTree(expectedDocument);`
`236`	`236`	`JsonNode actualJson = objectmapper.readTree(document);`
`237`	`237`	`String breakString = "breakHere";`
`238`		`- assertEquals(expectedJson.toString().length(), actualJson.toString().length());`
	`238`	`+ assertEquals("\nExpectedJson:\n" + expectedJson.toString() + "\nActual:\n" + actualJson.toString() + "\n", expectedJson.toString().length(), actualJson.toString().length());`
`239`	`239`	`} catch (IOException e) {`
`240`	`240`	`log.info(e.getMessage(),e);`
`241`	`241`	`}`