Skip to content

Commit 9a55c7d

Browse files
author
Anders Breid
authored
Add support for replica sets using URI (#393)
* Add support for spring.data.mongodb.uri * Removed possibility to set host, port, username and password fields for Mongo. * Update tests to use encrypted password * Manual test script for replica set tests
1 parent c7e526e commit 9a55c7d

File tree

24 files changed

+879
-310
lines changed

24 files changed

+879
-310
lines changed

.travis.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,10 @@ jobs:
3939
# To ensure docker containers are fully up and running we sleep 60s
4040
before_script:
4141
- source src/main/docker/env.bash
42-
- docker-compose -f src/main/docker/docker-compose.yml up -d mongodb rabbitmq eiffel-er jenkins mail-server
42+
- docker-compose -f src/main/docker/docker-compose.yml up -d mongodb mongodb-auth rabbitmq eiffel-er jenkins mail-server
4343
- sleep 60
4444
script:
45-
- mvn verify -DskipUTs -Der.url=http://localhost:8084/search/ -Drabbitmq.exchange.name=ei-exchange -Dspring.mail.host=localhost -Dspring.mail.port=1025 -Dwaitlist.fixedRateResend=1 -B
45+
- mvn verify -DskipUTs -Djasypt.encryptor.password=integrationtest -Dspring.config.location=src/integrationtests/resources/application.properties -B
4646
- stage: deploy
4747
name: deployGitHubPages
4848
script: skip # do not run default test scripts

pom.xml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -537,7 +537,6 @@
537537
<skipITs>${skipITs}</skipITs>
538538
<systemPropertyVariables>
539539
<systemTest>true</systemTest>
540-
<er.url>http://localhost:8080/eventrepository/search/</er.url>
541540
</systemPropertyVariables>
542541
<excludes>
543542
<exclude>**/test/*</exclude>
Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
#! /bin/bash
2+
3+
mongodb_uri="mongodb://replica1:27017,replica2:27017,replica3:27017"
4+
mongodb_database="ei-test-db"
5+
mailhog_uri="mongodb://mongodb:27017"
6+
rabbitmg_host="rabbitmq"
7+
jenkins_host="jenkins"
8+
jenkins_port="8080"
9+
mail_server_host="mail-server"
10+
docker_network="docker_eiffel_2.0_1"
11+
12+
test_command="mvn verify \
13+
-DskipUTs \
14+
-Der.url=http://eiffel-er:8080/search/ \
15+
-Drabbitmq.host=$rabbitmg_host \
16+
-Drabbitmq.exchange.name=ei-exchange \
17+
-Dspring.data.mongodb.uri=$mongodb_uri \
18+
-Dspring.data.mongodb.database=$mongodb_database \
19+
-Dmailhog.uri=$mailhog_uri \
20+
-Djenkins.host=$jenkins_host \
21+
-Djenkins.port=$jenkins_port \
22+
-Dspring.mail.host=$mail_server_host \
23+
-Dspring.mail.port=1025 \
24+
-Dwaitlist.fixedRateResend=1 -B"
25+
26+
function invalid_arguments {
27+
echo "missing command(s) setup | test"
28+
echo " "
29+
echo " setup :: start the docker containers and initiate replica set"
30+
echo " teardown :: stop the docker containers"
31+
echo " test :: execute integration tests"
32+
echo " "
33+
exit;
34+
}
35+
36+
function do_setup {
37+
echo "########################################"
38+
echo "###### STARTING DOCKER SERVICES ########"
39+
echo "########################################"
40+
41+
source src/main/docker/env.bash
42+
docker-compose -f src/main/docker/docker-compose.yml up -d mongodb rabbitmq eiffel-er jenkins mail-server
43+
docker-compose -f src/main/docker/docker-compose-replica-set.yml up -d
44+
45+
echo "Waiting 10 seconds for services to start"
46+
sleep 10
47+
echo 'rs.initiate( {
48+
_id : "rs0",
49+
members: [
50+
{ _id: 0, host: "replica1:27017" },
51+
{ _id: 1, host: "replica2:27017" },
52+
{ _id: 2, host: "replica3:27017" }
53+
]
54+
} )' | mongo localhost:27001 --quiet
55+
}
56+
57+
function do_teardown {
58+
echo "########################################"
59+
echo "###### STOPPING DOCKER SERVICES ########"
60+
echo "########################################"
61+
source src/main/docker/env.bash
62+
docker-compose -f src/main/docker/docker-compose.yml down
63+
docker-compose -f src/main/docker/docker-compose-replica-set.yml down
64+
65+
}
66+
67+
function do_test {
68+
echo "########################################"
69+
echo "###### EXECUTING TESTS ########"
70+
echo "########################################"
71+
docker run -it --rm --network=$docker_network -v $PWD:$PWD -v ~/.m2:/root/.m2 \
72+
-w $PWD maven:3.6-jdk-8 $test_command
73+
}
74+
75+
if [ $# -eq 0 ]; then
76+
invalid_arguments
77+
fi
78+
79+
if [[ "setup test teardown" =~ .*${1}.* ]];
80+
then
81+
while [ "$1" != "" ]
82+
do
83+
echo "Executing command: '$1'"
84+
case "$1" in
85+
setup) do_setup;;
86+
test) do_test;;
87+
teardown) do_teardown;;
88+
esac
89+
shift
90+
done
91+
else
92+
invalid_arguments
93+
STATUS=1
94+
fi
95+
96+
exit $STATUS

src/functionaltests/java/com/ericsson/ei/scaling/ScalingAndFailoverSteps.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ public void additional_instances_closed() throws Exception {
136136
@Then("^all event messages are processed$")
137137
public void messages_processed() throws Exception {
138138
int extraCheckDelay = 0;
139-
List<String> missingEventIds = dbManager.verifyEventsInDB(eventsIdList, extraCheckDelay);
139+
List<String> missingEventIds = super.dbManager.verifyEventsInDB(eventsIdList, extraCheckDelay);
140140
LOGGER.debug("Missing events: {}", missingEventIds.toString());
141141
assertEquals("Number of events missing in DB: " + missingEventIds.size(), 0, missingEventIds.size());
142142
}

src/functionaltests/java/com/ericsson/ei/utils/DataBaseManager.java

Lines changed: 82 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import org.springframework.stereotype.Component;
1313

1414
import com.mongodb.MongoClient;
15+
import com.mongodb.MongoClientURI;
1516
import com.mongodb.client.MongoCollection;
1617
import com.mongodb.client.MongoDatabase;
1718
import com.mongodb.client.model.Filters;
@@ -22,12 +23,14 @@
2223

2324
@Component
2425
public class DataBaseManager {
25-
26+
private static final int MAX_WAIT_TIME_MILLISECONDS = 30000;
27+
private static final int RETRY_EVERY_X_MILLISECONDS = 1000;
28+
2629
@Value("${spring.data.mongodb.database}")
2730
private String database;
2831

2932
@Value("${event_object_map.collection.name}")
30-
private String eventMapCollection;
33+
private String eventObjectCollectionName;
3134

3235
@Value("${aggregated.collection.name}")
3336
private String aggregatedCollectionName;
@@ -44,30 +47,22 @@ public class DataBaseManager {
4447

4548
private MongoClient mongoClient;
4649

47-
public int getMongoDbPort() {
48-
return mongoProperties.getPort();
49-
}
50-
51-
public String getMongoDbHost() {
52-
return mongoProperties.getHost();
53-
}
54-
5550
/**
5651
* Verify that aggregated object contains the expected information.
5752
*
58-
* @param checklist
59-
* list of checklist to check
53+
* @param checklist list of checklist to check
6054
* @return list of missing checklist
6155
* @throws InterruptedException
6256
*/
63-
public List<String> verifyAggregatedObjectInDB(List<String> checklist) throws InterruptedException {
64-
long stopTime = System.currentTimeMillis() + 30000;
57+
public List<String> verifyAggregatedObjectInDB(List<String> checklist)
58+
throws InterruptedException {
59+
long stopTime = System.currentTimeMillis() + MAX_WAIT_TIME_MILLISECONDS;
6560
while (!checklist.isEmpty() && stopTime > System.currentTimeMillis()) {
66-
checklist = compareArgumentsWithAggregatedObjectInDB(checklist);
61+
checklist = getListOfArgumentsNotFoundInDatabase(checklist);
6762
if (checklist.isEmpty()) {
6863
break;
6964
}
70-
TimeUnit.MILLISECONDS.sleep(1000);
65+
TimeUnit.MILLISECONDS.sleep(RETRY_EVERY_X_MILLISECONDS);
7166
}
7267
return checklist;
7368
}
@@ -79,82 +74,81 @@ public List<String> verifyAggregatedObjectInDB(List<String> checklist) throws In
7974
* @throws InterruptedException
8075
*/
8176
public boolean verifyAggregatedObjectExistsInDB() throws InterruptedException {
82-
long stopTime = System.currentTimeMillis() + 30000;
77+
long stopTime = System.currentTimeMillis() + MAX_WAIT_TIME_MILLISECONDS;
8378
while (stopTime > System.currentTimeMillis()) {
84-
mongoClient = new MongoClient(getMongoDbHost(), getMongoDbPort());
85-
MongoDatabase db = mongoClient.getDatabase(database);
86-
MongoCollection<Document> collection = db.getCollection(aggregatedCollectionName);
87-
List<Document> documents = collection.find().into(new ArrayList<>());
88-
TimeUnit.MILLISECONDS.sleep(1000);
79+
List<Document> documents = getDocumentsFromCollection(aggregatedCollectionName);
8980
if (!documents.isEmpty()) {
9081
return true;
9182
}
83+
TimeUnit.MILLISECONDS.sleep(RETRY_EVERY_X_MILLISECONDS);
9284
}
9385
return false;
9486
}
9587

96-
/**
97-
* Checks that aggregated object contains specified arguments.
98-
*
99-
* @param checklist
100-
* list of arguments
101-
* @return list of missing arguments
102-
*/
103-
private List<String> compareArgumentsWithAggregatedObjectInDB(List<String> checklist) {
104-
mongoClient = new MongoClient(getMongoDbHost(), getMongoDbPort());
105-
MongoDatabase db = mongoClient.getDatabase(database);
106-
MongoCollection<Document> collection = db.getCollection(aggregatedCollectionName);
107-
List<Document> documents = collection.find().into(new ArrayList<>());
108-
for (Document document : documents) {
109-
for (String expectedValue : new ArrayList<>(checklist)) {
110-
if (document.toString().contains(expectedValue)) {
111-
checklist.remove(expectedValue);
112-
}
113-
}
114-
}
115-
return checklist;
116-
}
117-
11888
/**
11989
* Verify that events are located in the database collection.
12090
*
121-
* @param eventsIdList
122-
* list of events IDs
91+
* @param eventsIdList list of events IDs
12392
* @return list of missing events
12493
* @throws InterruptedException
12594
*/
126-
public List<String> verifyEventsInDB(List<String> eventsIdList, int extraCheckDelay) throws InterruptedException {
127-
long stopTime = System.currentTimeMillis() + 30000 + extraCheckDelay;
95+
public List<String> verifyEventsInDB(List<String> eventsIdList, int extraCheckDelay)
96+
throws InterruptedException {
97+
long stopTime = System.currentTimeMillis() + MAX_WAIT_TIME_MILLISECONDS + extraCheckDelay;
12898
while (!eventsIdList.isEmpty() && stopTime > System.currentTimeMillis()) {
12999
eventsIdList = compareSentEventsWithEventsInDB(eventsIdList);
130100
if (eventsIdList.isEmpty()) {
131101
break;
132102
}
133-
TimeUnit.MILLISECONDS.sleep(1000);
103+
TimeUnit.MILLISECONDS.sleep(RETRY_EVERY_X_MILLISECONDS);
134104
}
135105
return eventsIdList;
136106
}
137107

108+
private List<String> getListOfArgumentsNotFoundInDatabase(final List<String> checklist) {
109+
final List<Document> documents = getDocumentsFromCollection(aggregatedCollectionName);
110+
111+
List<String> foundValues = new ArrayList<>();
112+
for (Document document : documents) {
113+
final List<String> valuesFoundInDocument = getValuesFoundInDocument(checklist,
114+
document);
115+
foundValues.addAll(valuesFoundInDocument);
116+
}
117+
118+
List<String> result = new ArrayList<>(checklist);
119+
result.removeAll(foundValues);
120+
return result;
121+
}
122+
123+
private List<String> getValuesFoundInDocument(final List<String> checklist,
124+
final Document document) {
125+
List<String> foundValues = new ArrayList<>();
126+
for (final String expectedValue : checklist) {
127+
if (document.toString().contains(expectedValue)) {
128+
foundValues.add(expectedValue);
129+
}
130+
}
131+
return foundValues;
132+
}
133+
138134
/**
139135
* Checks collection of events against event list.
140136
*
141-
* @param checklist
142-
* list of event IDs
137+
* @param checklist list of event IDs
143138
* @return list of missing events
144139
*/
145-
private List<String> compareSentEventsWithEventsInDB(List<String> checklist) {
146-
mongoClient = new MongoClient(getMongoDbHost(), getMongoDbPort());
147-
MongoDatabase db = mongoClient.getDatabase(database);
148-
MongoCollection<Document> collection = db.getCollection(eventMapCollection);
149-
List<Document> documents = collection.find().into(new ArrayList<>());
140+
private List<String> compareSentEventsWithEventsInDB(final List<String> checklist) {
141+
final List<Document> documents = getDocumentsFromCollection(eventObjectCollectionName);
142+
143+
List<String> foundIDs = new ArrayList<>();
150144
for (Document document : documents) {
151-
for (String expectedID : new ArrayList<>(checklist)) {
152-
if (expectedID.equals(document.get("_id").toString())) {
153-
checklist.remove(expectedID);
154-
}
155-
}
145+
final String documentId = document.get("_id").toString();
146+
foundIDs.add(documentId);
156147
}
157-
return checklist;
148+
149+
List<String> result = new ArrayList<>(checklist);
150+
result.removeAll(foundIDs);
151+
return result;
158152
}
159153

160154
/**
@@ -171,31 +165,41 @@ public String getValueFromQuery(List<String> databaseQueryResult, String key, in
171165
return jsonObject.get(key).toString();
172166
}
173167

174-
/**
175-
* Returns the size of the waitlist.
176-
*
177-
* @return int of the size of the waitlist.
178-
*/
179-
public int waitListSize() {
180-
mongoClient = new MongoClient(getMongoDbHost(), getMongoDbPort());
181-
MongoDatabase db = mongoClient.getDatabase(database);
182-
MongoCollection<Document> collection = db.getCollection(waitlistCollectionName);
183-
List<Document> documents = collection.find().into(new ArrayList<>());
184-
return documents.size();
185-
}
186-
187168
/**
188169
* Get a specific subscription from the database based on the subscription name.
189170
*
190171
* @param subscriptionName
191172
* @return the document as JSON string
192173
*/
193174
public String getSubscription(String subscriptionName) {
194-
mongoClient = new MongoClient(getMongoDbHost(), getMongoDbPort());
195-
MongoDatabase db = mongoClient.getDatabase(database);
196-
MongoCollection<Document> collection = db.getCollection(subscriptionCollectionName);
175+
MongoCollection<Document> collection = getCollection(subscriptionCollectionName);
197176
Bson filter = Filters.eq("subscriptionName", subscriptionName);
198177
Document document = collection.find(filter).first();
199178
return document.toJson();
200179
}
180+
181+
/**
182+
* Returns the size of the waitlist.
183+
*
184+
* @return int of the size of the waitlist.
185+
*/
186+
public int waitListSize() {
187+
List<Document> documents = getDocumentsFromCollection(waitlistCollectionName);
188+
return documents.size();
189+
}
190+
191+
private List<Document> getDocumentsFromCollection(String collectionName) {
192+
MongoCollection<Document> collection = getCollection(collectionName);
193+
List<Document> documents = collection.find().into(new ArrayList<>());
194+
return documents;
195+
}
196+
197+
private MongoCollection<Document> getCollection(String collectionName) {
198+
MongoClientURI uri = new MongoClientURI(mongoProperties.getUri());
199+
mongoClient = new MongoClient(uri);
200+
MongoDatabase db = mongoClient.getDatabase(database);
201+
MongoCollection<Document> collection = db.getCollection(collectionName);
202+
return collection;
203+
}
204+
201205
}

0 commit comments

Comments
 (0)