various test enhancements and MongoDB, Postgres, and Kafka to Oracle DB with TEQ/AQ work (#129)

Author: paulparkinson

grabdish/destroy-multicloud.sh

@@ -7,8 +7,21 @@
 echo Deleting the Verrazzano custom resource....
 kubectl delete verrazzano example-verrazzano
 
+echo Deleting the Verrazzano operator
+kubectl delete -f https://github.com/verrazzano/verrazzano/releases/latest/download/operator.yaml
+
 echo Set verrazzano-managed=false istio-injection=disabled ...
 kubectl label namespace msdataworkshop verrazzano-managed=false istio-injection=disabled --overwrite
 
-echo Viewing the uninstall logs...
-kubectl logs -f $(kubectl get pod -l job-name=uninstall-example-verrazzano -o jsonpath="{.items[0].metadata.name}")
+echo restarting/deleting pods to remove any envoy/sidecars
+export SERVICES="frontend order inventory supplier"
+for s in $SERVICES; do
+  echo "deletepod $s ..."
+  deletepod ${s}
+done
+
+echo pods...
+pods
+
+echo If necessary the uninstall logs may be viewed here...
+echo kubectl logs -f $(kubectl get pod -l job-name=uninstall-example-verrazzano -o jsonpath="{.items[0].metadata.name}")

grabdish/docs/AutoTest.md

@@ -44,14 +44,17 @@ sed -i.bak '/grabdish/d' ~/.bashrc
 echo "source $PWD/microservices-datadriven/grabdish/env.sh" >>~/.bashrc
 source microservices-datadriven/grabdish/env.sh
 source setup.sh
+
 ```
 
 8. Execute the test
 ```
 source test.sh
+
 ```
 
 9. Clean up
 ```
 source destroy.sh
+
 ```

grabdish/inventory-nodejs/inventory/app.js

@@ -165,7 +165,7 @@ async function processOrder() {
   let opStart = process.hrtime.bigint();
 
   try {
-//    oracledb.autoCommit = false;
+    oracledb.autoCommit = false;
     connection = await oracledb.getConnection();
     const orderQueue = await connection.getQueue(queueConfig.orderQueue, queueOptions);
     logStats(opName, opStart, process.hrtime.bigint());

grabdish/mongodb-kafka-postgres/install-mongodb.sh

@@ -5,4 +5,3 @@
 kubectl create -f mongodata-persistentvolumeclaim.yaml -n msdataworkshop
 kubectl create -f mongodb-deployment.yaml -n msdataworkshop
 kubectl create -f mongodb-service.yaml -n msdataworkshop
-#kubectl expose deployment mongodb --type=LoadBalancer -n msdataworkshop # remove as we have service

grabdish/mongodb-kafka-postgres/uninstall-all.sh

@@ -2,4 +2,16 @@
 ## Copyright (c) 2021 Oracle and/or its affiliates.
 ## Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
-kubectl delete -f ./kafka-all.yaml
+echo un-install kafka...
+kubectl delete -f ./kafka-all.yaml -n msdataworkshop
+
+echo un-install mongodb...
+kubectl delete -f mongodb-service.yaml -n msdataworkshop
+kubectl delete -f mongodb-deployment.yaml -n msdataworkshop
+kubectl delete -f mongodata-persistentvolumeclaim.yaml -n msdataworkshop
+
+echo un-install postgres
+kubectl delete -f postgres-service.yaml -n msdataworkshop
+kubectl delete -f postgres-deployment.yaml -n msdataworkshop
+kubectl delete -f postgres-storage.yaml -n msdataworkshop
+

grabdish/setup-multicloud.sh

@@ -2,11 +2,14 @@
 # Copyright (c) 2021 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
+# Fail on error
+set -e
+
 # The following setup and install of Verrazzano is taken directly from https://verrazzano.io/docs/setup/quickstart/
 if [[ $1 == "" ]]
 then
   echo CLUSTER_NAME argument not provided
-  echo This can be found in the ~./kube/config file
+  echo This can be found in the ~/.kube/config file
   echo Usage example : ./setup-multicloud.sh cluster-cyxypetwerq
   exit
 fi

grabdish/test

@@ -8,6 +8,10 @@ if ! (return 0 2>/dev/null); then
   exit
 fi
 
+echo Deleting any previous $GRABDISH_LOG/main-test.log
+rm $GRABDISH_LOG/main-test.log
+rm $GRABDISH_LOG/testlog-*
+
 $GRABDISH_HOME/utils/main-test.sh 2>&1 | tee -ai $GRABDISH_LOG/main-test.log
 
 $GRABDISH_HOME/utils/test-summary.sh

grabdish/test.sh

@@ -5,4 +5,76 @@
 # Fail on error
 set -e
 
-# todo...
+echo "TEST_LOG: Crash Recovery Install MongoDB, Postgres, and Kafka..."
+#Install MongoDB, Postgres, and Kafka
+cd $GRABDISH_HOME/mongodb-kafka-postgres;./install-all.sh
+echo "TEST_LOG: Crash Recovery waiting for kafka-broker kafka-cat mongodb postgres to start..."
+SERVICES="kafka-broker kafka-cat mongodb postgres"
+
+for s in $SERVICES; do
+  echo "Verify $s pod running..."
+  while test 1 -gt `kubectl get pods -n msdataworkshop | grep "${s}" | grep "1/1" | wc -l`; do
+    echo "Waiting for pod to start..."
+    sleep 5
+  done
+done
+
+echo "TEST_LOG: Crash Recovery kafka-broker kafka-cat mongodb postgres started"
+
+#Undeploy Order, Inventory, and Supplier Services and deploy the MongoDB, Postgres, and Kafka backed Order and Inventory implementations
+for i in order-helidon inventory-helidon supplier-helidon-se; do cd $GRABDISH_HOME/$i; ./undeploy.sh; done
+cd $GRABDISH_HOME/order-mongodb-kafka ; ./deploy.sh
+cd $GRABDISH_HOME/inventory-postgres-kafka ; ./deploy.sh
+cd $GRABDISH_HOME
+
+echo "TEST_LOG: Crash Recovery undeployed all and deployed mongodb and postgres services"
+
+SERVICES="order-mongodb-kafka inventory-postgres-kafka"
+for s in $SERVICES; do
+  echo "Verify $s pod running..."
+  while test 1 -gt `kubectl get pods -n msdataworkshop | grep "${s}" | grep "1/1" | wc -l`; do
+    echo "Waiting for pod to start..."
+    sleep 5
+  done
+done
+
+echo "TEST_LOG: Crash Recovery mongodb and postgres pods running "
+
+#Run tests against order-mongodb-kafka inventory-postgres-kafka implementations
+SERVICES="mongodb-postgres-kafka"
+ORDER_ID=65
+
+for s in $SERVICES; do
+  echo "Testing $s"
+  cd $GRABDISH_HOME
+  ORDER_ID=$(($ORDER_ID + 1000))
+  utils/func-test.sh "Crash test (success runs) $s" $ORDER_ID $s
+done
+
+echo "TEST_LOG: Crash Recovery mongodb and postgres tests complete "
+
+#Undeploy MongoDB, Postgres, and Kafka backed Order and Inventory implementations and deploy the Oracle + TEQ/AQ backed Order and Inventory implementations by copying and running the following commands.
+for i in order-mongodb-kafka inventory-postgres-kafka; do cd $GRABDISH_HOME/$i; ./undeploy.sh; done
+cd $GRABDISH_HOME/order-helidon ; ./deploy.sh
+cd $GRABDISH_HOME/inventory-helidon ; ./deploy.sh
+cd $GRABDISH_HOME/supplier-helidon-se ; ./deploy.sh
+cd $GRABDISH_HOME
+
+#Un-install MongoDB, Postgres, and Kafka
+cd $GRABDISH_HOME/mongodb-kafka-postgres;./uninstall-all.sh
+
+echo "TEST_LOG: Crash Recovery undeployed mongodb and postgres "
+
+#Run tests against Oracle DB + AQ implementations
+SERVICES="Oracle-TEQ"
+ORDER_ID=65
+
+# currently fails order 1165 due to "success inventory exists"
+#for s in $SERVICES; do
+#  echo "Testing $s"
+#  cd $GRABDISH_HOME
+#  ORDER_ID=$(($ORDER_ID + 1100))
+#  utils/func-test.sh "Crash test (success runs) $s" $ORDER_ID $s
+#done
+
+echo "TEST_LOG: Crash Recovery Oracle TEQ/AQ tests complete "

grabdish/utils/crashrecovery-test.sh

@@ -7,7 +7,7 @@ set -e
 
 TEST_STEP="$1"
 ORDER_ID="$2"
-TEST_SERVICE="$3"
+TEST_SERVICE="$3" #currently used solely to name log file
 
 function order() {
   echo '{"serviceName": "order", "commandName": "'"$2"'", "orderId": '"$1"', "orderItem": "sushi", "deliverTo": "780 PANORAMA DR, San francisco, CA"}'
@@ -20,7 +20,7 @@ function inventory() {
 function placeOrderTest() {
   # Place order
   local ORDER_ID="$1"
-  if wget --http-user grabdish --http-password "$TEST_UI_PASSWORD" --no-check-certificate --post-data "$(order "$ORDER_ID" 'placeOrder')" \
+  if wget -q --http-user grabdish --http-password "$TEST_UI_PASSWORD" --no-check-certificate --post-data "$(order "$ORDER_ID" 'placeOrder')" \
     --header='Content-Type: application/json' "$(state_get FRONTEND_URL)/placeorder" -O $GRABDISH_LOG/order; then
     echo "TEST_LOG: $TEST_STEP placeOrder $ORDER_ID succeeded"
   else
@@ -29,22 +29,42 @@ function placeOrderTest() {
 }
 
 function showOrderTest() {
+  echo "TEST_LOG: sleep for 20"
+  sleep 20
   # Show order 
   local ORDER_ID="$1"
   local SEARCH_FOR="$2"
-  if wget --http-user grabdish --http-password "$TEST_UI_PASSWORD" --no-check-certificate --post-data "$(order "$ORDER_ID" 'showorder')" \
+  if wget -q --http-user grabdish --http-password "$TEST_UI_PASSWORD" --no-check-certificate --post-data "$(order "$ORDER_ID" 'showorder')" \
     --header='Content-Type: application/json' "$(state_get FRONTEND_URL)/command" -O $GRABDISH_LOG/order; then
     echo "TEST_LOG: $TEST_STEP showOrder request $1 succeeded"
     if grep "$SEARCH_FOR" $GRABDISH_LOG/order >/dev/null; then
-      echo "TEST_LOG: $TEST_STEP showOrder $ORDER_ID matched"
+      echo "TEST_LOG: $TEST_STEP showOrder $ORDER_ID matched '$SEARCH_FOR'"
     else
-      echo "TEST_LOG_FAILED: $TEST_STEP showOrder $ORDER_ID nomatch"
+      echo "TEST_LOG_FAILED: $TEST_STEP showOrder $ORDER_ID expected '$SEARCH_FOR' but got... " | tr '\n' ' ' ;cat $GRABDISH_LOG/order
     fi
   else
     echo "TEST_LOG_FAILED: $TEST_STEP showOrder request $1 failed"
   fi
 }
 
+function verifyInventoryCountTest() {
+  local ITEM_ID="$1"
+  local SEARCH_FOR="$2"
+  local ORDER_ID="$3"
+  if wget --http-user grabdish --http-password "$TEST_UI_PASSWORD" --no-check-certificate --post-data "$(inventory "$ITEM_ID" 'getInventory')" \
+    --header='Content-Type: application/json' "$(state_get FRONTEND_URL)/command" -O $GRABDISH_LOG/inventory; then
+    echo "TEST_LOG: $TEST_STEP verifyInventoryCountTest request $1 succeeded"
+    if grep "$SEARCH_FOR" $GRABDISH_LOG/inventory >/dev/null; then
+      echo "TEST_LOG: $TEST_STEP verifyInventoryCountTest $ITEM_ID after ORDER_ID $ORDER_ID expected inventory count: '$SEARCH_FOR'"
+    else
+      echo "TEST_LOG: $TEST_STEP verifyInventoryCountTest $ITEM_ID after ORDER_ID $ORDER_ID unexpected inventory count, not '$SEARCH_FOR'" | tr '\n' ' ' ; cat $GRABDISH_LOG/inventory
+      echo ...
+    fi
+  else
+    echo "TEST_LOG_FAILED: $TEST_STEP verifyInventoryCountTest $ITEM_ID after ORDER_ID $ORDER_ID request $1 failed"
+  fi
+}
+
 function addInventoryTest() {
   # Add inventory 
   local ITEM_ID="$1"
@@ -58,16 +78,18 @@ function addInventoryTest() {
 
 
 # Show order and wait for status "no inventory"
+
+verifyInventoryCountTest "sushi" 0 "$ORDER_ID (before placing order)"
+
 placeOrderTest $ORDER_ID
 
 sleep 10
 
 showOrderTest $ORDER_ID 'failed inventory does not exist'
 
-
-# Add inventory
 addInventoryTest "sushi"
 
+verifyInventoryCountTest "sushi" 1 "$ORDER_ID (after adding inventory)"
 
 # Place second order 
 ORDER_ID=$(($ORDER_ID + 1))
@@ -78,9 +100,10 @@ sleep 10
 
 showOrderTest "$ORDER_ID" 'success inventory exists'
 
-#if [[ $TEST_SERVICE == "dotnet" ]]
+verifyInventoryCountTest "sushi" 0 "$ORDER_ID"
+
 if [[ $TEST_SERVICE != "" ]]
 then
-  echo writing service log to $GRABDISH_LOG/testlog-$TEST_SERVICE-$ORDER_ID
-  logpodnotail $TEST_SERVICE > $GRABDISH_LOG/testlog-$TEST_SERVICE-$ORDER_ID
-fi
+  echo writing log to $GRABDISH_LOG/testlog-$TEST_SERVICE
+  logpodnotail $TEST_SERVICE > $GRABDISH_LOG/testlog-$TEST_SERVICE
+fi

grabdish/utils/func-test.sh

@@ -4,7 +4,7 @@
 
 export k8s_pod=$1
 for line in $(kubectl get pods -n msdataworkshop | \
-  grep $k8s_pod | awk '{print $2}'); do
+  grep $k8s_pod | awk '{print $1}'); do
     echo "kubectl logs -f $line -n msdataworkshop"
     kubectl logs -f $line -n msdataworkshop
 done

grabdish/utils/logpod

@@ -4,7 +4,7 @@
 
 export k8s_pod=$1
 for line in $(kubectl get pods -n msdataworkshop | \
-  grep $k8s_pod | awk '{print $2}'); do
+  grep $k8s_pod | awk '{print $1}'); do
     echo "kubectl logs -f $line -n msdataworkshop"
     kubectl logs $line -n msdataworkshop
 done

grabdish/utils/logpodnotail

@@ -23,7 +23,7 @@ export TEST_UI_PASSWORD=`kubectl get secret frontendadmin -n msdataworkshop --te
 echo 'TEST_LOG: #####################################'
 
 # WALKTHROUGH
-echo "TEST_LOG: #### Testing Lab2: Walkthrough"
+echo "TEST_LOG: #### Testing Lab2: Walkthrough Undeploy..."
 
 # Undeploy to make it rerunable
 ./undeploy.sh
@@ -40,7 +40,7 @@ while test 0 -lt `kubectl get pods -n msdataworkshop | egrep 'frontend-helidon|i
 done
 
 # Deploy the java services
-echo "TEST_LOG: #### Testing Lab2: Walkthrough"
+echo "TEST_LOG: #### Testing Lab2: Walkthrough Deploy..."
 cd $GRABDISH_HOME
 ./deploy.sh
 
@@ -90,26 +90,32 @@ fi
 
 # Functional test on order 66/67
 utils/func-test.sh Walkthrough 66
+logpodnotail frontend > $GRABDISH_LOG/testlog-frontend-from-Walkthrough
+logpodnotail supplier > $GRABDISH_LOG/testlog-supplier-from-Walkthrough
+logpodnotail order > $GRABDISH_LOG/testlog-order-from-Walkthrough
+logpodnotail inventory > $GRABDISH_LOG/testlog-inventory-from-Walkthrough
 
 
 # POLYGLOT
 echo "TEST_LOG: #### Testing Lab3: Polyglot"
-
 # Deploy each inventory service and perform functional test
 while ! $(state_get NON_JAVA_BUILDS); do
   sleep 10
   echo "Waiting for NON_JAVA_BUILDS"
 done
 
 utils/polyglot-test.sh
+echo writing log to $GRABDISH_LOG/testlog-frontend-from-polyglot
+logpodnotail frontend > $GRABDISH_LOG/testlog-frontend-from-polyglot
+logpodnotail supplier > $GRABDISH_LOG/testlog-supplier-from-polyglot
+logpodnotail order > $GRABDISH_LOG/testlog-order-from-polyglot
 
-
-# SCALING
+## SCALING
 echo "TEST_LOG: #### Testing Lab4: Scaling"
 utils/scaling-test.sh
-
-
-# TRACING
+#
+#
+## TRACING
 echo "TEST_LOG: #### Testing Lab5: Tracing"
 utils/tracing-test.sh
 
@@ -120,7 +126,7 @@ utils/tracing-test.sh
 
 # TRACING
 echo "TEST_LOG: #### Testing Lab7: Transactional Tests: Compare MongoDB, Postgres, and Kafka to Oracle DB with TEQ/AQ"
-utils/tracing-test.sh
+utils/crashrecovery-test.sh
 
 # TEARDOWN
 # source destroy.sh

grabdish/utils/main-test.sh

@@ -6,7 +6,8 @@
 set -e
 
 # Deploy each inventory service and perform functional test
-SERVICES="inventory-python inventory-nodejs inventory-dotnet inventory-go inventory-helidon-se"
+#SERVICES="inventory-python inventory-nodejs inventory-dotnet inventory-go inventory-helidon-se"
+SERVICES="inventory-dotnet inventory-go inventory-python inventory-nodejs inventory-helidon-se"
 ORDER_ID=66
 
 cd $GRABDISH_HOME/inventory-helidon