[testing] Fix e2e tests timed out, but there's empty test log (#2810)

* Wait longer for e2e tests

* Let api integration test stream log output while testing

* Also fix initialization test

* Fix exit code on error

* Update test/initialization-test/run_test.sh

Co-Authored-By: Alexey Volkov <avolkov@google.com>

* Update according to CR

Co-authored-by: Alexey Volkov <alexey.volkov@ark-kun.com>

Author: 2 people

test/api-integration-test/run_test.sh

@@ -15,6 +15,7 @@
 # limitations under the License.
 
 set -xe
+set -o pipefail
 
 # K8s Namespace that all resources deployed to
 NAMESPACE=kubeflow
@@ -64,13 +65,13 @@ cd "${BASE_DIR}/${TEST_DIR}"
 export GO111MODULE=on
 
 echo "Run integration test..."
-TEST_RESULT=`go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true 2>&1`
-TEST_EXIT_CODE=$?
+LOG_FILE=$(mktemp)
+# Note, "set -o pipefail" at top of file is required to catch exit code of the pipe.
+TEST_EXIT_CODE=0 # reference for how to save exit code: https://stackoverflow.com/a/18622662
+go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true |& tee $LOG_FILE || TEST_EXIT_CODE=$?
 
-# Log the test result
-printf '%s\n' "$TEST_RESULT"
 # Convert test result to junit.xml
-printf '%s\n' "$TEST_RESULT" | go-junit-report > ${JUNIT_TEST_RESULT}
+< "$LOG_FILE" go-junit-report > "${JUNIT_TEST_RESULT}"
 
 echo "Copy test result to GCS ${RESULTS_GCS_DIR}/${JUNIT_TEST_RESULT}"
 gsutil cp ${JUNIT_TEST_RESULT} ${RESULTS_GCS_DIR}/${JUNIT_TEST_RESULT}

test/initialization-test/run_test.sh

@@ -15,6 +15,7 @@
 # limitations under the License.
 
 set -xe
+set -o pipefail
 
 # K8s Namespace that all resources deployed to
 NAMESPACE=kubeflow
@@ -64,13 +65,13 @@ cd "${BASE_DIR}/${TEST_DIR}"
 export GO111MODULE=on
 
 echo "Run Initialization test..."
-TEST_RESULT=`go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true 2>&1`
-TEST_EXIT_CODE=$?
+LOG_FILE=$(mktemp)
+# Note, "set -o pipefail" at top of file is required to catch exit code of the pipe.
+TEST_EXIT_CODE=0 # reference for how to save exit code: https://stackoverflow.com/a/18622662
+go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true |& tee $LOG_FILE || TEST_EXIT_CODE=$?
 
-# Log the test result
-printf '%s\n' "$TEST_RESULT"
 # Convert test result to junit.xml
-printf '%s\n' "$TEST_RESULT" | go-junit-report > ${JUNIT_TEST_RESULT}
+< "$LOG_FILE" go-junit-report > "${JUNIT_TEST_RESULT}"
 
 echo "Copy test result to GCS ${RESULTS_GCS_DIR}/${JUNIT_TEST_RESULT}"
 gsutil cp ${JUNIT_TEST_RESULT} ${RESULTS_GCS_DIR}/${JUNIT_TEST_RESULT}

test/presubmit-tests-with-pipeline-deployment.sh

@@ -31,7 +31,7 @@ usage()
 PLATFORM=gcp
 PROJECT=ml-pipeline-test
 TEST_RESULT_BUCKET=ml-pipeline-test
-TIMEOUT_SECONDS=1800
+TIMEOUT_SECONDS=1800 # 30 minutes
 NAMESPACE=kubeflow
 ENABLE_WORKLOAD_IDENTITY=true