Do not retry crashed tests (#443)

Based on previous changes crashed tests are fatal and
would always return a non-zero exit code.

With this change, the crashed tests are NOT retried
to make sure the app crash errors are surfaced without confusion.
Also added new tests and fixed existing ones as per the new no-retry behavior.

Author: ravimandala

bluepill/tests/BPIntegrationTests.m

@@ -90,8 +90,9 @@ - (void)testClonedSimulators {
     config.failureTolerance = @0;
     config.cloneSimulator = TRUE;
     // need to validate the configuration to fill in simDevice and simRuntime
-    [config validateConfigWithError:nil];
-    NSError *err;
+    NSError *err = nil;
+    [config validateConfigWithError:&err];
+    XCTAssert(err == nil);
     BPApp *app = [BPApp appWithConfig:config
                             withError:&err];
 

bp/src/SimulatorMonitor.m

@@ -195,7 +195,8 @@ - (void)onOutputReceived:(NSString *)output {
             NSString *testClass = (__self.currentClassName ?: __self.previousClassName);
             NSString *testName = (__self.currentTestName ?: __self.previousTestName);
             if (__self.testsState == Running) {
-                [BPUtils printInfo:CRASH withString:@"%@/%@ crashed app.", testClass, testName];
+                [self updateExecutedTestCaseList:testName inClass:testClass];
+                [BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Not retrying it.", testClass, testName];
                 [[BPStats sharedStats] endTimer:[NSString stringWithFormat:TEST_CASE_FORMAT, [BPStats sharedStats].attemptNumber, testClass, testName] withResult:@"CRASHED"];
             } else {
                 assert(__self.testsState == Idle);

bp/tests/BluepillTests.m

@@ -346,11 +346,12 @@ - (void)testReportFailureOnTimeoutCrashAndPass {
 }
 
 /**
- Execution plan: CRASH, TIMEOUT, PASS
+ Execution plan: CRASH
  */
-- (void)testReportFailureOnCrashTimeoutAndPass {
+- (void)testNoRetryOnCrash {
     self.config.stuckTimeout = @6;
-    self.config.testing_ExecutionPlan = @"CRASH TIMEOUT PASS";
+    self.config.testing_ExecutionPlan = @"CRASH";  // No retry
+    self.config.errorRetriesCount = @4;
     self.config.onlyRetryFailed = TRUE;
     NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
     self.config.testBundlePath = testBundlePath;
@@ -364,6 +365,49 @@ - (void)testReportFailureOnCrashTimeoutAndPass {
     XCTAssertTrue(exitCode == BPExitStatusAppCrashed);
 }
 
+
+/**
+ Execution plan: One test CRASHes and another one TIMEs OUT and PASSes on retry
+ */
+- (void)testReportFailureOnCrashAndTimeoutTests {
+    self.config.stuckTimeout = @6;
+    self.config.testing_ExecutionPlan = @"CRASH; SKIP TIMEOUT PASS";
+    self.config.onlyRetryFailed = TRUE;
+    self.config.failureTolerance = @1;
+    self.config.errorRetriesCount = @2;
+    NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
+    self.config.testBundlePath = testBundlePath;
+    NSString *tempDir = NSTemporaryDirectory();
+    NSError *error;
+    NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error];
+    NSLog(@"output directory is %@", outputDir);
+    self.config.outputDirectory = outputDir;
+
+    BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
+    XCTAssertTrue(exitCode == BPExitStatusAppCrashed);
+}
+
+/**
+ Execution plan: One test CRASHes and another one keeps timing out
+ */
+- (void)testReportBothCrashAndTimeout {
+    self.config.stuckTimeout = @6;
+    self.config.testing_ExecutionPlan = @"CRASH; SKIP TIMEOUT TIMEOUT";
+    self.config.onlyRetryFailed = TRUE;
+    self.config.failureTolerance = @1;
+    self.config.errorRetriesCount = @2;
+    NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
+    self.config.testBundlePath = testBundlePath;
+    NSString *tempDir = NSTemporaryDirectory();
+    NSError *error;
+    NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error];
+    NSLog(@"output directory is %@", outputDir);
+    self.config.outputDirectory = outputDir;
+
+    BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
+    XCTAssertTrue(exitCode == (BPExitStatusAppCrashed | BPExitStatusTestTimeout));
+}
+
 /**
  Execution plan: FAIL, TIMEOUT, PASS
  */

bp/tests/Resource Files/crash_tests_with_retry_attempt_2.xml

@@ -1,11 +1,8 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<testsuites name="Selected tests" tests="1" failures="1" errors="0">
-    <testsuite tests="1" failures="1" errors="0" name="BPSampleAppCrashingTests.xctest">
-        <testsuite tests="1" failures="1" errors="0" name="BPSampleAppCrashingTests">
-            <testcase classname="BPSampleAppCrashingTests" name="testAppCrash1">
-                <failure type="Failure" message="App Crashed"></failure>
-                <system-out></system-out>
-            </testcase>
+<testsuites name="Selected tests" tests="1" failures="0" errors="0">
+    <testsuite tests="1" failures="0" errors="0" name="BPSampleAppCrashingTests.xctest">
+        <testsuite tests="1" failures="0" errors="0" name="BPSampleAppCrashingTests">
+            <testcase classname="BPSampleAppCrashingTests" name="testAppCrash2"></testcase>
         </testsuite>
     </testsuite>
 </testsuites>