Skip to content

Commit 506e399

Browse files
ravimandalaravimandala
authored
Retry app crash tests and consider then non-fatal if they pass (#456)
Introducing a flag `retry-app-crash-tests` to retry tests that might have caused app crashes. When this flag is turned on, the execution will not fail if all app crash tests pass on retry. The retries will honor the `error-retries` count, restarts the sim before retrying the crashed test and makes the crash logs available, like before.
1 parent 08d3951 commit 506e399

File tree

6 files changed

+96
-22
lines changed

6 files changed

+96
-22
lines changed

‎README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ A full list supported options are listed here.
8989
| video-paths | -V | A list of videos that will be saved in the simulators. | N | n/a |
9090
| image-paths | -I | A list of images that will be saved in the simulators. | N | n/a |
9191
| unsafe-skip-xcode-version-check | | Skip Xcode version check | N | NO |
92+
| retry-app-crash-tests | | Retry tests that crashed app and consider it non-fatal if it passes on retry. | N | false |
9293

9394

9495
## Exit Status

‎bp/src/BPConfiguration.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ typedef NS_ENUM(NSInteger, BPProgram) {
7979
@property (nonatomic) BOOL saveDiagnosticsOnError;
8080
@property (nonatomic, strong) NSNumber *failureTolerance;
8181
@property (nonatomic) BOOL onlyRetryFailed;
82+
@property (nonatomic) BOOL retryAppCrashTests;
8283
@property (nonatomic, strong) NSArray *testCasesToSkip;
8384
@property (nonatomic, strong) NSArray *testCasesToRun;
8485
@property (nonatomic, strong) NSArray *allTestCases;

‎bp/src/BPConfiguration.m

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) {
103103
{'q', "quiet", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "quiet",
104104
"Turn off all output except fatal errors."},
105105
{'F', "only-retry-failed", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "onlyRetryFailed",
106-
"Only retry failed tests instead of all. Also retry test that timed-out/crashed. Note that app crashes are fatal even if the test passes on retry."},
106+
"Only retry failed tests instead of all. Also retry test that timed-out/crashed."},
107107
{'l', "list-tests", BP_MASTER, NO, NO, no_argument, NULL, BP_VALUE | BP_BOOL, "listTestsOnly",
108108
"Only list tests and exit without executing tests."},
109109
{'v', "verbose", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "verboseLogging",
@@ -143,7 +143,9 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) {
143143
{364, "test-plan-path", BP_MASTER | BP_SLAVE, NO, NO, required_argument, NULL, BP_VALUE | BP_PATH, "testPlanPath",
144144
"The path of a json file which describes the test plan. It is equivalent to the .xctestrun file generated by Xcode, but it can be generated by a different build system, e.g. Bazel"},
145145
{365, "unsafe-skip-xcode-version-check", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL , "unsafeSkipXcodeVersionCheck",
146-
" "},
146+
"Skip Xcode version check if using an Xcode version that is not officially supported the Bluepill version being used. Not safe/recommended and has a limited support."},
147+
{366, "retry-app-crash-tests", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "retryAppCrashTests",
148+
"Retry the tests after an app crash and if it passes on retry, consider them non-fatal."},
147149

148150
{0, 0, 0, 0, 0, 0, 0}
149151
};

‎bp/src/Bluepill.m

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -631,8 +631,10 @@ - (void)finishWithContext:(BPExecutionContext *)context {
631631
return;
632632

633633
case BPExitStatusAppCrashed:
634-
// Crashed test is considered fatal and shall not be retried
635-
self.finalExitStatus |= context.exitStatus;
634+
if (!self.config.retryAppCrashTests) {
635+
// Crashed test is considered fatal when retry is disabled
636+
self.finalExitStatus |= context.exitStatus;
637+
}
636638
NEXT([self proceed]);
637639
return;
638640

‎bp/src/SimulatorMonitor.m

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -195,8 +195,12 @@ - (void)onOutputReceived:(NSString *)output {
195195
NSString *testClass = (__self.currentClassName ?: __self.previousClassName);
196196
NSString *testName = (__self.currentTestName ?: __self.previousTestName);
197197
if (__self.testsState == Running) {
198-
[self updateExecutedTestCaseList:testName inClass:testClass];
199-
[BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Not retrying it.", testClass, testName];
198+
if (self.config.retryAppCrashTests) {
199+
[BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Configured to retry.", testClass, testName];
200+
} else {
201+
[self updateExecutedTestCaseList:testName inClass:testClass];
202+
[BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Retry disabled.", testClass, testName];
203+
}
200204
[[BPStats sharedStats] endTimer:[NSString stringWithFormat:TEST_CASE_FORMAT, [BPStats sharedStats].attemptNumber, testClass, testName] withResult:@"CRASHED"];
201205
} else {
202206
assert(__self.testsState == Idle);

‎bp/tests/BluepillTests.m

Lines changed: 80 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ @implementation BluepillTests
3535

3636
- (void)setUp {
3737
[super setUp];
38-
38+
3939
self.continueAfterFailure = NO;
4040
NSString *hostApplicationPath = [BPTestHelper sampleAppPath];
4141
NSString *testBundlePath = [BPTestHelper sampleAppNegativeTestsBundlePath];
@@ -222,11 +222,11 @@ - (void)testReportWithAppCrashingAndRetryOnlyFailedTestsSet {
222222
self.config.outputDirectory = outputDir;
223223
self.config.errorRetriesCount = @1;
224224
self.config.failureTolerance = @1;
225-
self.config.onlyRetryFailed = YES;
226-
225+
self.config.onlyRetryFailed = TRUE;
226+
227227
BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
228228
XCTAssertTrue(exitCode == BPExitStatusAppCrashed);
229-
229+
230230
NSString *junitReportPath = [outputDir stringByAppendingPathComponent:@"TEST-BPSampleAppCrashingTests-1-results.xml"];
231231
NSLog(@"JUnit file: %@", junitReportPath);
232232
NSString *expectedFilePath = [[[NSBundle bundleForClass:[self class]] resourcePath] stringByAppendingPathComponent:@"crash_tests_with_retry_attempt_1.xml"];
@@ -248,7 +248,7 @@ - (void)DISABLE_testAppCrashingAndRetryReportsCorrectExitCode {
248248
self.config.testing_crashOnAttempt = @1;
249249
self.config.errorRetriesCount = @2;
250250
self.config.failureTolerance = @1;
251-
self.config.onlyRetryFailed = YES;
251+
self.config.onlyRetryFailed = TRUE;
252252

253253
BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
254254
XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed);
@@ -326,11 +326,11 @@ - (void)testReportWithAppHangingTestsShouldReturnFailure {
326326
}
327327

328328
/**
329-
Execution plan: TIMEOUT, CRASH, PASS
329+
Execution plan: TIMEOUT, CRASH (not retried)
330330
*/
331331
- (void)testReportFailureOnTimeoutCrashAndPass {
332332
self.config.stuckTimeout = @6;
333-
self.config.testing_ExecutionPlan = @"TIMEOUT CRASH PASS";
333+
self.config.testing_ExecutionPlan = @"TIMEOUT CRASH";
334334
self.config.errorRetriesCount = @4;
335335
self.config.onlyRetryFailed = TRUE;
336336
NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
@@ -345,6 +345,48 @@ - (void)testReportFailureOnTimeoutCrashAndPass {
345345
XCTAssertTrue(exitCode == BPExitStatusAppCrashed);
346346
}
347347

348+
/**
349+
Execution plan: TIMEOUT, CRASH, CRASH w/ flag to retry crashes and consider them non-fatal
350+
*/
351+
- (void)testReportFailureOnTimeoutCrashAndCrashOnRetry {
352+
self.config.stuckTimeout = @6;
353+
self.config.retryAppCrashTests = TRUE;
354+
self.config.testing_ExecutionPlan = @"TIMEOUT CRASH CRASH";
355+
self.config.errorRetriesCount = @2;
356+
self.config.onlyRetryFailed = TRUE;
357+
NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
358+
self.config.testBundlePath = testBundlePath;
359+
NSString *tempDir = NSTemporaryDirectory();
360+
NSError *error;
361+
NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error];
362+
NSLog(@"output directory is %@", outputDir);
363+
self.config.outputDirectory = outputDir;
364+
365+
BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
366+
XCTAssertTrue(exitCode == (BPExitStatusTestTimeout | BPExitStatusAppCrashed));
367+
}
368+
369+
/**
370+
Execution plan: TIMEOUT, CRASH, PASS w/ flag to retry crashes and consider them non-fatal
371+
*/
372+
- (void)testReportSuccessOnTimeoutCrashAndPassOnRetry {
373+
self.config.stuckTimeout = @6;
374+
self.config.retryAppCrashTests = TRUE;
375+
self.config.testing_ExecutionPlan = @"TIMEOUT CRASH PASS";
376+
self.config.errorRetriesCount = @4;
377+
self.config.onlyRetryFailed = TRUE;
378+
NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
379+
self.config.testBundlePath = testBundlePath;
380+
NSString *tempDir = NSTemporaryDirectory();
381+
NSError *error;
382+
NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error];
383+
NSLog(@"output directory is %@", outputDir);
384+
self.config.outputDirectory = outputDir;
385+
386+
BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
387+
XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed);
388+
}
389+
348390
/**
349391
Execution plan: CRASH
350392
*/
@@ -387,6 +429,28 @@ - (void)testReportFailureOnCrashAndTimeoutTests {
387429
XCTAssertTrue(exitCode == BPExitStatusAppCrashed);
388430
}
389431

432+
/**
433+
Execution plan: Test crashes but passes on retry w/ retry app crash tests flag set
434+
*/
435+
- (void)testReportSuccessOnAppCrashTestPassesOnRetry {
436+
self.config.stuckTimeout = @6;
437+
self.config.retryAppCrashTests = TRUE;
438+
self.config.testing_ExecutionPlan = @"CRASH PASS; SKIP PASS";
439+
self.config.onlyRetryFailed = TRUE;
440+
self.config.failureTolerance = @1;
441+
self.config.errorRetriesCount = @2;
442+
NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
443+
self.config.testBundlePath = testBundlePath;
444+
NSString *tempDir = NSTemporaryDirectory();
445+
NSError *error;
446+
NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error];
447+
NSLog(@"output directory is %@", outputDir);
448+
self.config.outputDirectory = outputDir;
449+
450+
BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
451+
XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed);
452+
}
453+
390454
/**
391455
Execution plan: One test CRASHes and another one keeps timing out
392456
*/
@@ -457,7 +521,7 @@ - (void)testReportSuccessOnTimeoutAndPassOnRetry {
457521
self.config.stuckTimeout = @6;
458522
self.config.testing_ExecutionPlan = @"TIMEOUT PASS";
459523
self.config.errorRetriesCount = @4;
460-
self.config.onlyRetryFailed = YES;
524+
self.config.onlyRetryFailed = TRUE;
461525
self.config.failureTolerance = @0; // Not relevant
462526
NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
463527
self.config.testBundlePath = testBundlePath;
@@ -478,7 +542,7 @@ - (void)testReportFailureOnTimeoutAndNoRetry {
478542
self.config.stuckTimeout = @6;
479543
self.config.testing_ExecutionPlan = @"TIMEOUT";
480544
self.config.errorRetriesCount = @2;
481-
self.config.onlyRetryFailed = NO;
545+
self.config.onlyRetryFailed = FALSE;
482546
self.config.failureTolerance = @1; // Not relevant since it's not a test failure
483547
NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
484548
self.config.testBundlePath = testBundlePath;
@@ -500,7 +564,7 @@ - (void)testReportSuccessOnFailedTestAndPassOnRetryAll {
500564
self.config.testing_ExecutionPlan = @"FAIL PASS";
501565
self.config.errorRetriesCount = @4;
502566
self.config.onlyRetryFailed = NO; // Indicates to retry all tests when a test fails
503-
self.config.failureTolerance = @1; // Even though failureTolerance is non-zero it wouldn't retry because onlyRetryFailed = NO
567+
self.config.failureTolerance = @1;
504568
NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
505569
self.config.testBundlePath = testBundlePath;
506570
NSString *tempDir = NSTemporaryDirectory();
@@ -578,7 +642,7 @@ - (void)testRetryOnlyFailures {
578642
self.config.outputDirectory = outputDir;
579643
self.config.errorRetriesCount = @100;
580644
self.config.failureTolerance = @1;
581-
self.config.onlyRetryFailed = YES;
645+
self.config.onlyRetryFailed = TRUE;
582646
BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
583647
XCTAssert(exitCode == BPExitStatusTestsFailed);
584648
// Make sure all tests started on the first run
@@ -626,7 +690,7 @@ - (void)testKeepSimulatorWithAppCrashingTestsSet {
626690
NSString *testBundlePath = [BPTestHelper sampleAppCrashingTestsBundlePath];
627691
self.config.testBundlePath = testBundlePath;
628692
self.config.keepSimulator = YES;
629-
693+
630694
Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config];
631695
BPExitStatus exitCode = [bp run];
632696
XCTAssert(exitCode == BPExitStatusAppCrashed);
@@ -639,7 +703,7 @@ - (void)testKeepSimulatorWithAppHangingTestsSet {
639703
self.config.testBundlePath = testBundlePath;
640704
self.config.keepSimulator = YES;
641705
self.config.testing_ExecutionPlan = @"TIMEOUT";
642-
706+
643707
Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config];
644708
BPExitStatus exitCode = [bp run];
645709
XCTAssert(exitCode == BPExitStatusTestTimeout);
@@ -649,15 +713,15 @@ - (void)testDeleteSimulatorOnly {
649713
NSString *testBundlePath = [BPTestHelper sampleAppBalancingTestsBundlePath];
650714
self.config.testBundlePath = testBundlePath;
651715
self.config.keepSimulator = YES;
652-
716+
653717
Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config];
654718
BPExitStatus exitCode = [bp run];
655719
XCTAssert(exitCode == BPExitStatusAllTestsPassed);
656720
XCTAssertNotNil(bp.test_simulatorUDID);
657-
721+
658722
self.config.deleteSimUDID = bp.test_simulatorUDID;
659723
XCTAssertNotNil(self.config.deleteSimUDID);
660-
724+
661725
Bluepill *bp2 = [[Bluepill alloc ] initWithConfiguration:self.config];
662726
BPExitStatus exitCode2 = [bp2 run];
663727
XCTAssert(exitCode2 == BPExitStatusSimulatorDeleted);

0 commit comments

Comments
 (0)