Fix hangs (#49)

* Add a few test cases for when the app hangs/crashes on launch. (@ob)
* Fix for hanging and crashing before test run. (@vargon)
* Add a variable for tests that do not launch an application (@vargon)
* Clean up the message and use the new guard for detecting parsing tests (@ob)
* Add dump of current commits vs. master to start of script (@vargon)
* Lower timeout test timeout from 5 seconds to 2. (@vargon)

Author: vargon

BPSampleApp/BPSampleApp/AppDelegate.m

@@ -20,6 +20,17 @@ - (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(
     // Override point for customization after application launch.
 
     // This is a sample app for Bluepill testing.
+    if (getenv("_BP_TEST_CRASH_ON_LAUNCH")) {
+        char *p = NULL;
+        NSLog(@"CRASHING AT USER'S REQUEST");
+        strcpy(p, "I know this will crash my app");
+    }
+    if (getenv("_BP_TEST_HANG_ON_LAUNCH")) {
+        NSLog(@"HANGING AT USER'S REQUEST");
+        while(1) {
+            sleep(10);
+        }
+    }
     return YES;
 }
 

Bluepill-cli/BPInstanceTests/BPTreeParserTests.m

@@ -19,6 +19,8 @@
 
 @interface BPTreeParserTests : XCTestCase
 
+@property (nonatomic, strong) BPConfiguration* config;
+
 @end
 
 @implementation BPTreeParserTests
@@ -29,6 +31,8 @@ - (void)setUp {
         [BPUtils quietMode:YES];
         [BPUtils enableDebugOutput:NO];
     }
+    self.config = [[BPConfiguration alloc] init];
+    self.config.testing_NoAppWillRun = YES;
 }
 
 - (void)tearDown {
@@ -53,8 +57,7 @@ - (void)testParsingCrash {
 
     BPWriter *writer = getWriter();
     BPTreeParser *parser = [[BPTreeParser alloc] initWithWriter:writer];
-    BPConfiguration *config = [[BPConfiguration alloc] init];
-    SimulatorMonitor *monitor = [[SimulatorMonitor alloc] initWithConfiguration:config];
+    SimulatorMonitor *monitor = [[SimulatorMonitor alloc] initWithConfiguration:self.config];
 
     parser.delegate = monitor;
 
@@ -74,8 +77,7 @@ - (void)testCrashIntermixedWithPass {
 
     BPWriter *writer = getWriter();
     BPTreeParser *parser = [[BPTreeParser alloc] initWithWriter:writer];
-    BPConfiguration *config = [[BPConfiguration alloc] init];
-    SimulatorMonitor *monitor = [[SimulatorMonitor alloc] initWithConfiguration:config];
+    SimulatorMonitor *monitor = [[SimulatorMonitor alloc] initWithConfiguration:self.config];
 
     parser.delegate = monitor;
 
@@ -95,8 +97,7 @@ - (void)testBadFilenameParsing {
 
     BPWriter *writer = getWriter();
     BPTreeParser *parser = [[BPTreeParser alloc] initWithWriter:writer];
-    BPConfiguration *config = [[BPConfiguration alloc] init];
-    SimulatorMonitor *monitor = [[SimulatorMonitor alloc] initWithConfiguration:config];
+    SimulatorMonitor *monitor = [[SimulatorMonitor alloc] initWithConfiguration:self.config];
 
     parser.delegate = monitor;
 
@@ -116,8 +117,7 @@ - (void)testMissedCrash {
 
     BPWriter *writer = getWriter();
     BPTreeParser *parser = [[BPTreeParser alloc] initWithWriter:writer];
-    BPConfiguration *config = [[BPConfiguration alloc] init];
-    SimulatorMonitor *monitor = [[SimulatorMonitor alloc] initWithConfiguration:config];
+    SimulatorMonitor *monitor = [[SimulatorMonitor alloc] initWithConfiguration:self.config];
     monitor.maxTimeWithNoOutput = 2.0; // change the max output time to 2 seconds
 
     parser.delegate = monitor;
@@ -145,8 +145,7 @@ - (void)testErrorOnlyCrash {
 
     BPWriter *writer = getWriter();
     BPTreeParser *parser = [[BPTreeParser alloc] initWithWriter:writer];
-    BPConfiguration *config = [[BPConfiguration alloc] init];
-    SimulatorMonitor *monitor = [[SimulatorMonitor alloc] initWithConfiguration:config];
+    SimulatorMonitor *monitor = [[SimulatorMonitor alloc] initWithConfiguration:self.config];
 
     parser.delegate = monitor;
 

Bluepill-cli/BPInstanceTests/BluepillTests.m

@@ -47,6 +47,7 @@ - (void)setUp {
     self.config.jsonOutput = NO;
     self.config.headlessMode = NO;
     self.config.junitOutput = NO;
+    self.config.testing_NoAppWillRun = YES;
     NSString *path = @"testScheme.xcscheme";
     self.config.schemePath = [[[NSBundle bundleForClass:[self class]] resourcePath] stringByAppendingPathComponent:path];
     [BPUtils quietMode:YES];
@@ -78,17 +79,30 @@ - (void)tearDown {
     [super tearDown];
 }
 
-// This is a template to run Voyager's tests
-- (void)testVoyager {
-//    self.config.outputDirectory = @"/Users/khu/tmp/simulator";
-//    self.config.schemePath = @"/Users/khu/ios/mntf-ios-sample-app_trunk/./mntf-ios-sample-app.xcodeproj/xcshareddata/xcschemes/mntf-ios-sample-app-ui-tests.xcscheme";
-//    self.config.testCasesToRun = @[@"MNTFSampleAppUITests/testRotate", @"MNTFSampleAppUITests/testScrollToAnIndex"];
-//    self.config.appBundlePath =
-//    @"/Users/khu/ios/mntf-ios-sample-app_trunk/build/mntf-ios-sample-app/Build/Products/Debug-iphonesimulator/mntf-ios-sample-app.app";
-//    self.config.testBundlePath = @"/Users/khu/ios/mntf-ios-sample-app_trunk/build/mntf-ios-sample-app/Build/Products/Debug-iphonesimulator/mntf-ios-sample-app.app/PlugIns/mntf-ios-sample-app-UITests.xctest";
-//    BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
+- (void)testAppThatCrashesOnLaunch {
+    NSString *testBundlePath = [BPTestHelper sampleAppBalancingTestsBunldePath];
+    self.config.testBundlePath = testBundlePath;
+    self.config.testing_CrashAppOnLaunch = YES;
+    self.config.testing_NoAppWillRun = NO;
+    BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
+    XCTAssert(exitCode == BPExitStatusAppCrashed);
+
+    self.config.testing_NoAppWillRun = YES;
+}
+
+- (void)testAppThatHangsOnLaunch {
+    NSString *testBundlePath = [BPTestHelper sampleAppBalancingTestsBunldePath];
+    self.config.testBundlePath = testBundlePath;
+    self.config.testing_HangAppOnLaunch = YES;
+    self.config.testing_NoAppWillRun = NO;
+    self.config.stuckTimeout = @3;
+    BPExitStatus exitCode = [[[Bluepill alloc] initWithConfiguration:self.config] run];
+    XCTAssert(exitCode == BPExitStatusTestTimeout);
+
+    self.config.testing_NoAppWillRun = YES;
 }
 
+
 - (void)testRunningOnlyCertainTestcases {
     NSString *testBundlePath = [BPTestHelper sampleAppBalancingTestsBunldePath];
     self.config.testBundlePath = testBundlePath;
@@ -193,7 +207,7 @@ - (void)testReportWithFatalErrorTestsSet {
 }
 
 - (void)testReportWithAppHangingTestsSet {
-    self.config.stuckTimeout = @5;
+    self.config.stuckTimeout = @2;
     self.config.plainOutput = YES;
     NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
     self.config.testBundlePath = testBundlePath;

Bluepill-cli/Bluepill-cli/Bluepill/BPStats.h

@@ -36,9 +36,10 @@
 - (void)addTestFailure;
 - (void)addTestError;
 - (void)addSimulatorCrash;
+- (void)addApplicationCrash;
 - (void)addRetry;
 - (void)addTestRuntimeTimeout;
-- (void)addTestBPExitStatusTestTimeout;
+- (void)addTestOutputTimeout;
 - (void)addSimulatorCreateFailure;
 - (void)addSimulatorDeleteFailure;
 - (void)addSimulatorInstallFailure;

Bluepill-cli/Bluepill-cli/Bluepill/BPStats.m

@@ -24,7 +24,8 @@ @interface BPStats()
 @property (nonatomic, assign) NSInteger testsTotal;
 @property (nonatomic, assign) NSInteger testFailures;
 @property (nonatomic, assign) NSInteger testErrors;
-@property (nonatomic, assign) NSInteger crashes;
+@property (nonatomic, assign) NSInteger simCrashes;
+@property (nonatomic, assign) NSInteger appCrashes;
 @property (nonatomic, assign) NSInteger retries;
 @property (nonatomic, assign) NSInteger runtimeTimeout;
 @property (nonatomic, assign) NSInteger outputTimeout;
@@ -133,7 +134,11 @@ - (void)addTestError {
 }
 
 - (void)addSimulatorCrash {
-    self.crashes++;
+    self.simCrashes++;
+}
+
+- (void)addApplicationCrash {
+    self.appCrashes++;
 }
 
 - (void)addRetry {
@@ -144,7 +149,7 @@ - (void)addTestRuntimeTimeout {
     self.runtimeTimeout++;
 }
 
-- (void)addTestBPExitStatusTestTimeout {
+- (void)addTestOutputTimeout {
     self.outputTimeout++;
 }
 
@@ -188,7 +193,8 @@ - (void)generateFullReportWithWriter:(BPWriter *)writer exitCode:(int)exitCode {
     [writer writeLine:@"Timeout due to test run-time:   %d", self.runtimeTimeout];
     [writer writeLine:@"Timeout due to no output:       %d", self.outputTimeout];
     [writer writeLine:@"Retries:                        %d", self.retries];
-    [writer writeLine:@"Simulator Crashes:              %d", self.crashes];
+    [writer writeLine:@"Application Crashes:            %d", self.appCrashes];
+    [writer writeLine:@"Simulator Crashes:              %d", self.simCrashes];
     [writer writeLine:@"Simulator Creation Failures:    %d", self.simulatorCreateFailures];
     [writer writeLine:@"Simulator Deletion Failures:    %d", self.simulatorDeleteFailures];
     [writer writeLine:@"App Install Failures:           %d", self.simulatorInstallFailures];

Bluepill-cli/Bluepill-cli/Bluepill/Bluepill.m

@@ -308,10 +308,28 @@ - (void)checkProcessWithContext:(BPExecutionContext *)context {
         return;
     }
 
+    // This check should be last after all of the more specific tests
+    // It checks if the app is even running, which it must be at this point
+    // If it's not running and we passed the above checks (e.g., the tests are not yet completed)
+    // then it must mean the app has crashed.
+    // However, we have a short-circuit for tests because those may not actually run any app
+    if (!isRunning && context.pid > 0 && !self.config.testing_NoAppWillRun) {
+        // The tests ended before they even got started or the process is gone for some other reason
+        [[BPStats sharedStats] endTimer:RUN_TESTS(context.attemptNumber)];
+        [BPUtils printInfo:ERROR withString:@"Application crashed before tests started!"];
+        [[BPStats sharedStats] addApplicationCrash];
+        [self deleteSimulatorWithContext:context andStatus:BPExitStatusAppCrashed];
+        return;
+    }
+
     NEXT([self checkProcessWithContext:context]);
 }
 
 - (BOOL)isProcessRunningWithContext:(BPExecutionContext *)context {
+    if (self.config.testing_NoAppWillRun) {
+        return NO;
+    }
+    NSAssert(context.pid > 0, @"Application PID must be > 0");
     int rc = kill(context.pid, 0);
     return !(rc < 0);
 }

Bluepill-cli/Bluepill-cli/Simulator/SimulatorMonitor.m

@@ -15,6 +15,7 @@
 
 typedef NS_ENUM(NSInteger, SimulatorState) {
     Idle,
+    AppLaunched,
     Running,
     Completed
 };
@@ -172,6 +173,10 @@ - (void)onTestSuiteEnded:(NSString *)testSuiteName
 - (void)onOutputReceived:(NSString *)output {
     NSDate *currentTime = [NSDate date];
 
+    if (self.simulatorState == Idle) {
+        self.simulatorState = AppLaunched;
+    }
+
     self.currentOutputId++; // Increment the Output ID for this instance since we've moved on to the next bit of output
 
     __block NSUInteger previousOutputId = self.currentOutputId;
@@ -186,20 +191,24 @@ - (void)onOutputReceived:(NSString *)output {
                               forTestName:(__self.currentTestName ?: __self.previousTestName)
                                   inClass:(__self.currentClassName ?: __self.previousClassName)];
         __self.exitStatus = BPExitStatusAppCrashed;
-        [[BPStats sharedStats] addSimulatorCrash];
+        [[BPStats sharedStats] addApplicationCrash];
     }
     dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(__self.maxTimeWithNoOutput * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{
-        if (__self.currentOutputId == previousOutputId && __self.simulatorState == Running) {
+        if (__self.currentOutputId == previousOutputId && (__self.simulatorState == Running || __self.simulatorState == AppLaunched)) {
             NSString *testClass = (__self.currentClassName ?: __self.previousClassName);
             NSString *testName = (__self.currentTestName ?: __self.previousTestName);
-            [BPUtils printInfo:TIMEOUT withString:@" %10.6fs waiting for output from %@/%@",
-                                                   __self.maxTimeWithNoOutput, testClass, testName];
+            if (testClass == nil && testName == nil && (__self.simulatorState == AppLaunched || __self.simulatorState == Idle)) {
+                [BPUtils printInfo:ERROR withString:@"It appears that tests have not yet started. The test app has frozen prior to the first test."];
+            } else {
+                [BPUtils printInfo:TIMEOUT withString:@" %10.6fs waiting for output from %@/%@",
+                 __self.maxTimeWithNoOutput, testClass, testName];
+                [[BPStats sharedStats] endTimer:[NSString stringWithFormat:TEST_CASE_FORMAT, [BPStats sharedStats].attemptNumber, testClass, testName]];
+            }
             [__self stopTestsWithErrorMessage:@"Timed out waiting for the test to produce output. Test was aboorted."
                                   forTestName:testName
                                       inClass:testClass];
             __self.exitStatus = BPExitStatusTestTimeout;
-            [[BPStats sharedStats] endTimer:[NSString stringWithFormat:TEST_CASE_FORMAT, [BPStats sharedStats].attemptNumber, testClass, testName]];
-            [[BPStats sharedStats] addTestBPExitStatusTestTimeout];
+            [[BPStats sharedStats] addTestOutputTimeout];
         }
     });
     self.lastOutput = currentTime;
@@ -210,15 +219,12 @@ - (void)stopTestsWithErrorMessage:(NSString *)message forTestName:(NSString *)te
     // Timeout or crash on a test means we should skip it when we rerun the tests
     [self updateExecutedTestCaseList:testName inClass:testClass];
 
-    if (![[self.device stateString] isEqualToString:@"Shutdown"]) {
-        // self.appPID can be zero when running the parsing tests
-        // since we're not actually creating a simulator and running an app.
+    if (![[self.device stateString] isEqualToString:@"Shutdown"] && !self.config.testing_NoAppWillRun) {
         [BPUtils printInfo:ERROR withString:@"Will kill the process with appPID: %d", self.appPID];
-        if (self.appPID && (kill(self.appPID, 0) == 0) && (kill(self.appPID, SIGTERM) < 0)) {
-            [BPUtils printInfo:ERROR withString:@"Failed to kill the process with appPID: %d", self.appPID];
-            perror("kill");
-        } else {
-            [BPUtils printInfo:ERROR withString:@"Success killing the process with appPID: %d", self.appPID];
+        NSAssert(self.appPID > 0, @"Failed to find a valid PID");
+        if ((kill(self.appPID, 0) == 0) && (kill(self.appPID, SIGTERM) < 0)) {
+            [BPUtils printInfo:ERROR withString:@"Failed to kill the process with appPID: %d: %s",
+                self.appPID, strerror(errno)];
         }
     }
 

Bluepill-cli/Bluepill-cli/Simulator/SimulatorRunner.m

@@ -242,6 +242,13 @@ - (void)launchApplicationAndExecuteTestsWithParser:(BPTreeParser *)parser andCom
                                                                                                                                  config:self.config]];
     [appLaunchEnvironment addEntriesFromDictionary:argsAndEnv[@"env"]];
 
+    if (self.config.testing_CrashAppOnLaunch) {
+        appLaunchEnvironment[@"_BP_TEST_CRASH_ON_LAUNCH"] = @"YES";
+    }
+    if (self.config.testing_HangAppOnLaunch) {
+        appLaunchEnvironment[@"_BP_TEST_HANG_ON_LAUNCH"] = @"YES";
+    }
+
     // Intercept stdout, stderr and post as simulator-output events
     NSString *stdout_stderr = [NSString stringWithFormat:@"%@/tmp/stdout_stderr_%@", self.device.dataPath, [[self.device UDID] UUIDString]];
     NSString *simStdoutPath = [BPUtils mkstemp:stdout_stderr withError:nil];

Source/Shared/BPConfiguration.h

@@ -20,6 +20,11 @@
 
 @interface BPConfiguration : NSObject <NSCopying>
 
+/*
+ * WARNING: Any fields you add here need to be explicitly handled in the copyWithZone
+ * and mutableCopyWithZone methods. Yeah, it's stupid, we should fix it.
+ */
+
 @property (nonatomic, strong) NSString *appBundlePath;
 @property (nonatomic, strong) NSString *testBundlePath;
 @property (nonatomic, strong) NSArray *additionalTestBundles;
@@ -47,6 +52,11 @@
 @property (nonatomic, strong) NSNumber *listTestsOnly;
 @property (nonatomic) BOOL quiet;
 
+// These fields are for testing.
+@property (nonatomic) BOOL testing_CrashAppOnLaunch;
+@property (nonatomic) BOOL testing_HangAppOnLaunch;
+@property (nonatomic) BOOL testing_NoAppWillRun;
+
 // Generated fields
 @property (nonatomic, strong) NSString *xcodePath;
 

Source/Shared/BPConfiguration.m

@@ -248,6 +248,9 @@ - (id)mutableCopyWithZone:(NSZone *)zone {
     newConfig.simDeviceType = self.simDeviceType;
 #endif
     newConfig.xcodePath = self.xcodePath;
+    newConfig.testing_CrashAppOnLaunch = self.testing_CrashAppOnLaunch;
+    newConfig.testing_HangAppOnLaunch = self.testing_HangAppOnLaunch;
+    newConfig.testing_NoAppWillRun = self.testing_NoAppWillRun;
 
     return newConfig;
 }

scripts/bluepill.sh

@@ -25,6 +25,9 @@ bluepill_build()
 
 bluepill_test()
 {
+  # Dump our current diff state with master
+  git --no-pager log master..HEAD
+
   default_runtime=`grep BP_DEFAULT_RUNTIME ./Source/Shared/BPConstants.h | sed 's/.*BP_DEFAULT_RUNTIME *//;s/"//g;s/ *$//g;'`
   xcrun simctl list runtimes | grep -q "$default_runtime" || {
     echo "Your system doesn't contain latest runtime: iOS $default_runtime"