Skip to content

Commit 26a3761

Browse files
RainNapperRainNapper
authored
Record video using simctl (#441)
Use the XCTest listener protocol to add hooks for `xcrun simctl io [sim-id] recordVideo`.
1 parent 2e80798 commit 26a3761

File tree

10 files changed

+240
-27
lines changed

10 files changed

+240
-27
lines changed

‎README.md

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,10 @@ $ bluepill -c config.json
5454
A full list supported options are listed here.
5555

5656

57-
| Config Arguments | Command Line Arguments | Explanation | Required | Default value |
58-
|:----------------------:|:----------------------:|-------------------------------------------------------------------------------------|:--------:|:----------------:|
59-
| app | -a | The path to the host application to execute (your `.app`) | N | n/a |
60-
| xctestrun-path | | The path to the `.xctestrun` file that xcode leaves when you `build-for-testing`. | Y | n/a |
57+
| Config Arguments | Command Line Arguments | Explanation | Required | Default value |
58+
|:----------------------:|:----------------------:|----------------------------------------------------------------------------------------------|:--------:|:----------------:|
59+
| app | -a | The path to the host application to execute (your `.app`) | N | n/a |
60+
| xctestrun-path | | The path to the `.xctestrun` file that xcode leaves when you `build-for-testing`. | Y | n/a |
6161
| test-plan-path | | The path of a json file which describes the test plan. It is equivalent to the `.xctestrun` file generated by Xcode, but it can be generated by a different build system, e.g. Bazel | Y | n/a |
6262
| output-dir | -o | Directory where to put output log files. **(bluepill only)** | Y | n/a |
6363
| config | -c | Read options from the specified configuration file instead of the command line. | N | n/a |
@@ -86,6 +86,7 @@ A full list supported options are listed here.
8686
| help | -h | Help. | N | n/a |
8787
| runner-app-path | -u | The test runner for UI tests. | N | n/a |
8888
| screenshots-directory | n/a | Directory where simulator screenshots for failed ui tests will be stored. | N | n/a |
89+
| videos-directory | n/a | Directory where videos of test runs will be saved. If not provided, videos are not recorded. | N | n/a |
8990
| video-paths | -V | A list of videos that will be saved in the simulators. | N | n/a |
9091
| image-paths | -I | A list of images that will be saved in the simulators. | N | n/a |
9192
| unsafe-skip-xcode-version-check | | Skip Xcode version check | N | NO |

‎bluepill/tests/BPIntegrationTests.m

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ @interface BPIntegrationTests : XCTestCase
2121

2222
@implementation BPIntegrationTests
2323

24-
- (BPConfiguration *)generateConfig {
24+
- (BPConfiguration *)generateConfigWithVideoDir:(NSString *)videoDir {
2525
NSString *hostApplicationPath = [BPTestHelper sampleAppPath];
2626
NSString *testBundlePath = [BPTestHelper sampleAppBalancingTestsBundlePath];
2727
BPConfiguration *config = [[BPConfiguration alloc] initWithProgram:BP_MASTER];
@@ -36,9 +36,16 @@ - (BPConfiguration *)generateConfig {
3636
config.deviceType = @BP_DEFAULT_DEVICE_TYPE;
3737
config.headlessMode = YES;
3838
config.quiet = [BPUtils isBuildScript];
39+
if (videoDir != nil) {
40+
config.videosDirectory = videoDir;
41+
}
3942
return config;
4043
}
4144

45+
- (BPConfiguration *)generateConfig {
46+
return [self generateConfigWithVideoDir: nil];
47+
}
48+
4249
- (void)setUp {
4350
[super setUp];
4451
// Put setup code here. This method is called before the invocation of each test method in the class.
@@ -216,4 +223,59 @@ - (void)writeTestPlan {
216223
[jsonData writeToFile:[BPTestHelper testPlanPath] atomically:YES];
217224
}
218225

226+
- (void)testTwoBPInstancesWithVideo {
227+
NSFileManager *fileManager = [NSFileManager defaultManager];
228+
NSError *mkdtempError;
229+
NSString *path = [BPUtils mkdtemp:@"bpout" withError:&mkdtempError];
230+
XCTAssertNil(mkdtempError);
231+
232+
NSString* videoDirName = @"my_videos";
233+
NSString *videoPath = [path stringByAppendingPathComponent:videoDirName];
234+
BPConfiguration *config = [self generateConfigWithVideoDir:videoPath];
235+
config.numSims = @2;
236+
config.errorRetriesCount = @1;
237+
config.failureTolerance = @0;
238+
// This looks backwards but we want the main app to be the runner
239+
// and the sampleApp is launched from the callback.
240+
config.testBundlePath = [BPTestHelper sampleAppUITestBundlePath];
241+
config.testRunnerAppPath = [BPTestHelper sampleAppPath];
242+
config.appBundlePath = [BPTestHelper sampleAppUITestRunnerPath];
243+
244+
NSError *err;
245+
BPApp *app = [BPApp appWithConfig:config
246+
withError:&err];
247+
NSString *bpPath = [BPTestHelper bpExecutablePath];
248+
249+
// Run the tests through one time to flush out any weird errors that happen with video recording
250+
BPRunner *dryRunRunner = [BPRunner BPRunnerWithConfig:config withBpPath:bpPath];
251+
XCTAssert(dryRunRunner != nil);
252+
int dryRunRC = [dryRunRunner runWithBPXCTestFiles:app.testBundles];
253+
XCTAssert(dryRunRC == 0);
254+
XCTAssert([dryRunRunner.nsTaskList count] == 0);
255+
[fileManager removeItemAtPath:videoPath error:nil];
256+
NSArray *dryRunOutputContents = [fileManager contentsOfDirectoryAtPath:videoPath error:nil];
257+
XCTAssertEqual(dryRunOutputContents.count, 0);
258+
259+
// Start the real test now
260+
BPRunner *runner = [BPRunner BPRunnerWithConfig:config withBpPath:bpPath];
261+
XCTAssert(runner != nil);
262+
int rc = [runner runWithBPXCTestFiles:app.testBundles];
263+
XCTAssert(rc == 0);
264+
XCTAssert([runner.nsTaskList count] == 0);
265+
266+
NSError *dirContentsError;
267+
NSArray *directoryContent = [fileManager contentsOfDirectoryAtPath:videoPath error:&dirContentsError];
268+
XCTAssertNil(dirContentsError);
269+
XCTAssertNotNil(directoryContent);
270+
XCTAssertEqual(directoryContent.count, 2);
271+
272+
NSString *testClass = @"BPSampleAppUITests";
273+
NSSet *filenameSet = [NSSet setWithArray: directoryContent];
274+
XCTAssertEqual(filenameSet.count, 2);
275+
BOOL hasTest1 = [filenameSet containsObject: [NSString stringWithFormat:@"%@__%@__1.mp4", testClass, @"testExample"]];
276+
XCTAssertTrue(hasTest1);
277+
BOOL hasTest2 = [filenameSet containsObject: [NSString stringWithFormat:@"%@__%@__1.mp4", testClass, @"testExample2"]];
278+
XCTAssertTrue(hasTest2);
279+
}
280+
219281
@end

‎bp/src/BPConfiguration.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ typedef NS_ENUM(NSInteger, BPProgram) {
8787
@property (nonatomic, strong) NSString *outputDirectory;
8888
@property (nonatomic, strong) NSString *testTimeEstimatesJsonFile;
8989
@property (nonatomic, strong) NSString *screenshotsDirectory;
90+
@property (nonatomic, strong) NSString *videosDirectory;
9091
@property (nonatomic, strong) NSString *simulatorPreferencesFile;
9192
@property (nonatomic, strong) NSString *scriptFilePath;
9293
@property (nonatomic) BOOL headlessMode;

‎bp/src/BPConfiguration.m

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,8 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) {
146146
"Skip Xcode version check if using an Xcode version that is not officially supported the Bluepill version being used. Not safe/recommended and has a limited support."},
147147
{366, "retry-app-crash-tests", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "retryAppCrashTests",
148148
"Retry the tests after an app crash and if it passes on retry, consider them non-fatal."},
149-
149+
{367, "videos-directory", BP_MASTER | BP_SLAVE, NO, NO, required_argument, NULL, BP_VALUE | BP_PATH, "videosDirectory",
150+
"Directory where videos of test runs will be saved. If not provided, videos are not recorded."},
150151
{0, 0, 0, 0, 0, 0, 0}
151152
};
152153

@@ -776,6 +777,23 @@ - (BOOL)validateConfigWithError:(NSError *__autoreleasing *)errPtr {
776777
}
777778
}
778779

780+
if (self.videosDirectory) {
781+
if ([[NSFileManager defaultManager] fileExistsAtPath:self.videosDirectory isDirectory:&isdir]) {
782+
if (!isdir) {
783+
BP_SET_ERROR(errPtr, @"%@ is not a directory.", self.videosDirectory);
784+
return NO;
785+
}
786+
} else {
787+
// create the directory
788+
if (![[NSFileManager defaultManager] createDirectoryAtPath:self.videosDirectory
789+
withIntermediateDirectories:YES
790+
attributes:nil
791+
error:errPtr]) {
792+
return NO;
793+
}
794+
}
795+
}
796+
779797
if (self.simulatorPreferencesFile) {
780798
if ([[NSFileManager defaultManager] fileExistsAtPath:self.simulatorPreferencesFile isDirectory:&isdir]) {
781799
if (isdir) {

‎bp/src/BPTestBundleConnection.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
//
88

99
#import <Foundation/Foundation.h>
10+
#import "BPExecutionContext.h"
1011
#import "BPSimulator.h"
1112

1213
// This is a small subset of XCTestManager_IDEInterface protocol
@@ -16,9 +17,10 @@
1617

1718
@interface BPTestBundleConnection : NSObject
1819
@property (nonatomic, strong) BPConfiguration *config;
20+
@property (nonatomic, strong) BPExecutionContext *context;
1921
@property (nonatomic, strong) BPSimulator *simulator;
2022
@property (nonatomic, copy) void (^completionBlock)(NSError *, pid_t);
21-
- (instancetype)initWithDevice:(BPSimulator *)device andInterface:(id<BPTestBundleConnectionDelegate>)interface;
23+
- (instancetype)initWithContext:(BPExecutionContext *)context andInterface:(id<BPTestBundleConnectionDelegate>)interface;
2224
- (void)connectWithTimeout:(NSTimeInterval)timeout;
2325
- (void)startTestPlan;
2426
@end

‎bp/src/BPTestBundleConnection.m

Lines changed: 75 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,15 +49,19 @@ @interface BPTestBundleConnection()<XCTestManager_IDEInterface>
4949
@property (nonatomic, strong) dispatch_queue_t queue;
5050
@property (nonatomic, strong) NSString *bundleID;
5151
@property (nonatomic, assign) pid_t appProcessPID;
52+
@property (nonatomic, nullable) NSTask *recordVideoTask;
53+
//@property (nonatomic, nullable) NSPipe *recordVideoPipe;
54+
5255

5356
@end
5457

5558
@implementation BPTestBundleConnection
5659

57-
- (instancetype)initWithDevice:(BPSimulator *)simulator andInterface:(id<BPTestBundleConnectionDelegate>)interface {
60+
- (instancetype)initWithContext:(BPExecutionContext *)context andInterface:(id<BPTestBundleConnectionDelegate>)interface {
5861
self = [super init];
5962
if (self) {
60-
self.simulator = simulator;
63+
self.context = context;
64+
self.simulator = context.runner;
6165
self.interface = interface;
6266
self.queue = dispatch_queue_create("com.linkedin.bluepill.connection.queue", DISPATCH_QUEUE_PRIORITY_DEFAULT);
6367
}
@@ -82,6 +86,8 @@ - (void)connect {
8286
DTXTransport *transport = [self connectTransport];
8387
DTXConnection *connection = [[objc_lookUpClass("DTXConnection") alloc] initWithTransport:transport];
8488
[connection registerDisconnectHandler:^{
89+
// This is called when the task is abruptly terminated (e.g. if the test times out)
90+
[self stopVideoRecording:YES];
8591
[BPUtils printInfo:INFO withString:@"DTXConnection disconnected."];
8692
}];
8793
[connection
@@ -212,6 +218,63 @@ - (id)_XCT_initializationForUITestingDidFailWithError:(NSError *__strong)errPtr
212218
return nil;
213219
}
214220

221+
#pragma mark - Video Recording
222+
223+
static inline NSString* getVideoPath(NSString *directory, NSString *testClass, NSString *method, NSInteger attemptNumber)
224+
{
225+
return [NSString stringWithFormat:@"%@/%@__%@__%ld.mp4", directory, testClass, method, (long)attemptNumber];
226+
}
227+
228+
- (BOOL)shouldRecordVideo {
229+
return self.context.config.videosDirectory.length > 0;
230+
}
231+
232+
- (void)startVideoRecordingForTestClass:(NSString *)testClass method:(NSString *)method
233+
{
234+
[self stopVideoRecording:YES];
235+
NSString *videoFileName = getVideoPath(self.context.config.videosDirectory, testClass, method, self.context.attemptNumber);
236+
NSString *command = [NSString stringWithFormat:@"xcrun simctl io %@ recordVideo --force %@", [self.simulator UDID], videoFileName];
237+
NSTask *task = [BPUtils buildShellTaskForCommand:command];
238+
self.recordVideoTask = task;
239+
[task launch];
240+
[BPUtils printInfo:INFO withString:@"Started recording video to %@", videoFileName];
241+
[BPUtils printInfo:DEBUGINFO withString:@"Started recording video task with pid %d and command: %@", [task processIdentifier], [BPUtils getCommandStringForTask:task]];
242+
}
243+
244+
- (void)stopVideoRecording:(BOOL)forced
245+
{
246+
NSTask *task = self.recordVideoTask;
247+
if (task == nil) {
248+
if (!forced) {
249+
[BPUtils printInfo:ERROR withString: @"Tried to end video recording task normally, but there was no task."];
250+
}
251+
return;
252+
}
253+
254+
if (forced) {
255+
[BPUtils printInfo:ERROR withString: @"Found dangling video recording task. Stopping it."];
256+
}
257+
258+
if (![task isRunning]) {
259+
[BPUtils printInfo:ERROR withString:@"Video task exists but it was already terminated with status %d", [task terminationStatus]];
260+
}
261+
262+
[BPUtils printInfo:INFO withString:@"Stopping recording video."];
263+
[BPUtils printInfo:DEBUGINFO withString:@"Stopping video recording task with pid %d and command: %@", [task processIdentifier], [BPUtils getCommandStringForTask:task]];
264+
[task interrupt];
265+
[task waitUntilExit];
266+
267+
if ([task terminationStatus] != 0) {
268+
[BPUtils printInfo:ERROR withString:@"Video task was interrupted, but exited with non-zero status %d", [task terminationStatus]];
269+
}
270+
271+
NSString *filePath = [[task arguments].lastObject componentsSeparatedByString:@" "].lastObject;
272+
if (![[NSFileManager defaultManager] fileExistsAtPath:filePath]) {
273+
[BPUtils printInfo:ERROR withString:@"Video recording file missing, expected at path %@!", filePath];
274+
}
275+
self.recordVideoTask = nil;
276+
}
277+
215278
#pragma mark - XCTestManager_IDEInterface protocol
216279

217280
#pragma mark Process Launch Delegation
@@ -255,7 +318,7 @@ - (id)_XCT_getProgressForLaunch:(id)token {
255318

256319
- (id)_XCT_terminateProcess:(id)token {
257320
NSError *error;
258-
kill(self.appProcessPID, SIGTERM);
321+
kill(self.appProcessPID, SIGINT);
259322
DTXRemoteInvocationReceipt *receipt = [objc_lookUpClass("DTXRemoteInvocationReceipt") new];
260323
[receipt invokeCompletionWithReturnValue:token error:error];
261324
[BPUtils printInfo:DEBUGINFO withString:@"BPTestBundleConnection_XCT_terminateProcess with token %@", token];
@@ -277,6 +340,9 @@ - (id)_XCT_testSuite:(NSString *)tests didStartAt:(NSString *)time {
277340

278341
- (id)_XCT_testCaseDidStartForTestClass:(NSString *)testClass method:(NSString *)method {
279342
[BPUtils printInfo:DEBUGINFO withString:@"BPTestBundleConnection_XCT_testCaseDidStartForTestClass: %@ and method: %@", testClass, method];
343+
if ([self shouldRecordVideo]) {
344+
[self startVideoRecordingForTestClass:testClass method:method];
345+
}
280346
return nil;
281347
}
282348

@@ -298,12 +364,18 @@ - (id)_XCT_logMessage:(NSString *)message {
298364

299365
- (id)_XCT_testCaseDidFinishForTestClass:(NSString *)testClass method:(NSString *)method withStatus:(NSString *)statusString duration:(NSNumber *)duration {
300366
[BPUtils printInfo:DEBUGINFO withString: @"BPTestBundleConnection_XCT_testCaseDidFinishForTestClass: %@, method: %@, withStatus: %@, duration: %@", testClass, method, statusString, duration];
367+
if ([self shouldRecordVideo]) {
368+
[self stopVideoRecording:NO];
369+
}
301370
return nil;
302371
}
303372

304373
- (id)_XCT_testSuite:(NSString *)arg1 didFinishAt:(NSString *)time runCount:(NSNumber *)count withFailures:(NSNumber *)failureCount unexpected:(NSNumber *)unexpectedCount testDuration:(NSNumber *)testDuration totalDuration:(NSNumber *)totalTime {
305374
[BPUtils printInfo:DEBUGINFO withString: @"BPTestBundleConnection_XCT_testSuite: %@, didFinishAt: %@, runCount: %@, withFailures: %@, unexpectedCount: %@, testDuration: %@, totalDuration: %@", arg1, time, count, failureCount, unexpectedCount, testDuration, totalTime];
306375

376+
if ([self shouldRecordVideo]) {
377+
[self stopVideoRecording:YES];
378+
}
307379
return nil;
308380
}
309381

‎bp/src/BPUtils.h

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,29 @@ typedef NS_ENUM(int, BPKind) {
125125
* @return return the shell output
126126
*/
127127
+ (NSString *)runShell:(NSString *)command;
128+
129+
/*!
130+
* @discussion builds a task to run a shell command
131+
* @param command the shell command the task should run
132+
* @return an NSTask that will run the provided command.
133+
*/
134+
+ (NSTask *)buildShellTaskForCommand:(NSString *)command;
135+
136+
/*!
137+
* @discussion builds a task to run a shell command, pointing stdout and stderr to the provided pipe
138+
* @param command the shell command the task should run
139+
* @param pipe the pipe that stdout and stderr will be pointed to, so the caller can handle the output.
140+
* @return an NSTask that will run the provided command.
141+
*/
142+
+ (NSTask *)buildShellTaskForCommand:(NSString *)command withPipe:(NSPipe *)pipe;
143+
144+
/*!
145+
* @discussion builds a user readable representation of the command that a task is configured to run
146+
* @param task to get command from
147+
* @return a user readable string of the task's command
148+
*/
149+
+ (NSString *)getCommandStringForTask:(NSTask *)task;
150+
128151
+ (NSString *)getXcodeRuntimeVersion;
129152

130153
typedef BOOL (^BPRunBlock)(void);

‎bp/src/BPUtils.m

Lines changed: 27 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -189,29 +189,40 @@ + (BOOL)isStdOut:(NSString *)fileName {
189189

190190
+ (NSString *)runShell:(NSString *)command {
191191
NSAssert(command, @"Command should not be nil");
192-
NSTask *task = [[NSTask alloc] init];
193-
NSData *data;
194-
task.launchPath = @"/bin/sh";
195-
task.arguments = @[@"-c", command];
196192
NSPipe *pipe = [[NSPipe alloc] init];
197-
task.standardError = pipe;
198-
task.standardOutput = pipe;
193+
NSTask *task = [BPUtils buildShellTaskForCommand:command withPipe:pipe];
194+
NSAssert(task, @"task should not be nil");
199195
NSFileHandle *fh = pipe.fileHandleForReading;
200-
if (task) {
201-
[task launch];
202-
} else {
203-
NSAssert(task, @"task should not be nil");
204-
}
205-
if (fh) {
206-
data = [fh readDataToEndOfFile];
207-
} else {
208-
NSAssert(task, @"fh should not be nil");
209-
}
196+
NSAssert(fh, @"fh should not be nil");
197+
198+
[task launch];
199+
NSData *data = [fh readDataToEndOfFile];
210200
[task waitUntilExit];
211201
NSString *result = [[[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] stringByTrimmingCharactersInSet:[NSCharacterSet newlineCharacterSet]];
212202
return result;
213203
}
214204

205+
+ (NSTask *)buildShellTaskForCommand:(NSString *)command {
206+
return [BPUtils buildShellTaskForCommand:command withPipe: nil];
207+
}
208+
209+
+ (NSTask *)buildShellTaskForCommand:(NSString *)command withPipe:(NSPipe *)pipe {
210+
NSAssert(command, @"Command should not be nil");
211+
NSTask *task = [[NSTask alloc] init];
212+
task.launchPath = @"/bin/sh";
213+
task.arguments = @[@"-c", command];
214+
if (pipe != nil) {
215+
task.standardError = pipe;
216+
task.standardOutput = pipe;
217+
}
218+
NSAssert(task, @"task should not be nil");
219+
return task;
220+
}
221+
222+
+ (NSString *)getCommandStringForTask:(NSTask *)task {
223+
return [NSString stringWithFormat:@"%@ %@", [task launchPath], [[task arguments] componentsJoinedByString:@" "]];
224+
}
225+
215226
+ (BOOL)runWithTimeOut:(NSTimeInterval)timeout until:(BPRunBlock)block {
216227
if (!block) {
217228
return NO;

‎bp/src/Bluepill.m

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -432,7 +432,7 @@ - (void)connectTestBundleAndTestDaemonWithContext:(BPExecutionContext *)context
432432
// If the isTestRunnerContext is flipped on, don't connect testbundle again.
433433
return;
434434
}
435-
BPTestBundleConnection *bConnection = [[BPTestBundleConnection alloc] initWithDevice:context.runner andInterface:self];
435+
BPTestBundleConnection *bConnection = [[BPTestBundleConnection alloc] initWithContext:context andInterface:self];
436436
bConnection.simulator = context.runner;
437437
bConnection.config = self.config;
438438

0 commit comments

Comments
 (0)