Update scaffolded tests to enable error reporting

See wp-cli/wp-cli#4515

Author: danielbachhuber

features/bootstrap/FeatureContext.php

@@ -131,6 +131,12 @@ private static function get_process_env_variables() {
 		if ( $php_args = getenv( 'WP_CLI_PHP_ARGS' ) ) {
 			$env['WP_CLI_PHP_ARGS'] = $php_args;
 		}
+		if ( $php_used = getenv( 'WP_CLI_PHP_USED' ) ) {
+			$env['WP_CLI_PHP_USED'] = $php_used;
+		}
+		if ( $php = getenv( 'WP_CLI_PHP' ) ) {
+			$env['WP_CLI_PHP'] = $php;
+		}
 		if ( $travis_build_dir = getenv( 'TRAVIS_BUILD_DIR' ) ) {
 			$env['TRAVIS_BUILD_DIR'] = $travis_build_dir;
 		}
@@ -526,7 +532,8 @@ public function background_proc( $cmd ) {
 		$status = proc_get_status( $proc );
 
 		if ( !$status['running'] ) {
-			throw new RuntimeException( stream_get_contents( $pipes[2] ) );
+			$stderr = is_resource( $pipes[2] ) ? ( ': ' . stream_get_contents( $pipes[2] ) ) : '';
+			throw new RuntimeException( sprintf( "Failed to start background process '%s'%s.", $cmd, $stderr ) );
 		} else {
 			$this->running_procs[] = $proc;
 		}
@@ -617,7 +624,8 @@ public function install_wp( $subdir = '' ) {
 			'title' => 'WP CLI Site',
 			'admin_user' => 'admin',
 			'admin_email' => 'admin@example.com',
-			'admin_password' => 'password1'
+			'admin_password' => 'password1',
+			'skip-email' => true,
 		);
 
 		$install_cache_path = '';
@@ -658,7 +666,8 @@ public function install_wp_with_composer( $vendor_directory = 'vendor' ) {
 			'title' => 'WP CLI Site with both WordPress and wp-cli as Composer dependencies',
 			'admin_user' => 'admin',
 			'admin_email' => 'admin@example.com',
-			'admin_password' => 'password1'
+			'admin_password' => 'password1',
+			'skip-email' => true,
 		);
 
 		$this->proc( 'wp core install', $install_args )->run_check();
@@ -686,26 +695,13 @@ public function composer_require_current_wp_cli() {
 		$this->composer_command( 'require wp-cli/wp-cli:dev-master --optimize-autoloader --no-interaction' );
 	}
 
-	public function get_php_binary() {
-		if ( getenv( 'WP_CLI_PHP_USED' ) )
-			return getenv( 'WP_CLI_PHP_USED' );
-
-		if ( getenv( 'WP_CLI_PHP' ) )
-			return getenv( 'WP_CLI_PHP' );
-
-		if ( defined( 'PHP_BINARY' ) )
-			return PHP_BINARY;
-
-		return 'php';
-	}
-
 	public function start_php_server( $subdir = '' ) {
 		$dir = $this->variables['RUN_DIR'] . '/';
 		if ( $subdir ) {
 			$dir .= trim( $subdir, '/' ) . '/';
 		}
 		$cmd = Utils\esc_cmd( '%s -S %s -t %s -c %s %s',
-			$this->get_php_binary(),
+			Utils\get_php_binary(),
 			'localhost:8080',
 			$dir,
 			get_cfg_var( 'cfg_file_path' ),

features/bootstrap/Process.php

@@ -115,4 +115,20 @@ public function run_check() {
 
 		return $r;
 	}
+
+	/**
+	 * Run the command, but throw an Exception on error.
+	 * Same as `run_check()` above, but checks the correct stderr.
+	 *
+	 * @return ProcessRun
+	 */
+	public function run_check_stderr() {
+		$r = $this->run();
+
+		if ( $r->return_code || ! empty( $r->stderr ) ) {
+			throw new \RuntimeException( $r );
+		}
+
+		return $r;
+	}
 }

features/bootstrap/utils.php

@@ -833,6 +833,18 @@ function parse_ssh_url( $url, $component = -1 ) {
 			$bits[ $key ] = $matches[ $i ];
 		}
 	}
+
+	// Find the hostname from `vagrant ssh-config` automatically.
+	if ( preg_match( '/^vagrant:?/', $url ) ) {
+		if ( 'vagrant' === $bits['host'] && empty( $bits['scheme'] ) ) {
+			$ssh_config = shell_exec( 'vagrant ssh-config 2>/dev/null' );
+			if ( preg_match( '/Host\s(.+)/', $ssh_config, $matches ) ) {
+				$bits['scheme'] = 'vagrant';
+				$bits['host']   = $matches[1];
+			}
+		}
+	}
+
 	switch ( $component ) {
 		case PHP_URL_SCHEME:
 			return isset( $bits['scheme'] ) ? $bits['scheme'] : null;
@@ -1269,3 +1281,40 @@ function past_tense_verb( $verb ) {
 	}
 	return $verb . 'ed';
 }
+
+/**
+ * Get the path to the PHP binary used when executing WP-CLI.
+ *
+ * Environment values permit specific binaries to be indicated.
+ *
+ * @access public
+ * @category System
+ *
+ * @return string
+ */
+function get_php_binary() {
+	if ( $wp_cli_php_used = getenv( 'WP_CLI_PHP_USED' ) ) {
+		return $wp_cli_php_used;
+	}
+
+	if ( $wp_cli_php = getenv( 'WP_CLI_PHP' ) ) {
+		return $wp_cli_php;
+	}
+
+	// Available since PHP 5.4.
+	if ( defined( 'PHP_BINARY' ) ) {
+		return PHP_BINARY;
+	}
+
+	// @codingStandardsIgnoreLine
+	if ( @is_executable( PHP_BINDIR . '/php' ) ) {
+		return PHP_BINDIR . '/php';
+	}
+
+	// @codingStandardsIgnoreLine
+	if ( is_windows() && @is_executable( PHP_BINDIR . '/php.exe' ) ) {
+		return PHP_BINDIR . '/php.exe';
+	}
+
+	return 'php';
+}

features/steps/then.php

@@ -3,9 +3,9 @@
 use Behat\Gherkin\Node\PyStringNode,
     Behat\Gherkin\Node\TableNode;
 
-$steps->Then( '/^the return code should be (\d+)$/',
-	function ( $world, $return_code ) {
-		if ( $return_code != $world->result->return_code ) {
+$steps->Then( '/^the return code should( not)? be (\d+)$/',
+	function ( $world, $not, $return_code ) {
+		if ( ( ! $not && $return_code != $world->result->return_code ) || ( $not && $return_code == $world->result->return_code ) ) {
 			throw new RuntimeException( $world->result );
 		}
 	}

features/steps/when.php

@@ -6,7 +6,7 @@
 
 function invoke_proc( $proc, $mode ) {
 	$map = array(
-		'run' => 'run_check',
+		'run' => 'run_check_stderr',
 		'try' => 'run'
 	);
 	$method = $map[ $mode ];

utils/behat-tags.php

@@ -44,8 +44,10 @@ function version_tags( $prefix, $current, $operator = '<' ) {
 	version_tags( 'less-than-php', PHP_VERSION, '>' )
 );
 
-# Skip Github API tests by default because of rate limiting. See https://github.com/wp-cli/wp-cli/issues/1612
-$skip_tags[] = '@github-api';
+# Skip Github API tests if `GITHUB_TOKEN` not available because of rate limiting. See https://github.com/wp-cli/wp-cli/issues/1612
+if ( ! getenv( 'GITHUB_TOKEN' ) ) {
+	$skip_tags[] = '@github-api';
+}
 
 # Skip tests known to be broken.
 $skip_tags[] = '@broken';