From 3fe0ce015db50b9e3ca1ee11ff63bf1f91e60c59 Mon Sep 17 00:00:00 2001 From: ilyazub Date: Wed, 22 Nov 2023 18:35:49 +0100 Subject: [PATCH 1/6] Remove default `seed` value Changes * Remove the default `seed` value (ref: https://github.com/serpapi/turbo_tests/pull/33) * Use exit status from RSpec calls (ref: https://github.com/serpapi/turbo_tests/pull/20) --- lib/turbo_tests/cli.rb | 9 +-- lib/turbo_tests/reporter.rb | 48 ++++++++++--- lib/turbo_tests/runner.rb | 65 +++++++++--------- spec/cli_spec.rb | 130 +++++++++++++++++++++++------------- 4 files changed, 157 insertions(+), 95 deletions(-) diff --git a/lib/turbo_tests/cli.rb b/lib/turbo_tests/cli.rb index ed80ba7..6f033cc 100644 --- a/lib/turbo_tests/cli.rb +++ b/lib/turbo_tests/cli.rb @@ -98,7 +98,7 @@ def run end end - success = TurboTests::Runner.run( + exitstatus = TurboTests::Runner.run( formatters: formatters, tags: tags, files: @argv.empty? ? ["spec"] : @argv, @@ -109,11 +109,8 @@ def run seed: seed ) - if success - exit 0 - else - exit 1 - end + # From https://github.com/serpapi/turbo_tests/pull/20/ + exit exitstatus end end end diff --git a/lib/turbo_tests/reporter.rb b/lib/turbo_tests/reporter.rb index 4996dca..28d2ce2 100644 --- a/lib/turbo_tests/reporter.rb +++ b/lib/turbo_tests/reporter.rb @@ -4,8 +4,8 @@ module TurboTests class Reporter attr_writer :load_time - def self.from_config(formatter_config, start_time) - reporter = new(start_time) + def self.from_config(formatter_config, start_time, seed, seed_used) + reporter = new(start_time, seed, seed_used) formatter_config.each do |config| name, outputs = config.values_at(:name, :outputs) @@ -23,13 +23,15 @@ def self.from_config(formatter_config, start_time) attr_reader :pending_examples attr_reader :failed_examples - def initialize(start_time) + def initialize(start_time, seed, seed_used) @formatters = [] @pending_examples = [] @failed_examples = [] @all_examples = [] @messages = [] @start_time = start_time + @seed = seed + @seed_used = seed_used @load_time = 0 @errors_outside_of_examples_count = 0 end @@ -50,6 +52,33 @@ def add(name, outputs) end end + # Borrowed from RSpec::Core::Reporter + # https://github.com/rspec/rspec-core/blob/1eeadce5aa7137ead054783c31ff35cbfe9d07cc/lib/rspec/core/reporter.rb#L206 + def report(expected_example_count) + start(expected_example_count) + begin + yield self + ensure + finish + end + end + + def start(example_groups) + delegate_to_formatters(:seed, RSpec::Core::Notifications::SeedNotification.new(@seed, @seed_used)) + + report_number_of_tests(example_groups) + end + + def report_number_of_tests(groups) + name = ParallelTests::RSpec::Runner.test_file_name + + num_processes = groups.size + num_tests = groups.map(&:size).sum + tests_per_process = (num_processes == 0 ? 0 : num_tests.to_f / num_processes).round + + puts "#{num_processes} processes for #{num_tests} #{name}s, ~ #{tests_per_process} #{name}s per process" + end + def group_started(notification) delegate_to_formatters(:example_group_started, notification) end @@ -88,8 +117,7 @@ def error_outside_of_examples end def finish - # SEE: https://bit.ly/2NP87Cz - end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + end_time = RSpec::Core::Time.now delegate_to_formatters(:start_dump, RSpec::Core::Notifications::NullNotification) @@ -112,11 +140,11 @@ def finish )) delegate_to_formatters(:close, RSpec::Core::Notifications::NullNotification) - end - - def seed_notification(seed, seed_used) - puts RSpec::Core::Notifications::SeedNotification.new(seed, seed_used).fully_formatted - puts + delegate_to_formatters(:seed, + RSpec::Core::Notifications::SeedNotification.new( + @seed, + @seed_used, + )) end protected diff --git a/lib/turbo_tests/runner.rb b/lib/turbo_tests/runner.rb index 1a74b76..372f7bf 100644 --- a/lib/turbo_tests/runner.rb +++ b/lib/turbo_tests/runner.rb @@ -14,20 +14,19 @@ def self.run(opts = {}) formatters = opts[:formatters] tags = opts[:tags] - # SEE: https://bit.ly/2NP87Cz - start_time = opts.fetch(:start_time) { Process.clock_gettime(Process::CLOCK_MONOTONIC) } + start_time = opts.fetch(:start_time) { RSpec::Core::Time.now } runtime_log = opts.fetch(:runtime_log, nil) verbose = opts.fetch(:verbose, false) fail_fast = opts.fetch(:fail_fast, nil) count = opts.fetch(:count, nil) - seed = opts.fetch(:seed) || rand(0xFFFF).to_s - seed_used = !opts[:seed].nil? + seed = opts.fetch(:seed) + seed_used = !seed.nil? if verbose STDERR.puts "VERBOSE" end - reporter = Reporter.from_config(formatters, start_time) + reporter = Reporter.from_config(formatters, start_time, seed, seed_used) new( reporter: reporter, @@ -38,7 +37,7 @@ def self.run(opts = {}) fail_fast: fail_fast, count: count, seed: seed, - seed_used: seed_used + seed_used: seed_used, ).run end @@ -50,11 +49,12 @@ def initialize(opts) @verbose = opts[:verbose] @fail_fast = opts[:fail_fast] @count = opts[:count] + @seed = opts[:seed] + @seed_used = opts[:seed_used] + @load_time = 0 @load_count = 0 @failure_count = 0 - @seed = opts[:seed] - @seed_used = opts[:seed_used] @messages = Thread::Queue.new @threads = [] @@ -87,26 +87,25 @@ def run setup_tmp_dir subprocess_opts = { - record_runtime: use_runtime_info + record_runtime: use_runtime_info, } - report_number_of_tests(tests_in_groups) - - @reporter.seed_notification(@seed, @seed_used) - - wait_threads = tests_in_groups.map.with_index do |tests, process_id| - start_regular_subprocess(tests, process_id + 1, **subprocess_opts) - end - - handle_messages - - @reporter.finish + @reporter.report(tests_in_groups) do |reporter| + wait_threads = tests_in_groups.map.with_index do |tests, process_id| + start_regular_subprocess(tests, process_id + 1, **subprocess_opts) + end - @reporter.seed_notification(@seed, @seed_used) + handle_messages - @threads.each(&:join) + @threads.each(&:join) - @reporter.failed_examples.empty? && wait_threads.map(&:value).all?(&:success?) + if @reporter.failed_examples.empty? && wait_threads.map(&:value).all?(&:success?) + 0 + else + # From https://github.com/serpapi/turbo_tests/pull/20/ + wait_threads.map { |thread| thread.value.exitstatus }.max + end + end end private @@ -157,10 +156,18 @@ def start_subprocess(env, extra_args, tests, process_id, record_runtime:) [] end + seed_option = if @seed_used + [ + "--seed", @seed, + ] + else + [] + end + command = [ "rspec", *extra_args, - "--seed", @seed, + *seed_option, "--format", "TurboTests::JsonRowsFormatter", "--out", tmp_filename, *record_runtime_options, @@ -273,15 +280,5 @@ def handle_messages def fail_fast_met !@fail_fast.nil? && @failure_count >= @fail_fast end - - def report_number_of_tests(groups) - name = ParallelTests::RSpec::Runner.test_file_name - - num_processes = groups.size - num_tests = groups.map(&:size).sum - tests_per_process = (num_processes == 0 ? 0 : num_tests.to_f / num_processes).round - - puts "#{num_processes} processes for #{num_tests} #{name}s, ~ #{tests_per_process} #{name}s per process" - end end end diff --git a/spec/cli_spec.rb b/spec/cli_spec.rb index 2b2836a..243b7d1 100644 --- a/spec/cli_spec.rb +++ b/spec/cli_spec.rb @@ -1,15 +1,18 @@ RSpec.describe TurboTests::CLI do - subject(:output) { `bundle exec turbo_tests -f d #{fixture} --seed 1234`.strip } - before { output } - context "errors outside of examples" do - let(:expected_start_of_output) { - %( -1 processes for 1 specs, ~ 1 specs per process + subject(:output) { `bundle exec turbo_tests -f d #{fixture}`.strip } -Randomized with seed 1234 + context "when the 'seed' parameter was used" do + let(:seed) { 1234 } + subject(:output) { `bundle exec turbo_tests -f d #{fixture} --seed #{seed}`.strip } + + context "errors outside of examples" do + let(:expected_start_of_output) { +%( +Randomized with seed #{seed} +1 processes for 1 specs, ~ 1 specs per process An error occurred while loading #{fixture}. \e[31mFailure/Error: \e[0m\e[1;34m1\e[0m / \e[1;34m0\e[0m\e[0m @@ -20,61 +23,98 @@ \e[36m# #{fixture}:4:in `block in '\e[0m \e[36m# #{fixture}:1:in `'\e[0m ).strip - } - let(:expected_end_of_output) do - "0 examples, 0 failures\n"\ + } + + let(:expected_end_of_output) do + "0 examples, 0 failures\n"\ "\n\n"\ - "Randomized with seed 1234" + "Randomized with seed #{seed}" + end + + let(:fixture) { "./fixtures/rspec/errors_outside_of_examples_spec.rb" } + + it "reports" do + expect($?.exitstatus).to eql(1) + + expect(output).to start_with(expected_start_of_output) + expect(output).to end_with(expected_end_of_output) + end end - let(:fixture) { "./fixtures/rspec/errors_outside_of_examples_spec.rb" } + context "pending exceptions", :aggregate_failures do + let(:fixture) { "./fixtures/rspec/pending_exceptions_spec.rb" } - it "reports" do - expect($?.exitstatus).to eql(1) + it "reports" do + expect($?.exitstatus).to eql(0) + + [ + "is implemented but skipped with 'pending' (PENDING: TODO: skipped with 'pending')", + "is implemented but skipped with 'skip' (PENDING: TODO: skipped with 'skip')", + "is implemented but skipped with 'xit' (PENDING: Temporarily skipped with xit)", - expect(output).to start_with(expected_start_of_output) - expect(output).to end_with(expected_end_of_output) + "Pending: (Failures listed here are expected and do not affect your suite's status)", + ].each do |part| + expect(output).to include(part) + end + + expect(output).to end_with("3 examples, 0 failures, 3 pending\n\n\nRandomized with seed #{seed}") + end end end - context "pending exceptions", :aggregate_failures do - let(:fixture) { "./fixtures/rspec/pending_exceptions_spec.rb" } + context "when 'seed' parameter was not used" do + context "errors outside of examples" do + let(:expected_start_of_output) { +%( +1 processes for 1 specs, ~ 1 specs per process + +An error occurred while loading #{fixture}. +\e[31mFailure/Error: \e[0m\e[1;34m1\e[0m / \e[1;34m0\e[0m\e[0m +\e[31m\e[0m +\e[31mZeroDivisionError:\e[0m +\e[31m divided by 0\e[0m +\e[36m# #{fixture}:4:in `/'\e[0m +\e[36m# #{fixture}:4:in `block in '\e[0m +\e[36m# #{fixture}:1:in `'\e[0m +).strip + } + + let(:expected_end_of_output) do + "0 examples, 0 failures" + end + + let(:fixture) { "./fixtures/rspec/errors_outside_of_examples_spec.rb" } - it "reports" do - expect($?.exitstatus).to eql(0) + it "reports" do + expect($?.exitstatus).to eql(1) - [ - "is implemented but skipped with 'pending' (PENDING: TODO: skipped with 'pending')", - "is implemented but skipped with 'skip' (PENDING: TODO: skipped with 'skip')", - "is implemented but skipped with 'xit' (PENDING: Temporarily skipped with xit)", + expect(output).to start_with(expected_start_of_output) + expect(output).to end_with(expected_end_of_output) + end - "Pending: (Failures listed here are expected and do not affect your suite's status)", + it "exludes the seed message from the output" do + expect(output).to_not include("seed") + end + end - %{ -Fixture of spec file with pending failed examples is implemented but skipped with 'pending' - # TODO: skipped with 'pending' - Failure/Error: DEFAULT_FAILURE_NOTIFIER = lambda { |failure, _opts| raise failure } + context "pending exceptions", :aggregate_failures do + let(:fixture) { "./fixtures/rspec/pending_exceptions_spec.rb" } - expected: 3 - got: 2 + it "reports" do + expect($?.exitstatus).to eql(0) - (compared using ==) - }.strip, + [ + "is implemented but skipped with 'pending' (PENDING: TODO: skipped with 'pending')", + "is implemented but skipped with 'skip' (PENDING: TODO: skipped with 'skip')", + "is implemented but skipped with 'xit' (PENDING: Temporarily skipped with xit)", - %( -Fixture of spec file with pending failed examples is implemented but skipped with 'skip' - # TODO: skipped with 'skip' - ).strip, + "Pending: (Failures listed here are expected and do not affect your suite's status)", + ].each do |part| + expect(output).to include(part) + end - %( -Fixture of spec file with pending failed examples is implemented but skipped with 'xit' - # Temporarily skipped with xit - ).strip - ].each do |part| - expect(output).to include(part) + expect(output).to end_with("3 examples, 0 failures, 3 pending") end - - expect(output).to end_with("3 examples, 0 failures, 3 pending\n\n\nRandomized with seed 1234") end end From c5bf35cb17f8399e70ab6d8c560a2349c5643590 Mon Sep 17 00:00:00 2001 From: ilyazub Date: Mon, 3 Jun 2024 20:06:18 +0200 Subject: [PATCH 2/6] Fix tests with the `seed` option --- Gemfile.lock | 2 +- lib/turbo_tests/reporter.rb | 25 ++++++++++++++++--------- lib/turbo_tests/version.rb | 2 +- spec/cli_spec.rb | 7 ++++--- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 9e06c70..a95169f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - turbo_tests (2.2.3) + turbo_tests (2.2.4) parallel_tests (>= 3.3.0, < 5) rspec (>= 3.10) diff --git a/lib/turbo_tests/reporter.rb b/lib/turbo_tests/reporter.rb index 28d2ce2..9f281d7 100644 --- a/lib/turbo_tests/reporter.rb +++ b/lib/turbo_tests/reporter.rb @@ -53,9 +53,9 @@ def add(name, outputs) end # Borrowed from RSpec::Core::Reporter - # https://github.com/rspec/rspec-core/blob/1eeadce5aa7137ead054783c31ff35cbfe9d07cc/lib/rspec/core/reporter.rb#L206 - def report(expected_example_count) - start(expected_example_count) + # https://github.com/rspec/rspec-core/blob/5699fcdc4723087ff6139af55bd155ad9ad61a7b/lib/rspec/core/reporter.rb#L71 + def report(example_groups) + start(example_groups) begin yield self ensure @@ -63,10 +63,15 @@ def report(expected_example_count) end end - def start(example_groups) - delegate_to_formatters(:seed, RSpec::Core::Notifications::SeedNotification.new(@seed, @seed_used)) + def start(example_groups, time=RSpec::Core::Time.now) + @start = time + @load_time = (@start - @start_time).to_f report_number_of_tests(example_groups) + expected_example_count = example_groups.flatten(1).count + + delegate_to_formatters(:seed, RSpec::Core::Notifications::SeedNotification.new(@seed, @seed_used)) + delegate_to_formatters(:start, RSpec::Core::Notifications::StartNotification.new(expected_example_count, @load_time)) end def report_number_of_tests(groups) @@ -119,8 +124,10 @@ def error_outside_of_examples def finish end_time = RSpec::Core::Time.now - delegate_to_formatters(:start_dump, - RSpec::Core::Notifications::NullNotification) + @duration = end_time - @start_time + delegate_to_formatters :stop, RSpec::Core::Notifications::ExamplesNotification.new(self) + + delegate_to_formatters :start_dump, RSpec::Core::Notifications::NullNotification delegate_to_formatters(:dump_pending, RSpec::Core::Notifications::ExamplesNotification.new( self @@ -138,13 +145,13 @@ def finish @load_time, @errors_outside_of_examples_count )) - delegate_to_formatters(:close, - RSpec::Core::Notifications::NullNotification) delegate_to_formatters(:seed, RSpec::Core::Notifications::SeedNotification.new( @seed, @seed_used, )) + ensure + delegate_to_formatters :close, RSpec::Core::Notifications::NullNotification end protected diff --git a/lib/turbo_tests/version.rb b/lib/turbo_tests/version.rb index 3e7c830..4fa53b6 100644 --- a/lib/turbo_tests/version.rb +++ b/lib/turbo_tests/version.rb @@ -1,3 +1,3 @@ module TurboTests - VERSION = "2.2.3" + VERSION = "2.2.4" end diff --git a/spec/cli_spec.rb b/spec/cli_spec.rb index 243b7d1..9594056 100644 --- a/spec/cli_spec.rb +++ b/spec/cli_spec.rb @@ -11,9 +11,10 @@ context "errors outside of examples" do let(:expected_start_of_output) { %( -Randomized with seed #{seed} 1 processes for 1 specs, ~ 1 specs per process +Randomized with seed #{seed} + An error occurred while loading #{fixture}. \e[31mFailure/Error: \e[0m\e[1;34m1\e[0m / \e[1;34m0\e[0m\e[0m \e[31m\e[0m @@ -27,7 +28,7 @@ let(:expected_end_of_output) do "0 examples, 0 failures\n"\ - "\n\n"\ + "\n"\ "Randomized with seed #{seed}" end @@ -57,7 +58,7 @@ expect(output).to include(part) end - expect(output).to end_with("3 examples, 0 failures, 3 pending\n\n\nRandomized with seed #{seed}") + expect(output).to end_with("3 examples, 0 failures, 3 pending\n\nRandomized with seed #{seed}") end end end From 99a0e049bb9e75cd716939cb2c6bb331b0c9b699 Mon Sep 17 00:00:00 2001 From: ilyazub Date: Mon, 3 Jun 2024 21:12:34 +0200 Subject: [PATCH 3/6] Remove Snyk integration --- .github/workflows/snyk_ruby-analysis.yml | 33 ------------------------ 1 file changed, 33 deletions(-) delete mode 100644 .github/workflows/snyk_ruby-analysis.yml diff --git a/.github/workflows/snyk_ruby-analysis.yml b/.github/workflows/snyk_ruby-analysis.yml deleted file mode 100644 index 6df7782..0000000 --- a/.github/workflows/snyk_ruby-analysis.yml +++ /dev/null @@ -1,33 +0,0 @@ -# A sample workflow which checks out your Infrastructure as Code Configuration files, -# such as Kubernetes, Helm & Terraform and scans them for any security issues. -# The results are then uploaded to GitHub Security Code Scanning -# -# For more examples, including how to limit scans to only high-severity issues -# and fail PR checks, see https://github.com/snyk/actions/ - -name: Snyk Ruby - -on: - push: - branches: [ master ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ master ] - schedule: - - cron: '0 0 * * 0' - -jobs: - snyk: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Run Snyk to check configuration files for security issues - # Snyk can be used to break the build when it detects security issues. - # In this case we want to upload the issues to GitHub Code Scanning - continue-on-error: true - uses: snyk/actions/ruby@master - env: - # In order to use the Snyk Action you will need to have a Snyk API token. - # More details in https://github.com/snyk/actions#getting-your-snyk-token - # or you can signup for free at https://snyk.io/login - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} From 84f2487353bc22bbabc0163a1fcad45bc6d22984 Mon Sep 17 00:00:00 2001 From: ilyazub Date: Mon, 3 Jun 2024 21:25:07 +0200 Subject: [PATCH 4/6] Output errors outside of examples --- lib/turbo_tests/reporter.rb | 3 ++- lib/turbo_tests/runner.rb | 11 ++++++++--- spec/cli_spec.rb | 4 ++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/turbo_tests/reporter.rb b/lib/turbo_tests/reporter.rb index 9f281d7..417f06a 100644 --- a/lib/turbo_tests/reporter.rb +++ b/lib/turbo_tests/reporter.rb @@ -117,8 +117,9 @@ def message(message) @messages << message end - def error_outside_of_examples + def error_outside_of_examples(error_message) @errors_outside_of_examples_count += 1 + message error_message end def finish diff --git a/lib/turbo_tests/runner.rb b/lib/turbo_tests/runner.rb index 430f79a..9db297d 100644 --- a/lib/turbo_tests/runner.rb +++ b/lib/turbo_tests/runner.rb @@ -259,12 +259,17 @@ def handle_messages break end when "message" - @reporter.message(message[:message]) + if message[:message].include?("An error occurred") || message[:message].include?("occurred outside of examples") + @reporter.error_outside_of_examples(message[:message]) + @error = true + else + @reporter.message(message[:message]) + end when "seed" when "close" when "error" - @reporter.error_outside_of_examples - @error = true + # Do nothing + nil when "exit" exited += 1 if exited == @num_processes diff --git a/spec/cli_spec.rb b/spec/cli_spec.rb index 9594056..443c6bf 100644 --- a/spec/cli_spec.rb +++ b/spec/cli_spec.rb @@ -27,7 +27,7 @@ } let(:expected_end_of_output) do - "0 examples, 0 failures\n"\ + "0 examples, 0 failures, 1 error occurred outside of examples\n"\ "\n"\ "Randomized with seed #{seed}" end @@ -81,7 +81,7 @@ } let(:expected_end_of_output) do - "0 examples, 0 failures" + "0 examples, 0 failures, 1 error occurred outside of examples" end let(:fixture) { "./fixtures/rspec/errors_outside_of_examples_spec.rb" } From 6455e659f26b308b1a1a10e3970471676c112927 Mon Sep 17 00:00:00 2001 From: ilyazub Date: Tue, 4 Jun 2024 17:49:00 +0200 Subject: [PATCH 5/6] Make sure runtime errors are outputted Resolves https://github.com/serpapi/turbo_tests/issues/40 --- fixtures/rspec/no_method_error_spec.rb | 3 +++ spec/cli_spec.rb | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 fixtures/rspec/no_method_error_spec.rb diff --git a/fixtures/rspec/no_method_error_spec.rb b/fixtures/rspec/no_method_error_spec.rb new file mode 100644 index 0000000..c8c9689 --- /dev/null +++ b/fixtures/rspec/no_method_error_spec.rb @@ -0,0 +1,3 @@ +RSpec.describe "NoMethodError spec" do + it("fails") { expect(nil[:key]).to eql("value") } +end \ No newline at end of file diff --git a/spec/cli_spec.rb b/spec/cli_spec.rb index 443c6bf..6aae9b7 100644 --- a/spec/cli_spec.rb +++ b/spec/cli_spec.rb @@ -128,4 +128,21 @@ expect(output).to include("Test info in extra_failure_lines") end end + + describe "full error failure message and line" do + let(:fixture) { "./fixtures/rspec/no_method_error_spec.rb" } + + it "outputs file name and line number" do + expect($?.exitstatus).to eql(1) + + [ + "undefined method `[]' for nil:NilClass", + 'it("fails") { expect(nil[:key]).to eql("value") }', + "# #{fixture}:2:in `block (2 levels) in '", + "1 example, 1 failure", + ].each do |part| + expect(output).to include(part) + end + end + end end From 4115412f5265c74aca057849dbcf98221958c62a Mon Sep 17 00:00:00 2001 From: ilyazub Date: Tue, 4 Jun 2024 18:15:03 +0200 Subject: [PATCH 6/6] Fix tests for Ruby 3.3 Ruby 3.3 didn't output `NilClass` in `undefined method `[]' for nil` message Ref: https://github.com/serpapi/turbo_tests/actions/runs/9370241051/job/25796715330 --- spec/cli_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/cli_spec.rb b/spec/cli_spec.rb index 6aae9b7..93fa132 100644 --- a/spec/cli_spec.rb +++ b/spec/cli_spec.rb @@ -136,7 +136,7 @@ expect($?.exitstatus).to eql(1) [ - "undefined method `[]' for nil:NilClass", + "undefined method `[]' for nil", 'it("fails") { expect(nil[:key]).to eql("value") }', "# #{fixture}:2:in `block (2 levels) in '", "1 example, 1 failure",