diff --git a/.github/workflows/snyk_ruby-analysis.yml b/.github/workflows/snyk_ruby-analysis.yml deleted file mode 100644 index 6df7782..0000000 --- a/.github/workflows/snyk_ruby-analysis.yml +++ /dev/null @@ -1,33 +0,0 @@ -# A sample workflow which checks out your Infrastructure as Code Configuration files, -# such as Kubernetes, Helm & Terraform and scans them for any security issues. -# The results are then uploaded to GitHub Security Code Scanning -# -# For more examples, including how to limit scans to only high-severity issues -# and fail PR checks, see https://github.com/snyk/actions/ - -name: Snyk Ruby - -on: - push: - branches: [ master ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ master ] - schedule: - - cron: '0 0 * * 0' - -jobs: - snyk: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Run Snyk to check configuration files for security issues - # Snyk can be used to break the build when it detects security issues. - # In this case we want to upload the issues to GitHub Code Scanning - continue-on-error: true - uses: snyk/actions/ruby@master - env: - # In order to use the Snyk Action you will need to have a Snyk API token. - # More details in https://github.com/snyk/actions#getting-your-snyk-token - # or you can signup for free at https://snyk.io/login - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} diff --git a/Gemfile.lock b/Gemfile.lock index 9e06c70..a95169f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - turbo_tests (2.2.3) + turbo_tests (2.2.4) parallel_tests (>= 3.3.0, < 5) rspec (>= 3.10) diff --git a/fixtures/rspec/no_method_error_spec.rb b/fixtures/rspec/no_method_error_spec.rb new file mode 100644 index 0000000..c8c9689 --- /dev/null +++ b/fixtures/rspec/no_method_error_spec.rb @@ -0,0 +1,3 @@ +RSpec.describe "NoMethodError spec" do + it("fails") { expect(nil[:key]).to eql("value") } +end \ No newline at end of file diff --git a/lib/turbo_tests/cli.rb b/lib/turbo_tests/cli.rb index ed80ba7..6f033cc 100644 --- a/lib/turbo_tests/cli.rb +++ b/lib/turbo_tests/cli.rb @@ -98,7 +98,7 @@ def run end end - success = TurboTests::Runner.run( + exitstatus = TurboTests::Runner.run( formatters: formatters, tags: tags, files: @argv.empty? ? ["spec"] : @argv, @@ -109,11 +109,8 @@ def run seed: seed ) - if success - exit 0 - else - exit 1 - end + # From https://github.com/serpapi/turbo_tests/pull/20/ + exit exitstatus end end end diff --git a/lib/turbo_tests/reporter.rb b/lib/turbo_tests/reporter.rb index 4996dca..417f06a 100644 --- a/lib/turbo_tests/reporter.rb +++ b/lib/turbo_tests/reporter.rb @@ -4,8 +4,8 @@ module TurboTests class Reporter attr_writer :load_time - def self.from_config(formatter_config, start_time) - reporter = new(start_time) + def self.from_config(formatter_config, start_time, seed, seed_used) + reporter = new(start_time, seed, seed_used) formatter_config.each do |config| name, outputs = config.values_at(:name, :outputs) @@ -23,13 +23,15 @@ def self.from_config(formatter_config, start_time) attr_reader :pending_examples attr_reader :failed_examples - def initialize(start_time) + def initialize(start_time, seed, seed_used) @formatters = [] @pending_examples = [] @failed_examples = [] @all_examples = [] @messages = [] @start_time = start_time + @seed = seed + @seed_used = seed_used @load_time = 0 @errors_outside_of_examples_count = 0 end @@ -50,6 +52,38 @@ def add(name, outputs) end end + # Borrowed from RSpec::Core::Reporter + # https://github.com/rspec/rspec-core/blob/5699fcdc4723087ff6139af55bd155ad9ad61a7b/lib/rspec/core/reporter.rb#L71 + def report(example_groups) + start(example_groups) + begin + yield self + ensure + finish + end + end + + def start(example_groups, time=RSpec::Core::Time.now) + @start = time + @load_time = (@start - @start_time).to_f + + report_number_of_tests(example_groups) + expected_example_count = example_groups.flatten(1).count + + delegate_to_formatters(:seed, RSpec::Core::Notifications::SeedNotification.new(@seed, @seed_used)) + delegate_to_formatters(:start, RSpec::Core::Notifications::StartNotification.new(expected_example_count, @load_time)) + end + + def report_number_of_tests(groups) + name = ParallelTests::RSpec::Runner.test_file_name + + num_processes = groups.size + num_tests = groups.map(&:size).sum + tests_per_process = (num_processes == 0 ? 0 : num_tests.to_f / num_processes).round + + puts "#{num_processes} processes for #{num_tests} #{name}s, ~ #{tests_per_process} #{name}s per process" + end + def group_started(notification) delegate_to_formatters(:example_group_started, notification) end @@ -83,16 +117,18 @@ def message(message) @messages << message end - def error_outside_of_examples + def error_outside_of_examples(error_message) @errors_outside_of_examples_count += 1 + message error_message end def finish - # SEE: https://bit.ly/2NP87Cz - end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + end_time = RSpec::Core::Time.now + + @duration = end_time - @start_time + delegate_to_formatters :stop, RSpec::Core::Notifications::ExamplesNotification.new(self) - delegate_to_formatters(:start_dump, - RSpec::Core::Notifications::NullNotification) + delegate_to_formatters :start_dump, RSpec::Core::Notifications::NullNotification delegate_to_formatters(:dump_pending, RSpec::Core::Notifications::ExamplesNotification.new( self @@ -110,13 +146,13 @@ def finish @load_time, @errors_outside_of_examples_count )) - delegate_to_formatters(:close, - RSpec::Core::Notifications::NullNotification) - end - - def seed_notification(seed, seed_used) - puts RSpec::Core::Notifications::SeedNotification.new(seed, seed_used).fully_formatted - puts + delegate_to_formatters(:seed, + RSpec::Core::Notifications::SeedNotification.new( + @seed, + @seed_used, + )) + ensure + delegate_to_formatters :close, RSpec::Core::Notifications::NullNotification end protected diff --git a/lib/turbo_tests/runner.rb b/lib/turbo_tests/runner.rb index ce2db9e..9db297d 100644 --- a/lib/turbo_tests/runner.rb +++ b/lib/turbo_tests/runner.rb @@ -14,20 +14,19 @@ def self.run(opts = {}) formatters = opts[:formatters] tags = opts[:tags] - # SEE: https://bit.ly/2NP87Cz - start_time = opts.fetch(:start_time) { Process.clock_gettime(Process::CLOCK_MONOTONIC) } + start_time = opts.fetch(:start_time) { RSpec::Core::Time.now } runtime_log = opts.fetch(:runtime_log, nil) verbose = opts.fetch(:verbose, false) fail_fast = opts.fetch(:fail_fast, nil) count = opts.fetch(:count, nil) - seed = opts.fetch(:seed) || rand(0xFFFF).to_s - seed_used = !opts[:seed].nil? + seed = opts.fetch(:seed) + seed_used = !seed.nil? if verbose warn "VERBOSE" end - reporter = Reporter.from_config(formatters, start_time) + reporter = Reporter.from_config(formatters, start_time, seed, seed_used) new( reporter: reporter, @@ -38,7 +37,7 @@ def self.run(opts = {}) fail_fast: fail_fast, count: count, seed: seed, - seed_used: seed_used + seed_used: seed_used, ).run end @@ -50,11 +49,12 @@ def initialize(opts) @verbose = opts[:verbose] @fail_fast = opts[:fail_fast] @count = opts[:count] + @seed = opts[:seed] + @seed_used = opts[:seed_used] + @load_time = 0 @load_count = 0 @failure_count = 0 - @seed = opts[:seed] - @seed_used = opts[:seed_used] @messages = Thread::Queue.new @threads = [] @@ -87,26 +87,25 @@ def run setup_tmp_dir subprocess_opts = { - record_runtime: use_runtime_info + record_runtime: use_runtime_info, } - report_number_of_tests(tests_in_groups) - - @reporter.seed_notification(@seed, @seed_used) - - wait_threads = tests_in_groups.map.with_index do |tests, process_id| - start_regular_subprocess(tests, process_id + 1, **subprocess_opts) - end - - handle_messages - - @reporter.finish + @reporter.report(tests_in_groups) do |reporter| + wait_threads = tests_in_groups.map.with_index do |tests, process_id| + start_regular_subprocess(tests, process_id + 1, **subprocess_opts) + end - @reporter.seed_notification(@seed, @seed_used) + handle_messages - @threads.each(&:join) + @threads.each(&:join) - @reporter.failed_examples.empty? && wait_threads.map(&:value).all?(&:success?) + if @reporter.failed_examples.empty? && wait_threads.map(&:value).all?(&:success?) + 0 + else + # From https://github.com/serpapi/turbo_tests/pull/20/ + wait_threads.map { |thread| thread.value.exitstatus }.max + end + end end private @@ -157,12 +156,18 @@ def start_subprocess(env, extra_args, tests, process_id, record_runtime:) [] end + seed_option = if @seed_used + [ + "--seed", @seed, + ] + else + [] + end + command = [ *command_name, *extra_args, - "--seed", rand(0xFFFF).to_s, - "--format", "ParallelTests::RSpec::RuntimeLogger", - "--out", @runtime_log, + *seed_option, "--format", "TurboTests::JsonRowsFormatter", *record_runtime_options, *tests, @@ -254,12 +259,17 @@ def handle_messages break end when "message" - @reporter.message(message[:message]) + if message[:message].include?("An error occurred") || message[:message].include?("occurred outside of examples") + @reporter.error_outside_of_examples(message[:message]) + @error = true + else + @reporter.message(message[:message]) + end when "seed" when "close" when "error" - @reporter.error_outside_of_examples - @error = true + # Do nothing + nil when "exit" exited += 1 if exited == @num_processes @@ -277,15 +287,5 @@ def handle_messages def fail_fast_met !@fail_fast.nil? && @failure_count >= @fail_fast end - - def report_number_of_tests(groups) - name = ParallelTests::RSpec::Runner.test_file_name - - num_processes = groups.size - num_tests = groups.map(&:size).sum - tests_per_process = (num_processes == 0 ? 0 : num_tests.to_f / num_processes).round - - puts "#{num_processes} processes for #{num_tests} #{name}s, ~ #{tests_per_process} #{name}s per process" - end end end diff --git a/lib/turbo_tests/version.rb b/lib/turbo_tests/version.rb index 3e7c830..4fa53b6 100644 --- a/lib/turbo_tests/version.rb +++ b/lib/turbo_tests/version.rb @@ -1,3 +1,3 @@ module TurboTests - VERSION = "2.2.3" + VERSION = "2.2.4" end diff --git a/spec/cli_spec.rb b/spec/cli_spec.rb index 2b2836a..93fa132 100644 --- a/spec/cli_spec.rb +++ b/spec/cli_spec.rb @@ -1,15 +1,19 @@ RSpec.describe TurboTests::CLI do - subject(:output) { `bundle exec turbo_tests -f d #{fixture} --seed 1234`.strip } - before { output } - context "errors outside of examples" do - let(:expected_start_of_output) { - %( -1 processes for 1 specs, ~ 1 specs per process + subject(:output) { `bundle exec turbo_tests -f d #{fixture}`.strip } + + context "when the 'seed' parameter was used" do + let(:seed) { 1234 } -Randomized with seed 1234 + subject(:output) { `bundle exec turbo_tests -f d #{fixture} --seed #{seed}`.strip } + context "errors outside of examples" do + let(:expected_start_of_output) { +%( +1 processes for 1 specs, ~ 1 specs per process + +Randomized with seed #{seed} An error occurred while loading #{fixture}. \e[31mFailure/Error: \e[0m\e[1;34m1\e[0m / \e[1;34m0\e[0m\e[0m @@ -20,61 +24,98 @@ \e[36m# #{fixture}:4:in `block in '\e[0m \e[36m# #{fixture}:1:in `'\e[0m ).strip - } - let(:expected_end_of_output) do - "0 examples, 0 failures\n"\ - "\n\n"\ - "Randomized with seed 1234" + } + + let(:expected_end_of_output) do + "0 examples, 0 failures, 1 error occurred outside of examples\n"\ + "\n"\ + "Randomized with seed #{seed}" + end + + let(:fixture) { "./fixtures/rspec/errors_outside_of_examples_spec.rb" } + + it "reports" do + expect($?.exitstatus).to eql(1) + + expect(output).to start_with(expected_start_of_output) + expect(output).to end_with(expected_end_of_output) + end end - let(:fixture) { "./fixtures/rspec/errors_outside_of_examples_spec.rb" } + context "pending exceptions", :aggregate_failures do + let(:fixture) { "./fixtures/rspec/pending_exceptions_spec.rb" } - it "reports" do - expect($?.exitstatus).to eql(1) + it "reports" do + expect($?.exitstatus).to eql(0) - expect(output).to start_with(expected_start_of_output) - expect(output).to end_with(expected_end_of_output) + [ + "is implemented but skipped with 'pending' (PENDING: TODO: skipped with 'pending')", + "is implemented but skipped with 'skip' (PENDING: TODO: skipped with 'skip')", + "is implemented but skipped with 'xit' (PENDING: Temporarily skipped with xit)", + + "Pending: (Failures listed here are expected and do not affect your suite's status)", + ].each do |part| + expect(output).to include(part) + end + + expect(output).to end_with("3 examples, 0 failures, 3 pending\n\nRandomized with seed #{seed}") + end end end - context "pending exceptions", :aggregate_failures do - let(:fixture) { "./fixtures/rspec/pending_exceptions_spec.rb" } + context "when 'seed' parameter was not used" do + context "errors outside of examples" do + let(:expected_start_of_output) { +%( +1 processes for 1 specs, ~ 1 specs per process + +An error occurred while loading #{fixture}. +\e[31mFailure/Error: \e[0m\e[1;34m1\e[0m / \e[1;34m0\e[0m\e[0m +\e[31m\e[0m +\e[31mZeroDivisionError:\e[0m +\e[31m divided by 0\e[0m +\e[36m# #{fixture}:4:in `/'\e[0m +\e[36m# #{fixture}:4:in `block in '\e[0m +\e[36m# #{fixture}:1:in `'\e[0m +).strip + } + + let(:expected_end_of_output) do + "0 examples, 0 failures, 1 error occurred outside of examples" + end - it "reports" do - expect($?.exitstatus).to eql(0) + let(:fixture) { "./fixtures/rspec/errors_outside_of_examples_spec.rb" } - [ - "is implemented but skipped with 'pending' (PENDING: TODO: skipped with 'pending')", - "is implemented but skipped with 'skip' (PENDING: TODO: skipped with 'skip')", - "is implemented but skipped with 'xit' (PENDING: Temporarily skipped with xit)", + it "reports" do + expect($?.exitstatus).to eql(1) - "Pending: (Failures listed here are expected and do not affect your suite's status)", + expect(output).to start_with(expected_start_of_output) + expect(output).to end_with(expected_end_of_output) + end - %{ -Fixture of spec file with pending failed examples is implemented but skipped with 'pending' - # TODO: skipped with 'pending' - Failure/Error: DEFAULT_FAILURE_NOTIFIER = lambda { |failure, _opts| raise failure } + it "exludes the seed message from the output" do + expect(output).to_not include("seed") + end + end - expected: 3 - got: 2 + context "pending exceptions", :aggregate_failures do + let(:fixture) { "./fixtures/rspec/pending_exceptions_spec.rb" } - (compared using ==) - }.strip, + it "reports" do + expect($?.exitstatus).to eql(0) - %( -Fixture of spec file with pending failed examples is implemented but skipped with 'skip' - # TODO: skipped with 'skip' - ).strip, + [ + "is implemented but skipped with 'pending' (PENDING: TODO: skipped with 'pending')", + "is implemented but skipped with 'skip' (PENDING: TODO: skipped with 'skip')", + "is implemented but skipped with 'xit' (PENDING: Temporarily skipped with xit)", - %( -Fixture of spec file with pending failed examples is implemented but skipped with 'xit' - # Temporarily skipped with xit - ).strip - ].each do |part| - expect(output).to include(part) - end + "Pending: (Failures listed here are expected and do not affect your suite's status)", + ].each do |part| + expect(output).to include(part) + end - expect(output).to end_with("3 examples, 0 failures, 3 pending\n\n\nRandomized with seed 1234") + expect(output).to end_with("3 examples, 0 failures, 3 pending") + end end end @@ -87,4 +128,21 @@ expect(output).to include("Test info in extra_failure_lines") end end + + describe "full error failure message and line" do + let(:fixture) { "./fixtures/rspec/no_method_error_spec.rb" } + + it "outputs file name and line number" do + expect($?.exitstatus).to eql(1) + + [ + "undefined method `[]' for nil", + 'it("fails") { expect(nil[:key]).to eql("value") }', + "# #{fixture}:2:in `block (2 levels) in '", + "1 example, 1 failure", + ].each do |part| + expect(output).to include(part) + end + end + end end