Benchmark refactor (#3074)

Author: mullermp

.github/workflows/benchmark.yml

@@ -17,13 +17,15 @@ jobs:
   benchmark:
     runs-on: ubuntu-latest
     env:
+      GH_EVENT: ${{github.event_name}}
       GH_REF: ${{github.head_ref}}
       GH_REPO: ${{github.repository}}
-      GH_EVENT: ${{github.event_name}}
+      EXECUTION_ENV: github-action
     strategy:
       fail-fast: false
       matrix:
-        ruby: [jruby-9.2, 2.6, 2.7, 3.2, 3.3]
+        # Supported ruby versions (except 2.5 - benchmarks missing)
+        ruby: [2.6, 2.7, '3.0', 3.1, 3.2, 3.3, jruby-9.3, jruby-9.4]
 
     steps:
       - name: Setup Ruby
@@ -32,29 +34,31 @@ jobs:
           ruby-version: ${{ matrix.ruby }}
 
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - name: Install gems
         run: |
-          bundle config set --local without 'docs repl development'
+          bundle config set --local without 'development docs rbs repl'
           bundle install
 
-      - name: SDK Build
+      - name: Build SDK
         run: bundle exec rake build
 
-      - name: Benchmark
+      - name: Benchmark gems
         env:
           JRUBY_OPTS: -J-Xmx4g
-        run: EXECUTION_ENV=github-action bundle exec rake benchmark
+        run: bundle exec rake benchmark:run
 
-      - name: configure aws credentials
+      - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
-          role-to-assume: arn:aws:iam::469596866844:role/aws-sdk-ruby-performance-reporter
-          role-session-name: benchmark-report
-          aws-region: us-west-2
+          role-to-assume: arn:aws:iam::373952703873:role/BenchmarkReporter
+          role-session-name: benchmark-reporter
+          aws-region: us-east-1
 
-      - name: Archive benchmark report
-        run: bundle exec rake benchmark:archive
+      - name: Upload benchmark report
+        run: bundle exec rake benchmark:upload-report
 
-      - name: Upload benchmark metrics
+      - name: Put benchmark metrics
         run: bundle exec rake benchmark:put-metrics

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ permissions:
   contents: read
 
 jobs:
-  sig:
+  rbs-test:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -30,20 +30,21 @@ jobs:
 
       - name: Install gems
         run: |
-          bundle config set --local with 'build signature'
+          bundle config set --local with 'build rbs'
           bundle install
 
-      - name: SDK Build
+      - name: Build SDK
         run: bundle exec rake build
 
-      - name: rbs testing
+      - name: RBS Test
         run: bundle exec rake rbs:test
 
   test:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
+        # Supported ruby versions
         ruby: [2.5, 2.6, 2.7, '3.0', 3.1, 3.2, 3.3, jruby-9.2, jruby-9.3, jruby-9.4]
         env: [KITCHEN_SINK, CRT]
 
@@ -55,19 +56,19 @@ jobs:
 
       - uses: actions/checkout@v4
 
-      - name: Setup environment
+      - name: Setup ENV
         run: |
           echo "${{ matrix.env }}=1" >> $GITHUB_ENV
 
       - name: Install gems
         run: |
-          bundle config set --local without 'docs repl development'
+          bundle config set --local with 'build test'
           bundle install
 
-      - name: SDK Build
+      - name: Build SDK
         run: bundle exec rake build
 
-      - name: Tests
+      - name: Test
         env:
           JRUBY_OPTS: -J-Xmx4g
         run: bundle exec rake test:spec

.gitignore

@@ -7,6 +7,7 @@
 /Gemfile.lock
 /coverage
 *.gem
+benchmark_report.json
 
 /scratchpad
 .idea

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tasks/benchmark"]
+	path = tasks/benchmark
+	url = git@github.com:aws/aws-sdk-ruby-benchmark-tools.git

Gemfile

@@ -8,40 +8,41 @@ gem 'rake', require: false
 gem 'aws-crt' if ENV['CRT']
 gem 'http-2'
 gem 'jmespath'
+if defined?(JRUBY_VERSION)
+  # get the latest jruby-openssl to support sigv4a
+  # see: https://github.com/jruby/jruby-openssl/issues/30
+  gem 'jruby-openssl'
+end
 
 # protocol parsers
 gem 'json'
 gem 'nokogiri', '>= 1.6.8.1'
 gem 'oga'
 gem 'rexml', '= 3.2.6' # Temporary Workaround (https://github.com/ruby/rexml/issues/131)
-
 # These protocol parsers do not have java gems
 unless defined?(JRUBY_VERSION)
   gem 'libxml-ruby'
   gem 'oj'
   gem 'ox'
 end
 
-if defined?(JRUBY_VERSION)
-  # get the latest jruby-openssl to support sigv4a
-  # see: https://github.com/jruby/jruby-openssl/issues/30
-  gem 'jruby-openssl'
-end
-
-group :test do
-  gem 'addressable'
-  gem 'cucumber'
-  gem 'webmock'
+group :benchmark do
+  gem 'memory_profiler'
 
-  gem 'multipart-post'
-  gem 'rspec'
+  # required for uploading report/putting metrics
+  gem 'aws-sdk-cloudwatch'
+  gem 'aws-sdk-s3'
 end
 
 group :build do
   gem 'kramdown'
   gem 'mustache'
 end
 
+group :development do
+  gem 'rubocop', '1.28.0'
+end
+
 group :docs do
   gem 'rdiscount', platforms: :ruby
   # When updating yard, override app.js from upstream to:
@@ -52,19 +53,19 @@ group :docs do
   gem 'yard-sitemap', '~> 1.0'
 end
 
-group :repl do
-  gem 'pry'
+group :rbs do
+  gem 'rbs', platforms: :ruby
 end
 
-group :development do
-  gem 'rubocop', '1.28.0'
+group :repl do
+  gem 'pry'
 end
 
-group :benchmark do
-  gem 'benchmark'
-  gem 'memory_profiler'
-end
+group :test do
+  gem 'addressable'
+  gem 'cucumber'
+  gem 'webmock'
 
-group :signature do
-  gem 'rbs', platforms: :ruby
+  gem 'multipart-post'
+  gem 'rspec'
 end