Merge pull request #155 from hiroyuki-sato/topic/newer-jruby

First porting to Embulk 0.11.0

Author: joker1007

.github/workflows/check.yml

@@ -7,6 +7,10 @@ jobs:
     # pull_request: run only when the PR is submitted from a forked repository, not within this repository.
     if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
     strategy:
+      matrix:
+        jruby_version:
+        - 9.3.10.0
+        - 9.4.2.0
       fail-fast: false
     steps:
     - uses: actions/checkout@v3
@@ -17,17 +21,13 @@ jobs:
         distribution: "temurin"
     - uses: ruby/setup-ruby@v1
       with:
-        ruby-version: 'jruby-9.1.17.0' # 9.1.15.0 doesn't available in setup-ruby@v1
+        ruby-version: 'jruby-${{ matrix.jruby_version }}'
         bundler-cache: true
     - name: show ruby version
       run: ruby -v
-    # Gem::LoadError: You have already activated rake 10.4.2,
-    # but your Gemfile requires rake 13.0.6.
-    - name: gem install rake -v 13.0.6
-      run: gem install rake -v 13.0.6
     - name: bundle install
       run: bundle install
-    - name: install embulk
-      run: "curl -L -o embulk.jar https://github.com/embulk/embulk/releases/download/v0.8.39/embulk-0.8.39.jar"
+    - name: install embulk.jar
+      run: "curl -L -o embulk.jar https://github.com/embulk/embulk/releases/download/v0.10.49/embulk-0.10.49.jar"
     - name: rake test
       run: bundle exec env RUBYOPT="-r ./embulk.jar" rake test

Gemfile

@@ -1,8 +1,7 @@
 source 'https://rubygems.org/'
 
 gemspec
-gem 'embulk', '< 0.10'
-gem 'liquid', '= 4.0.0' # the version included in embulk.jar
+gem 'embulk', '> 0.10.0'
 gem 'embulk-parser-none'
 gem 'embulk-parser-jsonl'
 gem 'pry-nav'

embulk-output-bigquery.gemspec

@@ -14,18 +14,13 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec)/})
   spec.require_paths = ["lib"]
 
-  # TODO
-  # signet 0.12.0 and google-api-client 0.33.0 require >= Ruby 2.4.
-  # Embulk 0.9 use JRuby 9.1.X.Y and it's compatible with Ruby 2.3.
-  # So, force install signet < 0.12 and google-api-client < 0.33.0
-  # Also, representable version >= 3.1.0 requires Ruby version >= 2.4
-  spec.add_dependency 'signet', '~> 0.7', '< 0.12.0'
-  spec.add_dependency 'google-api-client','< 0.33.0'
+  # the latest version
+  spec.add_dependency 'google-api-client','= 0.53.0'
   spec.add_dependency 'time_with_zone'
-  spec.add_dependency "representable", ['~> 3.0.0', '< 3.1']
-  # faraday 1.1.0 require >= Ruby 2.4.
-  # googleauth 0.9.0 requires faraday ~> 0.12
-  spec.add_dependency "faraday", '~> 0.12'
+  spec.add_dependency 'thwait'
+  # activesupport require Ruby >= 2.7.0
+  # jruby-9.3.0.0 is MRI 2.6 compatible
+  spec.add_dependency 'activesupport', "< 7.0"
 
   spec.add_development_dependency 'bundler', ['>= 1.10.6']
   spec.add_development_dependency 'rake', ['>= 10.0']

lib/embulk/output/bigquery/bigquery_client.rb

@@ -121,7 +121,7 @@ def load_from_gcs(object_uris, table)
               opts = {}
 
               Embulk.logger.debug { "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts})" }
-              response = with_network_retry { client.insert_job(@project, body, opts) }
+              response = with_network_retry { client.insert_job(@project, body, **opts) }
               unless @task['is_skip_job_result_check']
                 response = wait_load('Load', response)
               end
@@ -222,7 +222,7 @@ def load(path, table, write_disposition: 'WRITE_APPEND')
                 # },
               }
               Embulk.logger.debug { "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts})" }
-              response = with_network_retry { client.insert_job(@project, body, opts) }
+              response = with_network_retry { client.insert_job(@project, body, **opts) }
               if @task['is_skip_job_result_check']
                 response
               else
@@ -278,7 +278,7 @@ def copy(source_table, destination_table, destination_dataset = nil, write_dispo
 
               opts = {}
               Embulk.logger.debug { "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts})" }
-              response = with_network_retry { client.insert_job(@project, body, opts) }
+              response = with_network_retry { client.insert_job(@project, body, **opts) }
               wait_load('Copy', response)
             rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
               response = {status_code: e.status_code, message: e.message, error_class: e.class}
@@ -372,7 +372,7 @@ def create_dataset(dataset = nil, reference: nil)
             end
             opts = {}
             Embulk.logger.debug { "embulk-output-bigquery: insert_dataset(#{@project}, #{dataset}, #{@location_for_log}, #{body}, #{opts})" }
-            with_network_retry { client.insert_dataset(@project, body, opts) }
+            with_network_retry { client.insert_dataset(@project, body, **opts) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 409 && /Already Exists:/ =~ e.message
               # ignore 'Already Exists' error
@@ -447,7 +447,7 @@ def create_table_if_not_exists(table, dataset: nil, options: nil)
 
             opts = {}
             Embulk.logger.debug { "embulk-output-bigquery: insert_table(#{@project}, #{dataset}, #{@location_for_log}, #{body}, #{opts})" }
-            with_network_retry { client.insert_table(@project, dataset, body, opts) }
+            with_network_retry { client.insert_table(@project, dataset, body, **opts) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 409 && /Already Exists:/ =~ e.message
               # ignore 'Already Exists' error

test/helper.rb

@@ -4,14 +4,14 @@
 require 'test/unit'
 require 'test/unit/rr'
 
+# Embulk 0.10.x introduced new bootstrap mechanism.
+# https://github.com/embulk/embulk/blob/641f35fec064cca7b1a7314d634a4b64ef8637f1/embulk-ruby/test/vanilla/run-test.rb#L8-L13
+static_initializer = Java::org.embulk.EmbulkDependencyClassLoader.staticInitializer().useSelfContainedJarFiles()
+static_initializer.java_send :initialize
+
+require 'embulk/java/bootstrap'
 require 'embulk'
-begin
-  # Embulk ~> 0.8.x
-  Embulk.setup
-rescue NotImplementedError, NoMethodError, NameError
-  # Embulk ~> 0.9.x
-  require 'embulk/java/bootstrap'
-end
+
 Embulk.logger = Embulk::Logger.new('/dev/null')
 
 APP_ROOT = File.expand_path('../', __dir__)

test/test_transaction.rb

@@ -109,7 +109,7 @@ def test_replace
           task = Bigquery.configure(config, schema, processor_count)
           any_instance_of(BigqueryClient) do |obj|
             mock(obj).get_dataset(config['dataset'])
-            mock(obj).create_table_if_not_exists(config['temp_table'], {:options=>{"expiration_time"=>nil}})
+            mock(obj).create_table_if_not_exists(config['temp_table'], options: {"expiration_time"=>nil})
             mock(obj).create_table_if_not_exists(config['table'])
             mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_TRUNCATE')
             mock(obj).delete_table(config['temp_table'])
@@ -122,7 +122,7 @@ def test_replace_with_partitioning
           task = Bigquery.configure(config, schema, processor_count)
           any_instance_of(BigqueryClient) do |obj|
             mock(obj).get_dataset(config['dataset'])
-            mock(obj).create_table_if_not_exists(config['temp_table'],{:options=>{"expiration_time"=>nil}})
+            mock(obj).create_table_if_not_exists(config['temp_table'], options: {"expiration_time"=>nil})
             mock(obj).create_table_if_not_exists(config['table'])
             mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_TRUNCATE')
             mock(obj).delete_table(config['temp_table'])
@@ -138,7 +138,7 @@ def test_replace_backup
           any_instance_of(BigqueryClient) do |obj|
             mock(obj).get_dataset(config['dataset'])
             mock(obj).get_dataset(config['dataset_old'])
-            mock(obj).create_table_if_not_exists(config['temp_table'], {:options=>{"expiration_time"=>nil}})
+            mock(obj).create_table_if_not_exists(config['temp_table'], options: {"expiration_time"=>nil})
             mock(obj).create_table_if_not_exists(config['table'])
             mock(obj).create_table_if_not_exists(config['table_old'], dataset: config['dataset_old'])
 
@@ -158,7 +158,7 @@ def test_replace_backup_auto_create_dataset
             mock(obj).create_dataset(config['dataset'])
             mock(obj).create_dataset(config['dataset_old'], reference: config['dataset'])
             mock(obj).create_table_if_not_exists(config['table'])
-            mock(obj).create_table_if_not_exists(config['temp_table'],{:options=>{"expiration_time"=>nil}})
+            mock(obj).create_table_if_not_exists(config['temp_table'], options: {"expiration_time"=>nil})
             mock(obj).create_table_if_not_exists(config['table_old'], dataset: config['dataset_old'])
 
             mock(obj).get_table_or_partition(config['table'])
@@ -176,7 +176,7 @@ def test_replace_backup_with_partitioning
           any_instance_of(BigqueryClient) do |obj|
             mock(obj).get_dataset(config['dataset'])
             mock(obj).get_dataset(config['dataset_old'])
-            mock(obj).create_table_if_not_exists(config['temp_table'],{:options=>{"expiration_time"=>nil}})
+            mock(obj).create_table_if_not_exists(config['temp_table'], options: {"expiration_time"=>nil})
             mock(obj).create_table_if_not_exists(config['table'])
             mock(obj).create_table_if_not_exists(config['table_old'], dataset: config['dataset_old'])
 
@@ -196,7 +196,7 @@ def test_append
           task = Bigquery.configure(config, schema, processor_count)
           any_instance_of(BigqueryClient) do |obj|
             mock(obj).get_dataset(config['dataset'])
-            mock(obj).create_table_if_not_exists(config['temp_table'], {:options=>{"expiration_time"=>nil}})
+            mock(obj).create_table_if_not_exists(config['temp_table'], options: {"expiration_time"=>nil})
             mock(obj).create_table_if_not_exists(config['table'])
             mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_APPEND')
             mock(obj).delete_table(config['temp_table'])
@@ -209,7 +209,7 @@ def test_append_with_partitioning
           task = Bigquery.configure(config, schema, processor_count)
           any_instance_of(BigqueryClient) do |obj|
             mock(obj).get_dataset(config['dataset'])
-            mock(obj).create_table_if_not_exists(config['temp_table'], {:options=>{"expiration_time"=>nil}})
+            mock(obj).create_table_if_not_exists(config['temp_table'], options: {"expiration_time"=>nil})
             mock(obj).create_table_if_not_exists(config['table'])
             mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_APPEND')
             mock(obj).delete_table(config['temp_table'])