Merge pull request #339 from wordpress-mobile/add/s3-binary-upload

Add Upload to S3 Action

Author: jkmassel

.rubocop_todo.yml

@@ -126,28 +126,7 @@ RSpec/DescribedClass:
 # Include: **/*_spec*rb*, **/spec/**/*
 RSpec/FilePath:
   Exclude:
-    - 'spec/android_localize_helper_spec.rb'
-    - 'spec/android_merge_translators_strings_spec.rb'
-    - 'spec/android_version_helper_spec.rb'
-    - 'spec/an_localize_libs_action_spec.rb'
-    - 'spec/an_metadata_update_helper_spec.rb'
-    - 'spec/an_update_metadata_source_spec.rb'
-    - 'spec/configuration_spec.rb'
-    - 'spec/configure_helper_spec.rb'
-    - 'spec/encryption_helper_spec.rb'
-    - 'spec/git_helper_spec.rb'
-    - 'spec/github_helper_spec.rb'
-    - 'spec/ios_git_helper_spec.rb'
-    - 'spec/ios_lint_localizations_spec.rb'
-    - 'spec/ios_merge_translators_strings_spec.rb'
-    - 'spec/release_notes_helper_spec.rb'
-    - 'spec/check_localization_progress_spec.rb'
-    - 'spec/ios_generate_strings_file_from_code_spec.rb'
-    - 'spec/ios_l10n_helper_spec.rb'
-    - 'spec/ios_merge_strings_files_spec.rb'
-    - 'spec/gp_update_metadata_source_spec.rb'
-    - 'spec/ios_download_strings_files_from_glotpress_spec.rb'
-    - 'spec/ios_extract_keys_from_strings_files_spec.rb'
+    - 'spec/*_spec.rb'
 
 # Offense count: 8
 # Cop supports --auto-correct.

CHANGELOG.md

@@ -11,6 +11,7 @@ _None_
 ### New Features
 
 * Introduce new `ios_extract_keys_from_strings_files` action. [#338]
+* Add Upload to S3 Action. [#339]
 
 ### Bug Fixes
 

lib/fastlane/plugin/wpmreleasetoolkit/actions/common/upload_to_s3.rb

@@ -0,0 +1,112 @@
+require 'fastlane/action'
+require 'digest/sha1'
+
+module Fastlane
+  module Actions
+    module SharedValues
+      S3_UPLOADED_FILE_PATH = :S3_UPLOADED_FILE_PATH
+    end
+
+    class UploadToS3Action < Action
+      def self.run(params)
+        file_path = params[:file]
+        file_name = File.basename(file_path)
+
+        bucket = params[:bucket]
+        key = params[:key] || file_name
+
+        if params[:auto_prefix] == true
+          file_name_hash = Digest::SHA1.hexdigest(file_name)
+          key = [file_name_hash, key].join('/')
+        end
+
+        UI.user_error!("File already exists in S3 bucket #{bucket} at #{key}") if file_is_already_uploaded?(bucket, key)
+
+        UI.message("Uploading #{file_path} to: #{key}")
+
+        File.open(file_path, 'rb') do |file|
+          Aws::S3::Client.new().put_object(
+            body: file,
+            bucket: bucket,
+            key: key
+          )
+        rescue Aws::S3::Errors::ServiceError => e
+          UI.crash!("Unable to upload file to S3: #{e.message}")
+        end
+
+        UI.success('Upload Complete')
+
+        Actions.lane_context[SharedValues::S3_UPLOADED_FILE_PATH] = key
+
+        return key
+      end
+
+      def self.file_is_already_uploaded?(bucket, key)
+        response = Aws::S3::Client.new().head_object(
+          bucket: bucket,
+          key: key
+        )
+        return response[:content_length].positive?
+      rescue Aws::S3::Errors::NotFound
+        return false
+      end
+
+      def self.description
+        'Uploads a given file to S3'
+      end
+
+      def self.authors
+        ['Automattic']
+      end
+
+      def self.return_value
+        'Returns the object\'s derived S3 key'
+      end
+
+      def self.details
+        'Uploads a file to S3, and makes a pre-signed URL available in the lane context'
+      end
+
+      def self.available_options
+        [
+          FastlaneCore::ConfigItem.new(
+            key: :bucket,
+            description: 'The bucket that will store the file',
+            optional: false,
+            type: String,
+            verify_block: proc { |bucket| UI.user_error!('You must provide a valid bucket name') if bucket.empty? }
+          ),
+          FastlaneCore::ConfigItem.new(
+            key: :key,
+            description: 'The path to the file within the bucket. If `nil`, will default to the `file\'s basename',
+            optional: true,
+            type: String,
+            verify_block: proc { |key|
+              next if key.is_a?(String) && !key.empty?
+
+              UI.user_error!('The provided key must not be empty. Use nil instead if you want to default to the file basename')
+            }
+          ),
+          FastlaneCore::ConfigItem.new(
+            key: :file,
+            description: 'The path to the local file on disk',
+            optional: false,
+            type: String,
+            verify_block: proc { |f| UI.user_error!("Path `#{f}` does not exist.") unless File.file?(f) }
+          ),
+          FastlaneCore::ConfigItem.new(
+            key: :auto_prefix,
+            description: 'Generate a derived prefix based on the filename that makes it harder to guess the URL of the uploaded object',
+            optional: true,
+            default_value: true,
+            type: Boolean
+          ),
+        ]
+      end
+
+      def self.is_supported?(platform)
+        true
+      end
+    end
+  end
+end

spec/spec_helper.rb

@@ -32,6 +32,12 @@ def allow_fastlane_action_sh
   allow(FastlaneCore::Helper).to receive(:sh_enabled?).and_return(true)
 end
 
+# Allow us to do `.with` matching against a `File` instance to a particular path in RSpec expectations
+# (Because `File.open(path)` returns a different instance of `File` for the same path on each call)
+RSpec::Matchers.define :file_instance_of do |path|
+  match { |actual| actual.is_a?(File) && actual.path == path }
+end
+
 # Allows to assert if an `Action.sh` command has been triggered by the action under test.
 # Requires `allow_fastlane_action_sh` to have been called so that `Action.sh` actually calls `Open3.popen2e`
 #
@@ -75,3 +81,16 @@ def in_tmp_dir
     end
   end
 end
+
+# Executes the given block with a temporary file with the given `file_name`
+def with_tmp_file(named: nil, content: '')
+  in_tmp_dir do |tmp_dir|
+    file_name = named || ('a'..'z').to_a.sample(8).join # 8-character random file name if nil
+    file_path = File.join(tmp_dir, file_name)
+
+    File.write(file_path, content)
+    yield file_path
+  ensure
+    File.delete(file_path)
+  end
+end

spec/upload_to_s3_spec.rb

@@ -0,0 +1,162 @@
+require_relative './spec_helper'
+
+describe Fastlane::Actions::UploadToS3Action do
+  let(:client) { instance_double(Aws::S3::Client) }
+  let(:test_bucket) { 'a8c-wpmrt-unit-tests-bucket' }
+
+  before do
+    allow(Aws::S3::Client).to receive(:new).and_return(client)
+  end
+
+  # Stub head_object to return a specific content_length
+  def stub_s3_response_for_file(key, exists: true)
+    content_length = exists == true ? 1 : 0
+    allow(client).to(receive(:head_object))
+                 .with(bucket: test_bucket, key: key)
+                 .and_return(Aws::S3::Types::HeadObjectOutput.new(content_length: content_length))
+  end
+
+  describe 'uploading a file with valid parameters' do
+    it 'generates a prefix for the key by default' do
+      expected_key = '939c39398db2405e791e205778ff70f85dff620e/a8c-key1'
+      stub_s3_response_for_file(expected_key, exists: false)
+
+      with_tmp_file(named: 'input_file_1') do |file_path|
+        expect(client).to receive(:put_object).with(body: file_instance_of(file_path), bucket: test_bucket, key: expected_key)
+
+        return_value = run_described_fastlane_action(
+          bucket: test_bucket,
+          key: 'a8c-key1',
+          file: file_path
+        )
+
+        expect(return_value).to eq(expected_key)
+        expect(Fastlane::Actions.lane_context[Fastlane::Actions::SharedValues::S3_UPLOADED_FILE_PATH]).to eq(expected_key)
+      end
+    end
+
+    it 'generates a prefix for the key when using auto_prefix:true' do
+      expected_key = '8bde1a7a04300df27b52f4383dc997e5fbbff180/a8c-key2'
+      stub_s3_response_for_file(expected_key, exists: false)
+
+      with_tmp_file(named: 'input_file_2') do |file_path|
+        expect(client).to receive(:put_object).with(body: file_instance_of(file_path), bucket: test_bucket, key: expected_key)
+
+        return_value = run_described_fastlane_action(
+          bucket: test_bucket,
+          key: 'a8c-key2',
+          file: file_path,
+          auto_prefix: true
+        )
+
+        expect(return_value).to eq(expected_key)
+        expect(Fastlane::Actions.lane_context[Fastlane::Actions::SharedValues::S3_UPLOADED_FILE_PATH]).to eq(expected_key)
+      end
+    end
+
+    it 'uses the provided key verbatim when using auto_prefix:false' do
+      expected_key = 'a8c-key1'
+      stub_s3_response_for_file(expected_key, exists: false)
+
+      with_tmp_file do |file_path|
+        expect(client).to receive(:put_object).with(body: file_instance_of(file_path), bucket: test_bucket, key: expected_key)
+
+        return_value = run_described_fastlane_action(
+          bucket: test_bucket,
+          key: 'a8c-key1',
+          file: file_path,
+          auto_prefix: false
+        )
+
+        expect(return_value).to eq(expected_key)
+        expect(Fastlane::Actions.lane_context[Fastlane::Actions::SharedValues::S3_UPLOADED_FILE_PATH]).to eq(expected_key)
+      end
+    end
+
+    it 'correctly appends the key if it contains subdirectories' do
+      expected_key = '939c39398db2405e791e205778ff70f85dff620e/subdir/a8c-key1'
+      stub_s3_response_for_file(expected_key, exists: false)
+
+      with_tmp_file(named: 'input_file_1') do |file_path|
+        expect(client).to receive(:put_object).with(body: file_instance_of(file_path), bucket: test_bucket, key: expected_key)
+
+        return_value = run_described_fastlane_action(
+          bucket: test_bucket,
+          key: 'subdir/a8c-key1',
+          file: file_path
+        )
+
+        expect(return_value).to eq(expected_key)
+        expect(Fastlane::Actions.lane_context[Fastlane::Actions::SharedValues::S3_UPLOADED_FILE_PATH]).to eq(expected_key)
+      end
+    end
+
+    it 'uses the filename as the key if one is not provided' do
+      expected_key = 'c125bd799c6aad31092b02e440a8fae25b45a2ad/test_file_1'
+      stub_s3_response_for_file(expected_key, exists: false)
+
+      with_tmp_file(named: 'test_file_1') do |file_path|
+        expect(client).to receive(:put_object).with(body: file_instance_of(file_path), bucket: test_bucket, key: expected_key)
+
+        return_value = run_described_fastlane_action(
+          bucket: test_bucket,
+          file: file_path
+        )
+
+        expect(return_value).to eq(expected_key)
+        expect(Fastlane::Actions.lane_context[Fastlane::Actions::SharedValues::S3_UPLOADED_FILE_PATH]).to eq(expected_key)
+      end
+    end
+  end
+
+  describe 'uploading a file with invalid parameters' do
+    it 'fails if bucket is empty or nil' do
+      expect do
+        with_tmp_file do |file_path|
+          run_described_fastlane_action(
+            bucket: '',
+            key: 'key',
+            file: file_path
+          )
+        end
+      end.to raise_error(FastlaneCore::Interface::FastlaneError, 'You must provide a valid bucket name')
+    end
+
+    it 'fails if an empty key is provided' do
+      expect do
+        with_tmp_file do |file_path|
+          run_described_fastlane_action(
+            bucket: test_bucket,
+            key: '',
+            file: file_path
+          )
+        end
+      end.to raise_error(FastlaneCore::Interface::FastlaneError, 'The provided key must not be empty. Use nil instead if you want to default to the file basename')
+    end
+
+    it 'fails if local file does not exist' do
+      expect do
+        run_described_fastlane_action(
+          bucket: test_bucket,
+          key: 'key',
+          file: 'this-file-does-not-exist.txt'
+        )
+      end.to raise_error(FastlaneCore::Interface::FastlaneError, 'Path `this-file-does-not-exist.txt` does not exist.')
+    end
+
+    it 'fails if the file already exists on S3' do
+      expected_key = 'a62f2225bf70bfaccbc7f1ef2a397836717377de/key'
+      stub_s3_response_for_file(expected_key)
+
+      with_tmp_file(named: 'key') do |file_path|
+        expect do
+          run_described_fastlane_action(
+            bucket: test_bucket,
+            key: 'key',
+            file: file_path
+          )
+        end.to raise_error(FastlaneCore::Interface::FastlaneError, "File already exists in S3 bucket #{test_bucket} at #{expected_key}")
+      end
+    end
+  end
+end