Load files from the s3 API

Author: jcoyne

.github/workflows/ruby.yml

@@ -19,8 +19,16 @@ jobs:
         with:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: true
+      - name: Start MinIO
+        run: docker compose up -d minio
+      - name: Wait for MinIO to be ready
+        run: |
+          timeout 60 bash -c 'until curl -f http://localhost:9000/minio/health/ready; do sleep 2; done'
       - name: Run tests
         run: bin/rake test:prepare spec
+      - name: Stop MinIO
+        if: always()
+        run: docker compose down
   lint:
     runs-on: ubuntu-latest
     steps:

Gemfile

@@ -82,3 +82,5 @@ gem 'connection_pool'
 group :production do
   gem 'newrelic_rpm'
 end
+
+gem "aws-sdk-s3", "~> 1.199"

Gemfile.lock

@@ -77,6 +77,25 @@ GEM
     airbrussh (1.5.3)
       sshkit (>= 1.6.1, != 1.7.0)
     ast (2.4.3)
+    aws-eventstream (1.4.0)
+    aws-partitions (1.1172.0)
+    aws-sdk-core (3.233.0)
+      aws-eventstream (~> 1, >= 1.3.0)
+      aws-partitions (~> 1, >= 1.992.0)
+      aws-sigv4 (~> 1.9)
+      base64
+      bigdecimal
+      jmespath (~> 1, >= 1.6.1)
+      logger
+    aws-sdk-kms (1.113.0)
+      aws-sdk-core (~> 3, >= 3.231.0)
+      aws-sigv4 (~> 1.5)
+    aws-sdk-s3 (1.199.1)
+      aws-sdk-core (~> 3, >= 3.231.0)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.5)
+    aws-sigv4 (1.12.1)
+      aws-eventstream (~> 1, >= 1.0.2)
     base64 (0.3.0)
     bcrypt_pbkdf (1.1.1)
     bcrypt_pbkdf (1.1.1-arm64-darwin)
@@ -186,6 +205,7 @@ GEM
     jbuilder (2.14.1)
       actionview (>= 7.0.0)
       activesupport (>= 7.0.0)
+    jmespath (1.6.2)
     json (2.15.1)
     jwt (3.1.2)
       base64
@@ -422,6 +442,7 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  aws-sdk-s3 (~> 1.199)
   bootsnap (>= 1.4.2)
   cancancan
   capistrano (~> 3.0)

app/controllers/file_controller.rb

@@ -3,6 +3,8 @@
 ##
 # API for delivering files from stacks
 class FileController < ApplicationController
+  include ActionController::Live
+
   rescue_from ActionController::MissingFile do
     render plain: 'File not found', status: :not_found
   end
@@ -24,7 +26,14 @@ def show
       ip: request.remote_ip
     )
 
-    send_file current_file.path, filename: current_file.file_name, disposition:
+    s3_object = current_file.s3_object
+    send_stream(
+      filename: current_file.file_name, # Sets the filename for the download
+      type: s3_object.content_type, # Sets the content type
+      disposition:
+    ) do |stream|
+      IO.copy_stream(s3_object.body, stream)
+    end
   end
   # rubocop:enable Metrics/AbcSize
 

app/models/s3_client_factory.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# Creates an S3 client instance.
+class S3ClientFactory
+  def self.create_client
+    Aws::S3::Client.new(
+      region: 'us-east-1',
+      endpoint: Settings.s3.endpoint,
+      force_path_style: true, # Often required for custom endpoints
+      access_key_id: Settings.s3.access_key_id,
+      secret_access_key: Settings.s3.secret_access_key
+    )
+  end
+end

app/models/stacks_file.rb

@@ -27,6 +27,12 @@ def readable?
     path && File.world_readable?(path)
   end
 
+  def s3_object
+    S3ClientFactory.create_client.get_object(bucket: Settings.s3.bucket, key: storage_root.relative_path)
+  rescue Aws::S3::Errors::NoSuchKey
+    raise "Unable to find file at #{path}"
+  end
+
   def mtime
     @mtime ||= File.mtime(path) if readable?
   end

compose.yaml

@@ -1,13 +1,23 @@
-version: '3.6'
-
 services:
   web:
     build: .
     ports:
       - 3001:3000
     environment:
-      SECRET_KEY_BASE: 'e55fb68eb0f1f4f82fbb56eafeebf1c134518fd466d1c3d01b2bfd3d6b5cd9347b4c15eecb3bdc0e4c5bc226dba626a23bb7b9ddb14f709638571f910002f639'
-      RAILS_LOG_TO_STDOUT: 'true'
+      SECRET_KEY_BASE: "e55fb68eb0f1f4f82fbb56eafeebf1c134518fd466d1c3d01b2bfd3d6b5cd9347b4c15eecb3bdc0e4c5bc226dba626a23bb7b9ddb14f709638571f910002f639"
+      RAILS_LOG_TO_STDOUT: "true"
       REMOTE_USER: blalbrit@stanford.edu
-      SETTINGS__CORS__ALLOW_ORIGIN_URL: 'http://localhost:3000'
-    tty: true
+      SETTINGS__CORS__ALLOW_ORIGIN_URL: "http://localhost:3000"
+    tty: true
+  minio:
+    image: minio/minio
+    environment:
+      MINIO_ROOT_USER: minio-user
+      MINIO_ROOT_PASSWORD: minio-password
+      MINIO_UPDATE: off
+    command: server /data --json --console-address ':9090'
+    ports: ["9000:9000/tcp", "9090:9090/tcp"] # open http://127.0.0.1:9090 (9000 is the API port)
+    volumes:
+      - minio_data:/data
+volumes:
+  minio_data:

config/settings.yml

@@ -4,6 +4,12 @@ features:
 stacks:
   storage_root: /stacks
 
+s3:
+  access_key_id: ~
+  secret_access_key: ~
+  bucket: stacks-test
+  endpoint: https://sul-weka-s3.stanford.edu
+
 imageserver:
   base_uri: "http://imageserver-prod.stanford.edu/iiif/2/"
 

config/settings/test.yml

@@ -15,3 +15,9 @@ metrics_api_url: "https://example.com"
 
 stacks:
   storage_root: spec/fixtures
+
+s3:
+  access_key_id: minio-user
+  secret_access_key: minio-password
+  bucket: stacks-test
+  endpoint: http://localhost:9000

spec/rails_helper.rb

@@ -60,4 +60,26 @@
   # The different available types are documented in the features, such as in
   # https://relishapp.com/rspec/rspec-rails/docs
   config.infer_spec_type_from_file_location!
+
+  config.before(:suite) do
+    s3_client = S3ClientFactory.create_client
+    bucket_name = Settings.s3.bucket
+    begin
+      s3_client.create_bucket(bucket: bucket_name)
+    rescue Aws::S3::Errors::BucketAlreadyOwnedByYou
+      # Bucket already exists, do nothing
+    end
+
+    # Add test files into MinIO
+    object_key = 'bb/000/cr/7262/bb000cr7262/content/8ff299eda08d7c506273840d52a03bf3'
+    file_path = "spec/fixtures/#{object_key}"
+    File.open(file_path, 'rb') do |file|
+      s3_client.put_object(
+        bucket: bucket_name,
+        key: object_key,
+        body: file
+      )
+    end
+  end
 end
+WebMock.disable_net_connect!(allow_localhost: true)