Load files from the s3 API

Author: jcoyne

.github/workflows/ruby.yml

@@ -19,8 +19,16 @@ jobs:
         with:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: true
+      - name: Start MinIO
+        run: docker compose up -d minio
+      - name: Wait for MinIO to be ready
+        run: |
+          timeout 60 bash -c 'until curl -f http://localhost:9000/minio/health/ready; do sleep 2; done'
       - name: Run tests
         run: bin/rake test:prepare spec
+      - name: Stop MinIO
+        if: always()
+        run: docker compose down
   lint:
     runs-on: ubuntu-latest
     steps:

Gemfile

@@ -82,3 +82,5 @@ gem 'connection_pool'
 group :production do
   gem 'newrelic_rpm'
 end
+
+gem "aws-sdk-s3", "~> 1.199"

Gemfile.lock

@@ -77,6 +77,25 @@ GEM
     airbrussh (1.5.3)
       sshkit (>= 1.6.1, != 1.7.0)
     ast (2.4.3)
+    aws-eventstream (1.4.0)
+    aws-partitions (1.1172.0)
+    aws-sdk-core (3.233.0)
+      aws-eventstream (~> 1, >= 1.3.0)
+      aws-partitions (~> 1, >= 1.992.0)
+      aws-sigv4 (~> 1.9)
+      base64
+      bigdecimal
+      jmespath (~> 1, >= 1.6.1)
+      logger
+    aws-sdk-kms (1.113.0)
+      aws-sdk-core (~> 3, >= 3.231.0)
+      aws-sigv4 (~> 1.5)
+    aws-sdk-s3 (1.199.1)
+      aws-sdk-core (~> 3, >= 3.231.0)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.5)
+    aws-sigv4 (1.12.1)
+      aws-eventstream (~> 1, >= 1.0.2)
     base64 (0.3.0)
     bcrypt_pbkdf (1.1.1)
     bcrypt_pbkdf (1.1.1-arm64-darwin)
@@ -186,6 +205,7 @@ GEM
     jbuilder (2.14.1)
       actionview (>= 7.0.0)
       activesupport (>= 7.0.0)
+    jmespath (1.6.2)
     json (2.15.1)
     jwt (3.1.2)
       base64
@@ -422,6 +442,7 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  aws-sdk-s3 (~> 1.199)
   bootsnap (>= 1.4.2)
   cancancan
   capistrano (~> 3.0)

app/controllers/file_controller.rb

@@ -3,6 +3,8 @@
 ##
 # API for delivering files from stacks
 class FileController < ApplicationController
+  include ActionController::Live
+
   rescue_from ActionController::MissingFile do
     render plain: 'File not found', status: :not_found
   end
@@ -24,7 +26,16 @@ def show
       ip: request.remote_ip
     )
 
-    send_file current_file.path, filename: current_file.file_name, disposition:
+    response.headers['Content-Length'] = current_file.content_length
+    send_stream(
+      filename: current_file.file_name, # Sets the filename for the download
+      type: current_file.content_type, # Sets the content type
+      disposition:
+    ) do |stream|
+      current_file.s3_object do |chunk|
+        stream.write(chunk)
+      end
+    end
   end
   # rubocop:enable Metrics/AbcSize
 

app/controllers/object_controller.rb

@@ -19,7 +19,9 @@ def show
     zip_kit_stream(filename: "#{druid}.zip") do |zip|
       files.each do |stacks_file|
         zip.write_file(stacks_file.file_name, modification_time: stacks_file.mtime) do |sink|
-          File.open(stacks_file.path, "rb") { |file_input| IO.copy_stream(file_input, sink) }
+          stacks_file.s3_object do |chunk|
+            sink.write(chunk)
+          end
         end
       end
     end

app/models/s3_client_factory.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# Creates an S3 client instance.
+class S3ClientFactory
+  def self.create_client
+    Aws::S3::Client.new(
+      region: 'us-east-1',
+      endpoint: Settings.s3.endpoint,
+      force_path_style: true, # Often required for custom endpoints
+      access_key_id: Settings.s3.access_key_id,
+      secret_access_key: Settings.s3.secret_access_key
+    )
+  end
+end

app/models/stacks_file.rb

@@ -24,23 +24,30 @@ def id
 
   # Some files exist but have unreadable permissions, treat these as non-existent
   def readable?
-    path && File.world_readable?(path)
+    true
+    # path && File.world_readable?(path)
+  end
+
+  def s3_object(&)
+    @s3_object ||= S3ClientFactory.create_client.get_object(bucket: Settings.s3.bucket, key: s3_key, &)
+  rescue Aws::S3::Errors::NoSuchKey
+    raise "Unable to find file at #{s3_key}"
   end
 
   def mtime
-    @mtime ||= File.mtime(path) if readable?
+    @mtime ||= s3_object.last_modified
   end
 
   def etag
     mtime&.to_i
   end
 
   def content_length
-    @content_length ||= File.size(path) if readable?
+    cocina_file['size']
   end
 
-  def path
-    @path ||= storage_root.absolute_path
+  def content_type
+    cocina_file['hasMimeType']
   end
 
   # Used as the IIIF identifier for retrieving this file from the image server
@@ -61,10 +68,6 @@ def wowza_identifier
     "#{File.dirname(file_path)}/#{streaming_url_file_segment}"
   end
 
-  def storage_root
-    @storage_root ||= StorageRoot.new(cocina:, file_name:)
-  end
-
   def stacks_rights
     @stacks_rights ||= StacksRights.new(cocina:, file_name:)
   end
@@ -76,4 +79,18 @@ def streamable?
     accepted_formats = [".mov", ".mp4", ".mpeg", ".m4a", ".mp3"]
     accepted_formats.include? File.extname(file_name)
   end
+
+  private
+
+  def cocina_file
+    @cocina_file ||= cocina.find_file(file_name)
+  end
+
+  def s3_key
+    @s3_key ||= storage_root.relative_path
+  end
+
+  def storage_root
+    @storage_root ||= StorageRoot.new(cocina:, file_name:)
+  end
 end

app/models/storage_root.rb

@@ -12,19 +12,15 @@ def initialize(file_name:, cocina:)
   end
 
   def relative_path
-    @relative_path = File.join(treeified_id, druid, 'content', @md5)
-  end
-
-  def absolute_path
-    @absolute_path ||= File.join(Settings.stacks.storage_root, relative_path)
+    File.join(treeified_id, druid, 'content', @md5)
   end
 
   private
 
   attr_reader :druid
 
   def druid_parts
-    @druid_parts ||= druid.match(DRUID_PARTS_PATTERN)
+    druid.match(DRUID_PARTS_PATTERN)
   end
 
   def treeified_id

compose.yaml

@@ -1,13 +1,23 @@
-version: '3.6'
-
 services:
   web:
     build: .
     ports:
       - 3001:3000
     environment:
-      SECRET_KEY_BASE: 'e55fb68eb0f1f4f82fbb56eafeebf1c134518fd466d1c3d01b2bfd3d6b5cd9347b4c15eecb3bdc0e4c5bc226dba626a23bb7b9ddb14f709638571f910002f639'
-      RAILS_LOG_TO_STDOUT: 'true'
+      SECRET_KEY_BASE: "e55fb68eb0f1f4f82fbb56eafeebf1c134518fd466d1c3d01b2bfd3d6b5cd9347b4c15eecb3bdc0e4c5bc226dba626a23bb7b9ddb14f709638571f910002f639"
+      RAILS_LOG_TO_STDOUT: "true"
       REMOTE_USER: blalbrit@stanford.edu
-      SETTINGS__CORS__ALLOW_ORIGIN_URL: 'http://localhost:3000'
-    tty: true
+      SETTINGS__CORS__ALLOW_ORIGIN_URL: "http://localhost:3000"
+    tty: true
+  minio:
+    image: minio/minio
+    environment:
+      MINIO_ROOT_USER: minio-user
+      MINIO_ROOT_PASSWORD: minio-password
+      MINIO_UPDATE: off
+    command: server /data --json --console-address ':9090'
+    ports: ["9000:9000/tcp", "9090:9090/tcp"] # open http://127.0.0.1:9090 (9000 is the API port)
+    volumes:
+      - minio_data:/data
+volumes:
+  minio_data:

config/settings.yml

@@ -4,6 +4,12 @@ features:
 stacks:
   storage_root: /stacks
 
+s3:
+  access_key_id: ~
+  secret_access_key: ~
+  bucket: stacks-test
+  endpoint: https://sul-weka-s3.stanford.edu
+
 imageserver:
   base_uri: "http://imageserver-prod.stanford.edu/iiif/2/"