Merge pull request #311 from dainst/field_hub_streamed_files

Support arbitrary file sizes in FieldHub

Author: dersmon

server/Dockerfile

@@ -12,8 +12,8 @@
 #   - https://pkgs.org/ - resource for finding needed packages
 #   - Ex: hexpm/elixir:1.13.0-erlang-24.2-debian-bullseye-20210902-slim
 #
-ARG BUILDER_IMAGE="hexpm/elixir:1.13.0-erlang-24.2-debian-bullseye-20210902-slim"
-ARG RUNNER_IMAGE="debian:bullseye-20210902-slim"
+ARG BUILDER_IMAGE="hexpm/elixir:1.18.3-erlang-25.3.2.20-debian-bookworm-20250407-slim"
+ARG RUNNER_IMAGE="debian:bookworm-20250407-slim"
 
 FROM ${BUILDER_IMAGE} as builder
 

server/config/config.exs

@@ -31,7 +31,6 @@ config :field_hub,
   couchdb_user_password: "app_user_password",
   valid_file_variants: [:thumbnail_image, :original_image],
   file_index_cache_name: :file_info,
-  file_max_size: 1_000_000_000,
   user_tokens_cache_name: :user_tokens,
   max_project_identifier_length: 30
 

server/config/test.exs

@@ -23,6 +23,4 @@ config :logger, :console, level: :error
 config :phoenix, :plug_init_mode, :runtime
 
 config :field_hub,
-  file_directory_root: "test/tmp",
-  # ~10mb instead of the 1gb default value
-  file_max_size: 10_000_000
+  file_directory_root: "test/tmp"

server/lib/field_hub/file_store.ex

@@ -179,6 +179,26 @@ defmodule FieldHub.FileStore do
     result
   end
 
+  @doc """
+  Open a io_device for a new file.
+
+  The io_device can then be used to write chunked/streamed data without having to
+  load the whole file into memory at once.
+
+  __Parameters__
+
+  - `uuid` the uuid for the file (will be used as its file_name).
+  - `project_identifier` the project's name.
+  - `file_variant` a valid file variant, one of `#{inspect(@valid_file_variants)}`.
+
+  Returns `{:ok, io_device}` on success or `{:error, posix}` on failure.
+  """
+  def create_write_io_device(uuid, project_identifier, file_variant) do
+    directory = get_variant_directory(project_identifier, file_variant)
+    file_path = "#{directory}/#{uuid}"
+    File.open(file_path, [:write])
+  end
+
   @doc """
   Mark all file variants for a UUID/file as deleted.
 

server/lib/field_hub_web/rest/api/file.ex

@@ -3,7 +3,7 @@ defmodule FieldHubWeb.Rest.Api.Rest.File do
 
   alias FieldHub.FileStore
 
-  @max_size Application.compile_env(:field_hub, :file_max_size)
+  require Logger
 
   def index(conn, %{"project" => project, "types" => types}) when is_list(types) do
     parsed_types =
@@ -89,29 +89,47 @@ defmodule FieldHubWeb.Rest.Api.Rest.File do
   end
 
   def update(conn, %{"project" => project, "id" => uuid, "type" => type}) when is_binary(type) do
-    parsed_type = parse_type(type)
+    with {:parsed_type, parsed_type} when is_atom(parsed_type) <-
+           {:parsed_type, parse_type(type)},
+         {:io_opened, {:ok, io_device}} <-
+           {:io_opened, FileStore.create_write_io_device(uuid, project, parsed_type)},
+         {:stream, {:ok, %Plug.Conn{} = conn}} <- {:stream, stream_body(conn, io_device)} do
+      FileStore.clear_cache(project)
+
+      send_resp(conn, 201, Jason.encode!(%{info: "File created."}))
+    else
+      {:parsed_type, {:error, type}} ->
+        send_resp(conn, 400, Jason.encode!(%{reason: "Unknown file type: #{type}"}))
+
+      {:io_opened, posix} ->
+        Logger.error(
+          "Got `#{posix}` while trying to open file `#{uuid}` (#{type}) for project `#{project}`."
+        )
+
+        send_resp(conn, 500, Jason.encode!(%{reason: "Unable to write file."}))
+
+      {:stream, {:error, term}} ->
+        Logger.error(
+          "Got `#{term}` error while trying to stream request body for `#{uuid}` (#{type}) for project `#{project}`."
+        )
+
+        send_resp(conn, 500, Jason.encode!(%{reason: "Unable to write file."}))
+    end
+  end
 
-    conn
-    |> read_body(length: @max_size)
+  defp stream_body(conn, io_device) do
+    read_body(conn)
     |> case do
       {:ok, data, conn} ->
-        case parsed_type do
-          {:error, type} ->
-            send_resp(conn, 400, Jason.encode!(%{reason: "Unknown file type: #{type}"}))
+        IO.binwrite(io_device, data)
+        {:ok, conn}
 
-          valid ->
-            FileStore.store(Zarex.sanitize(uuid), Zarex.sanitize(project), valid, data)
-            send_resp(conn, 201, Jason.encode!(%{info: "File created."}))
-        end
+      {:more, data, conn} ->
+        IO.binwrite(io_device, data)
+        stream_body(conn, io_device)
 
-      {:more, _partial_body, conn} ->
-        send_resp(
-          conn,
-          413,
-          Jason.encode!(%{
-            reason: "Payload too large, maximum of #{Sizeable.filesize(@max_size)} bytes allowed."
-          })
-        )
+      error ->
+        error
     end
   end
 

server/mix.exs

@@ -4,7 +4,7 @@ defmodule FieldHub.MixProject do
   def project do
     [
       app: :field_hub,
-      version: "3.3.2",
+      version: "3.4.0",
       elixir: "~> 1.18",
       elixirc_paths: elixirc_paths(Mix.env()),
       compilers: Mix.compilers(),

server/test/field_hub_web/controllers/api/file_controller_test.exs

@@ -59,17 +59,17 @@ defmodule FieldHubWeb.Api.Rest.FileTest do
     assert conn.status == 400
   end
 
-  test "PUT /files/:project/:uuid with file exceeding maximum size throws 413", %{conn: conn} do
-    roughly_20_mb = String.duplicate("0123456789", 2_000_000)
+  # test "PUT /files/:project/:uuid with file exceeding maximum size throws 413", %{conn: conn} do
+  #   roughly_20_mb = String.duplicate("0123456789", 2_000_000)
 
-    conn =
-      conn
-      |> put_req_header("authorization", @basic_auth)
-      |> put_req_header("content-type", "image/png")
-      |> put("/files/#{@project}/1234?type=original_image", roughly_20_mb)
+  #   conn =
+  #     conn
+  #     |> put_req_header("authorization", @basic_auth)
+  #     |> put_req_header("content-type", "image/png")
+  #     |> put("/files/#{@project}/1234?type=original_image", roughly_20_mb)
 
-    assert conn.status == 413
-  end
+  #   assert conn.status == 413
+  # end
 
   test "GET /files/:project/:uuid returns 404 for non-existent file", %{conn: conn} do
     credentials = Base.encode64("#{@user_name}:#{@user_password}")