Direct to S3 Uploads

README.md

@@ -495,6 +495,11 @@ config :ex_aws,
 
 > For your host configuration, please examine the approved [AWS Hostnames](http://docs.aws.amazon.com/general/latest/gr/rande.html).  There are often multiple hostname formats for AWS regions, and it will not work unless you specify the correct one.
 
+### Browser Uploads direct to Amazon S3
+
+For many use cases, it is unnecessary to proxy file uploads through the application layer. Certain cloud storage platforms, such as Amazon S3, allow for browsers to upload files directly to S3 to take the load off and bandwidth off of your application layer.
+
+To use direct-to-s3 uploads, please reference: https://github.com/stavro/arc/wiki/Direct-to-S3-Uploads
 
 # Full Example
 

lib/arc/file.ex

@@ -3,19 +3,14 @@ defmodule Arc.File do
 
   def generate_temporary_path(file \\ nil) do
     extension = Path.extname((file && file.path) || "")
-
-    file_name =
-      :crypto.rand_bytes(20)
-      |> Base.encode32()
-      |> Kernel.<>(extension)
-
+    file_name = Arc.UUID.generate() <> extension
     Path.join(System.tmp_dir, file_name)
   end
 
   # Given a remote file
   def new(remote_path = "http" <> _) do
     case save_file(remote_path) do
-      {:ok, local_path} -> %Arc.File{path: local_path, file_name: Path.basename(remote_path)}
+      {:ok, local_path, file_name} -> %Arc.File{path: local_path, file_name: file_name}
       :error -> {:error, :invalid_file_path}
     end
   end
@@ -59,24 +54,36 @@ defmodule Arc.File do
       |> Kernel.<>(Path.extname(remote_path))
 
     case save_temp_file(local_path, remote_path) do
-      :ok -> {:ok, local_path}
+      {:ok, file_name} -> {:ok, local_path, file_name}
       _   -> :error
     end
   end
 
   defp save_temp_file(local_path, remote_path) do
     remote_file = get_remote_path(remote_path)
 
-    case remote_file do
-      {:ok, body} -> File.write(local_path, body)
-      {:error, error} -> {:error, error}
+    with {:ok, body, file_name} <- get_remote_path(remote_path),
+         :ok <- File.write(local_path, body) do
+      {:ok, file_name}
     end
   end
 
   defp get_remote_path(remote_path) do
     case HTTPoison.get(remote_path) do
-      {:ok, %{status_code: 200, body: body}} -> {:ok, body}
+      {:ok, %{status_code: 200, body: body, headers: headers}} ->
+        file_name = resolve_file_name(headers) || Path.basename(remote_path)
+        {:ok, body, file_name}
       other -> {:error, :invalid_file_path}
     end
   end
+
+  defp resolve_file_name(response) do
+    response
+    |> Map.get(:headers)
+    |> Enum.find(fn {k, v} -> String.downcase(to_string(k)) == "content-disposition" end)
+    |> case do
+      {_k, v} -> v
+      _ -> nil
+    end
+  end
 end

lib/arc/storage/s3.ex

@@ -16,20 +16,83 @@ defmodule Arc.Storage.S3 do
   end
 
   def url(definition, version, file_and_scope, options \\ []) do
+    compose_s3_key(definition, version, file_and_scope)
+    |> url(options)
+  end
+
+  def url(s3_key, options \\ []) when is_binary(s3_key) do
     case Keyword.get(options, :signed, false) do
-      false -> build_url(definition, version, file_and_scope, options)
-      true  -> build_signed_url(definition, version, file_and_scope, options)
+      false -> build_url(s3_key, options)
+      true  -> build_signed_url(s3_key, options)
     end
   end
 
   def delete(definition, version, {file, scope}) do
-    bucket
-    |> ExAws.S3.delete_object(s3_key(definition, version, {file, scope}))
+    compose_s3_key(definition, version, {file, scope})
+    |> delete()
+  end
+
+  def delete(s3_key) when is_binary(s3_key) do
+    bucket()
+    |> ExAws.S3.delete_object(s3_key)
     |> ExAws.request()
 
     :ok
   end
 
+  @doc """
+  Generates an upload form capable of submitting a file directly to Amazon S3.
+
+  ## Options
+
+    * `:bucket` - Which bucket the upload will be placed in. Defaults to the
+      Arc bucket.
+    * `:key` - The S3 key (or file path) where the upload will be stored,
+      defaults to `uploads/#{Arc.UUID.generate()}`
+    * `:acl` - The access control policy to apply to the uploaded file. If you
+      do not want the uploaded file to be made available to the general public,
+      you should use the value `private`.  To make the uploaded file publicly
+      available, use the value `public-read`. Defaults to `private`.
+    * `:expires_in` - A value in seconds that specifies how long the policy
+      document will be valid for. Once a policy document has expired, the
+      upload form will no longer work.
+    * `:content_length_range` - Content length range as `[min, max]`, where
+       S3 will check that the size of an uploaded file is between a given
+       minimum and maximum value (in bytes). If this rule is not included in a
+       policy document, users will be able to upload files of any size up to
+       the 5GB limit imposed by S3.
+    * `:content_disposition` - Content header passed through to Amazon S3.  This
+      allows you to specify that the file is an attachment, and what filename
+      the upload should be downloaded as.  Eg. `attachment; filename=image.png`
+    * `:content_type`- The content type (mime type) that will be applied to the
+      uploaded file, for example image/jpeg for JPEG picture files. If you do
+      not know what type of file a user will upload, you can either let the
+      user choose the file prior to generating this upload form to determine
+      the appropriate content type.  If you do not set the content type with
+      this field, S3 will use the default value application/octet-stream which
+      may prevent some web browsers from being able to display the file
+      properly.
+    * `:success_action_redirect` - The URL address to which the user’s web
+      browser will be redirected after the file is uploaded. This URL should
+      point to a “Successful Upload” page on your web site, so you can inform
+      your users that their files have been accepted. S3 will add bucket,
+      key and etag parameters to this URL value to inform your web application
+      of the location and hash value of the uploaded file.
+  """
+  def html_upload_form(options \\ []) do
+    ex_aws_config = ExAws.Config.new(:s3, Application.get_all_env(:ex_aws))
+
+    options = Keyword.merge([
+      ex_aws_config: ex_aws_config,
+      key: "uploads/#{Arc.UUID.generate()}",
+      acl: "private",
+      expires_in: 3600,
+      bucket: bucket()
+    ], options)
+
+    Arc.Storage.S3.HtmlUploadForm.generate(options)
+  end
+
   #
   # Private
   #
@@ -68,19 +131,20 @@ defmodule Arc.Storage.S3 do
     end
   end
 
-  defp build_url(definition, version, file_and_scope, _options) do
-    Path.join host, s3_key(definition, version, file_and_scope)
+
+  defp build_url(s3_key, options \\ []) when is_binary(s3_key) do
+    Path.join(host, s3_key)
   end
 
-  defp build_signed_url(definition, version, file_and_scope, options) do
-    options = put_in options[:expire_in], Keyword.get(options, :expire_in, @default_expiry_time)
-    options = put_in options[:virtual_host], virtual_host
+  defp build_signed_url(s3_key, options \\ []) do
+    defaults = [expire_in: @default_expiry_time, virtual_host: virtual_host()]
+    ex_aws_options = Keyword.merge(defaults, options)
     config = ExAws.Config.new(:s3, Application.get_all_env(:ex_aws))
-    {:ok, url} = ExAws.S3.presigned_url(config, :get, bucket, s3_key(definition, version, file_and_scope), options)
+    {:ok, url} = ExAws.S3.presigned_url(config, :get, bucket, s3_key, ex_aws_options)
     url
   end
 
-  defp s3_key(definition, version, file_and_scope) do
+  defp compose_s3_key(definition, version, file_and_scope) do
     Path.join([
       definition.storage_dir(version, file_and_scope),
       Arc.Definition.Versioning.resolve_file_name(definition, version, file_and_scope)
@@ -97,7 +161,7 @@ defmodule Arc.Storage.S3 do
   end
 
   defp default_host do
-    case virtual_host do
+    case virtual_host() do
       true -> "https://#{bucket}.s3.amazonaws.com"
       _    -> "https://s3.amazonaws.com/#{bucket}"
     end

lib/arc/storage/s3/html_upload_form.ex

@@ -0,0 +1,161 @@
+defmodule Arc.Storage.S3.HtmlUploadForm do
+  defstruct action: nil, meta: %{}, fields: []
+
+  def generate(options) do
+    ex_aws_config = Keyword.fetch!(options, :ex_aws_config)
+    form_expires_at = Keyword.get(options, :expires_in, 3600) |> expiration_date() # Defaults to 1 hour
+
+    key = Keyword.fetch!(options, :key)
+    acl = Keyword.fetch!(options, :acl)
+    bucket = Keyword.fetch!(options, :bucket)
+
+    timestamp = :calendar.universal_time()
+
+    html_upload_form = %__MODULE__{
+      action: "https://#{bucket}.s3.amazonaws.com/",
+      meta: %{
+        expires_at: form_expires_at,
+        bucket: bucket
+      },
+      fields: %{
+        "key" => key,
+        "acl" => acl,
+        "x-amz-credential" => amz_credential(ex_aws_config.access_key_id, timestamp, ex_aws_config.region),
+        "x-amz-algorithm" => "AWS4-HMAC-SHA256",
+        "x-amz-date" => amz_datetime(timestamp),
+      }
+    }
+
+    html_upload_form =
+      html_upload_form
+      |> apply_content_headers(options)
+      |> apply_redirect_headers(options)
+
+    policy = generate_encoded_policy(html_upload_form, options)
+    signature = sign_policy(ex_aws_config, policy, timestamp)
+
+    html_upload_form
+    |> add_form_field_exact("policy", policy)
+    |> add_form_field_exact("x-amz-signature", signature)
+  end
+
+  defp add_form_field_exact(form, key, value) do
+    %__MODULE__{form | fields: Map.put(form.fields, key, value)}
+  end
+
+  defp apply_content_headers(form, options) do
+    [:content_disposition, :content_type] |> Enum.reduce(form, fn(key, form) ->
+      if value = Keyword.get(options, key) do
+        add_form_field_exact(form, dasherize(key), value)
+      else
+        form
+      end
+    end)
+  end
+
+  defp apply_redirect_headers(form, options) do
+    [:success_action_redirect] |> Enum.reduce(form, fn(key, form) ->
+      if value = Keyword.get(options, key) do
+        add_form_field_exact(form, key, value)
+      else
+        form
+      end
+    end)
+  end
+
+  defp dasherize(key) do
+    key |> to_string() |> String.replace("_", "-")
+  end
+
+  defp generate_encoded_policy(form, options) do
+    policy_conditions =
+      [%{"bucket" => form.meta.bucket} | to_list_of_maps(form.fields)]
+      |> append_content_length_range_policy(options)
+
+    policy_document = %{
+      expiration: iso_z(form.meta.expires_at),
+      conditions: policy_conditions
+    }
+
+    policy_document
+    |> Poison.encode!()
+    |> Base.encode64()
+  end
+
+  defp append_content_length_range_policy(conditions, options) do
+    if range = Keyword.get(options, :content_length_range) do
+      conditions ++ [["content-length-range" | range]]
+    else
+      conditions
+    end
+  end
+
+  defp to_list_of_maps(map) do
+    map
+    |> Map.to_list()
+    |> Enum.map(fn {k,v} -> %{k => v} end)
+  end
+
+  def sign_policy(config, encoded_policy, timestamp) do
+    ExAws.Auth.Signatures.generate_signature_v4("s3", config, timestamp, encoded_policy)
+  end
+
+  defp amz_credential(access_key_id, timestamp, region) do
+    [
+      access_key_id,
+      amz_date(timestamp),
+      region,
+      "s3",
+      "aws4_request"
+    ] |> Enum.join("/")
+  end
+
+  defp amz_date({{year, month, day}, _}) do
+    Enum.join([
+      year,
+      zero_pad(month),
+      zero_pad(day)
+    ])
+  end
+
+  defp iso_z({{year, month, day}, {hour, min, secs}}) do
+    Enum.join([
+      year,
+      "-",
+      zero_pad(month),
+      "-",
+      zero_pad(day),
+      "T",
+      zero_pad(hour),
+      ":",
+      zero_pad(min),
+      ":",
+      zero_pad(secs),
+      "Z"
+    ])
+  end
+
+  defp amz_datetime({{year, month, day}, {hour, min, secs}}) do
+    Enum.join([
+      year,
+      zero_pad(month),
+      zero_pad(day),
+      "T",
+      zero_pad(hour),
+      zero_pad(min),
+      zero_pad(secs),
+      "Z"
+    ])
+  end
+
+  defp expiration_date(seconds_ahead) do
+    :calendar.universal_time()
+    |> :calendar.datetime_to_gregorian_seconds()
+    |> Kernel.+(seconds_ahead)
+    |> :calendar.gregorian_seconds_to_datetime()
+  end
+
+  defp zero_pad(<<_>> = val) when is_binary(val), do: "0" <> val
+  defp zero_pad(val) when is_binary(val), do: val
+  defp zero_pad(non_binary), do: zero_pad(to_string(non_binary))
+end