diff --git a/README.md b/README.md index 1647908..878598f 100644 --- a/README.md +++ b/README.md @@ -495,6 +495,11 @@ config :ex_aws, > For your host configuration, please examine the approved [AWS Hostnames](http://docs.aws.amazon.com/general/latest/gr/rande.html). There are often multiple hostname formats for AWS regions, and it will not work unless you specify the correct one. +### Browser Uploads direct to Amazon S3 + +For many use cases, it is unnecessary to proxy file uploads through the application layer. Certain cloud storage platforms, such as Amazon S3, allow for browsers to upload files directly to S3 to take the load off and bandwidth off of your application layer. + +To use direct-to-s3 uploads, please reference: https://github.com/stavro/arc/wiki/Direct-to-S3-Uploads # Full Example diff --git a/lib/arc/file.ex b/lib/arc/file.ex index 704f309..71bc7cc 100644 --- a/lib/arc/file.ex +++ b/lib/arc/file.ex @@ -3,19 +3,14 @@ defmodule Arc.File do def generate_temporary_path(file \\ nil) do extension = Path.extname((file && file.path) || "") - - file_name = - :crypto.rand_bytes(20) - |> Base.encode32() - |> Kernel.<>(extension) - + file_name = Arc.UUID.generate() <> extension Path.join(System.tmp_dir, file_name) end # Given a remote file def new(remote_path = "http" <> _) do case save_file(remote_path) do - {:ok, local_path} -> %Arc.File{path: local_path, file_name: Path.basename(remote_path)} + {:ok, local_path, file_name} -> %Arc.File{path: local_path, file_name: file_name} :error -> {:error, :invalid_file_path} end end @@ -59,7 +54,7 @@ defmodule Arc.File do |> Kernel.<>(Path.extname(remote_path)) case save_temp_file(local_path, remote_path) do - :ok -> {:ok, local_path} + {:ok, file_name} -> {:ok, local_path, file_name} _ -> :error end end @@ -67,16 +62,28 @@ defmodule Arc.File do defp save_temp_file(local_path, remote_path) do remote_file = get_remote_path(remote_path) - case remote_file do - {:ok, body} -> File.write(local_path, body) - {:error, error} -> {:error, error} + with {:ok, body, file_name} <- get_remote_path(remote_path), + :ok <- File.write(local_path, body) do + {:ok, file_name} end end defp get_remote_path(remote_path) do case HTTPoison.get(remote_path) do - {:ok, %{status_code: 200, body: body}} -> {:ok, body} + {:ok, %{status_code: 200, body: body, headers: headers}} -> + file_name = resolve_file_name(headers) || Path.basename(remote_path) + {:ok, body, file_name} other -> {:error, :invalid_file_path} end end + + defp resolve_file_name(response) do + response + |> Map.get(:headers) + |> Enum.find(fn {k, v} -> String.downcase(to_string(k)) == "content-disposition" end) + |> case do + {_k, v} -> v + _ -> nil + end + end end diff --git a/lib/arc/storage/s3.ex b/lib/arc/storage/s3.ex index d4dd896..26c6095 100644 --- a/lib/arc/storage/s3.ex +++ b/lib/arc/storage/s3.ex @@ -16,20 +16,83 @@ defmodule Arc.Storage.S3 do end def url(definition, version, file_and_scope, options \\ []) do + compose_s3_key(definition, version, file_and_scope) + |> url(options) + end + + def url(s3_key, options \\ []) when is_binary(s3_key) do case Keyword.get(options, :signed, false) do - false -> build_url(definition, version, file_and_scope, options) - true -> build_signed_url(definition, version, file_and_scope, options) + false -> build_url(s3_key, options) + true -> build_signed_url(s3_key, options) end end def delete(definition, version, {file, scope}) do - bucket - |> ExAws.S3.delete_object(s3_key(definition, version, {file, scope})) + compose_s3_key(definition, version, {file, scope}) + |> delete() + end + + def delete(s3_key) when is_binary(s3_key) do + bucket() + |> ExAws.S3.delete_object(s3_key) |> ExAws.request() :ok end + @doc """ + Generates an upload form capable of submitting a file directly to Amazon S3. + + ## Options + + * `:bucket` - Which bucket the upload will be placed in. Defaults to the + Arc bucket. + * `:key` - The S3 key (or file path) where the upload will be stored, + defaults to `uploads/#{Arc.UUID.generate()}` + * `:acl` - The access control policy to apply to the uploaded file. If you + do not want the uploaded file to be made available to the general public, + you should use the value `private`. To make the uploaded file publicly + available, use the value `public-read`. Defaults to `private`. + * `:expires_in` - A value in seconds that specifies how long the policy + document will be valid for. Once a policy document has expired, the + upload form will no longer work. + * `:content_length_range` - Content length range as `[min, max]`, where + S3 will check that the size of an uploaded file is between a given + minimum and maximum value (in bytes). If this rule is not included in a + policy document, users will be able to upload files of any size up to + the 5GB limit imposed by S3. + * `:content_disposition` - Content header passed through to Amazon S3. This + allows you to specify that the file is an attachment, and what filename + the upload should be downloaded as. Eg. `attachment; filename=image.png` + * `:content_type`- The content type (mime type) that will be applied to the + uploaded file, for example image/jpeg for JPEG picture files. If you do + not know what type of file a user will upload, you can either let the + user choose the file prior to generating this upload form to determine + the appropriate content type. If you do not set the content type with + this field, S3 will use the default value application/octet-stream which + may prevent some web browsers from being able to display the file + properly. + * `:success_action_redirect` - The URL address to which the user’s web + browser will be redirected after the file is uploaded. This URL should + point to a “Successful Upload” page on your web site, so you can inform + your users that their files have been accepted. S3 will add bucket, + key and etag parameters to this URL value to inform your web application + of the location and hash value of the uploaded file. + """ + def html_upload_form(options \\ []) do + ex_aws_config = ExAws.Config.new(:s3, Application.get_all_env(:ex_aws)) + + options = Keyword.merge([ + ex_aws_config: ex_aws_config, + key: "uploads/#{Arc.UUID.generate()}", + acl: "private", + expires_in: 3600, + bucket: bucket() + ], options) + + Arc.Storage.S3.HtmlUploadForm.generate(options) + end + # # Private # @@ -68,19 +131,20 @@ defmodule Arc.Storage.S3 do end end - defp build_url(definition, version, file_and_scope, _options) do - Path.join host, s3_key(definition, version, file_and_scope) + + defp build_url(s3_key, options \\ []) when is_binary(s3_key) do + Path.join(host, s3_key) end - defp build_signed_url(definition, version, file_and_scope, options) do - options = put_in options[:expire_in], Keyword.get(options, :expire_in, @default_expiry_time) - options = put_in options[:virtual_host], virtual_host + defp build_signed_url(s3_key, options \\ []) do + defaults = [expire_in: @default_expiry_time, virtual_host: virtual_host()] + ex_aws_options = Keyword.merge(defaults, options) config = ExAws.Config.new(:s3, Application.get_all_env(:ex_aws)) - {:ok, url} = ExAws.S3.presigned_url(config, :get, bucket, s3_key(definition, version, file_and_scope), options) + {:ok, url} = ExAws.S3.presigned_url(config, :get, bucket, s3_key, ex_aws_options) url end - defp s3_key(definition, version, file_and_scope) do + defp compose_s3_key(definition, version, file_and_scope) do Path.join([ definition.storage_dir(version, file_and_scope), Arc.Definition.Versioning.resolve_file_name(definition, version, file_and_scope) @@ -97,7 +161,7 @@ defmodule Arc.Storage.S3 do end defp default_host do - case virtual_host do + case virtual_host() do true -> "https://#{bucket}.s3.amazonaws.com" _ -> "https://s3.amazonaws.com/#{bucket}" end diff --git a/lib/arc/storage/s3/html_upload_form.ex b/lib/arc/storage/s3/html_upload_form.ex new file mode 100644 index 0000000..b18d0d7 --- /dev/null +++ b/lib/arc/storage/s3/html_upload_form.ex @@ -0,0 +1,161 @@ +defmodule Arc.Storage.S3.HtmlUploadForm do + defstruct action: nil, meta: %{}, fields: [] + + def generate(options) do + ex_aws_config = Keyword.fetch!(options, :ex_aws_config) + form_expires_at = Keyword.get(options, :expires_in, 3600) |> expiration_date() # Defaults to 1 hour + + key = Keyword.fetch!(options, :key) + acl = Keyword.fetch!(options, :acl) + bucket = Keyword.fetch!(options, :bucket) + + timestamp = :calendar.universal_time() + + html_upload_form = %__MODULE__{ + action: "https://#{bucket}.s3.amazonaws.com/", + meta: %{ + expires_at: form_expires_at, + bucket: bucket + }, + fields: %{ + "key" => key, + "acl" => acl, + "x-amz-credential" => amz_credential(ex_aws_config.access_key_id, timestamp, ex_aws_config.region), + "x-amz-algorithm" => "AWS4-HMAC-SHA256", + "x-amz-date" => amz_datetime(timestamp), + } + } + + html_upload_form = + html_upload_form + |> apply_content_headers(options) + |> apply_redirect_headers(options) + + policy = generate_encoded_policy(html_upload_form, options) + signature = sign_policy(ex_aws_config, policy, timestamp) + + html_upload_form + |> add_form_field_exact("policy", policy) + |> add_form_field_exact("x-amz-signature", signature) + end + + defp add_form_field_exact(form, key, value) do + %__MODULE__{form | fields: Map.put(form.fields, key, value)} + end + + defp apply_content_headers(form, options) do + [:content_disposition, :content_type] |> Enum.reduce(form, fn(key, form) -> + if value = Keyword.get(options, key) do + add_form_field_exact(form, dasherize(key), value) + else + form + end + end) + end + + defp apply_redirect_headers(form, options) do + [:success_action_redirect] |> Enum.reduce(form, fn(key, form) -> + if value = Keyword.get(options, key) do + add_form_field_exact(form, key, value) + else + form + end + end) + end + + defp dasherize(key) do + key |> to_string() |> String.replace("_", "-") + end + + defp generate_encoded_policy(form, options) do + policy_conditions = + [%{"bucket" => form.meta.bucket} | to_list_of_maps(form.fields)] + |> append_content_length_range_policy(options) + + policy_document = %{ + expiration: iso_z(form.meta.expires_at), + conditions: policy_conditions + } + + policy_document + |> Poison.encode!() + |> Base.encode64() + end + + defp append_content_length_range_policy(conditions, options) do + if range = Keyword.get(options, :content_length_range) do + conditions ++ [["content-length-range" | range]] + else + conditions + end + end + + defp to_list_of_maps(map) do + map + |> Map.to_list() + |> Enum.map(fn {k,v} -> %{k => v} end) + end + + def sign_policy(config, encoded_policy, timestamp) do + ExAws.Auth.Signatures.generate_signature_v4("s3", config, timestamp, encoded_policy) + end + + defp amz_credential(access_key_id, timestamp, region) do + [ + access_key_id, + amz_date(timestamp), + region, + "s3", + "aws4_request" + ] |> Enum.join("/") + end + + defp amz_date({{year, month, day}, _}) do + Enum.join([ + year, + zero_pad(month), + zero_pad(day) + ]) + end + + defp iso_z({{year, month, day}, {hour, min, secs}}) do + Enum.join([ + year, + "-", + zero_pad(month), + "-", + zero_pad(day), + "T", + zero_pad(hour), + ":", + zero_pad(min), + ":", + zero_pad(secs), + "Z" + ]) + end + + defp amz_datetime({{year, month, day}, {hour, min, secs}}) do + Enum.join([ + year, + zero_pad(month), + zero_pad(day), + "T", + zero_pad(hour), + zero_pad(min), + zero_pad(secs), + "Z" + ]) + end + + defp expiration_date(seconds_ahead) do + :calendar.universal_time() + |> :calendar.datetime_to_gregorian_seconds() + |> Kernel.+(seconds_ahead) + |> :calendar.gregorian_seconds_to_datetime() + end + + defp zero_pad(<<_>> = val) when is_binary(val), do: "0" <> val + defp zero_pad(val) when is_binary(val), do: val + defp zero_pad(non_binary), do: zero_pad(to_string(non_binary)) +end diff --git a/lib/arc/uuid.ex b/lib/arc/uuid.ex new file mode 100644 index 0000000..9fe6309 --- /dev/null +++ b/lib/arc/uuid.ex @@ -0,0 +1,48 @@ +defmodule Arc.UUID do + + @doc """ + Generates a version 4 (random) UUID. + """ + def generate do + bingenerate() |> encode() + end + + defp bingenerate do + <> = :crypto.strong_rand_bytes(16) + <> + end + + defp encode(<< a1::4, a2::4, a3::4, a4::4, + a5::4, a6::4, a7::4, a8::4, + b1::4, b2::4, b3::4, b4::4, + c1::4, c2::4, c3::4, c4::4, + d1::4, d2::4, d3::4, d4::4, + e1::4, e2::4, e3::4, e4::4, + e5::4, e6::4, e7::4, e8::4, + e9::4, e10::4, e11::4, e12::4 >>) do + << e(a1), e(a2), e(a3), e(a4), e(a5), e(a6), e(a7), e(a8), ?-, + e(b1), e(b2), e(b3), e(b4), ?-, + e(c1), e(c2), e(c3), e(c4), ?-, + e(d1), e(d2), e(d3), e(d4), ?-, + e(e1), e(e2), e(e3), e(e4), e(e5), e(e6), e(e7), e(e8), e(e9), e(e10), e(e11), e(e12) >> + end + + @compile {:inline, e: 1} + + defp e(0), do: ?0 + defp e(1), do: ?1 + defp e(2), do: ?2 + defp e(3), do: ?3 + defp e(4), do: ?4 + defp e(5), do: ?5 + defp e(6), do: ?6 + defp e(7), do: ?7 + defp e(8), do: ?8 + defp e(9), do: ?9 + defp e(10), do: ?a + defp e(11), do: ?b + defp e(12), do: ?c + defp e(13), do: ?d + defp e(14), do: ?e + defp e(15), do: ?f +end diff --git a/test/storage/s3_html_upload_test.exs b/test/storage/s3_html_upload_test.exs new file mode 100644 index 0000000..e76921f --- /dev/null +++ b/test/storage/s3_html_upload_test.exs @@ -0,0 +1,102 @@ +defmodule ArcTest.Storage.S3HtmlUpload do + use ExUnit.Case, async: false + + @img "test/support/image.png" + + setup_all do + Application.ensure_all_started(:hackney) + Application.ensure_all_started(:httpoison) + Application.ensure_all_started(:ex_aws) + Application.put_env :arc, :virtual_host, false + Application.put_env :arc, :bucket, { :system, "ARC_TEST_BUCKET" } + Application.put_env :ex_aws, :access_key_id, System.get_env("ARC_TEST_S3_KEY") + Application.put_env :ex_aws, :secret_access_key, System.get_env("ARC_TEST_S3_SECRET") + end + + def upload_image(image, options \\ []) do + form = Arc.Storage.S3.html_upload_form(options) + multipart_fields = form.fields |> Map.to_list() |> Kernel.++([{:file, image}]) + {:ok, response} = HTTPoison.post(form.action, {:multipart, multipart_fields}) + + if response.status_code === 204 do + {:ok, form.fields["key"], response} + else + {:error, response.body} + end + end + + def public_url(key) do + Arc.Storage.S3.url(key) + end + + def signed_url(key) do + Arc.Storage.S3.url(key, signed: true) + end + + def is_private(key) do + {:ok, res} = + key + |> public_url() + |> HTTPoison.head() + + res.status_code == 403 + end + + def is_privately_accessible(key) do + {:ok, res} = + key + |> signed_url() + |> HTTPoison.get() + + res.status_code == 200 + end + + def header_value(key, header) do + key + |> public_url() + |> HTTPoison.head!() + |> Map.get(:headers) + |> Enum.find(fn {k, v} -> k == header end) + |> case do + {_k, v} -> v + _ -> nil + end + end + + @tag :s3 + test "files are private by default" do + {:ok, key, response} = upload_image(@img) + assert is_private(key) + assert is_privately_accessible(key) + end + + @tag :s3 + test "files can be made public" do + {:ok, key, response} = upload_image(@img, acl: "public-read") + refute is_private(key) + assert is_privately_accessible(key) + end + + @tag :s3 + test "files can specify content disposition" do + disposition = "attachment; filename=\"test.png\"" + {:ok, key, response} = upload_image(@img, acl: "public-read", content_disposition: disposition) + refute is_private(key) + assert header_value(key, "Content-Disposition") == disposition + end + + @tag :s3 + test "uploads can specify content-length range" do + disposition = "attachment; filename=\"test.png\"" + {:error, error_message} = upload_image(@img, acl: "public-read", content_length_range: [0, 100]) + assert error_message =~ ~r/EntityTooLarge/ + end + + @tag :s3 + test "Storing an uploaded file will respect the content disposition" do + disposition = "attachment; filename=\"test.png\"" + {:ok, key, response} = upload_image(@img, acl: "public-read", content_disposition: disposition) + file = Arc.File.new(public_url(key)) + IO.inspect(file) + end +end