Merge branch 'feature/s3' into 'develop'

S3 support

Closes #65

See merge request pleroma/pleroma!303
This commit is contained in:
kaniini 2018-08-28 00:29:49 +00:00
commit 49b165ddc6
5 changed files with 70 additions and 2 deletions

View file

@ -12,7 +12,9 @@
config :pleroma, Pleroma.Upload,
uploads: "uploads",
strip_exif: false
strip_exif: false,
use_s3: false,
s3_bucket: nil
config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"]

View file

@ -24,3 +24,23 @@ config :pleroma, Pleroma.Repo,
database: "pleroma_dev",
hostname: "localhost",
pool_size: 10
# Configure S3 support if desired.
# The public S3 endpoint is different depending on region and provider,
# consult your S3 provider's documentation for details on what to use.
#
# config :pleroma, Pleroma.Upload,
# use_s3: true,
# bucket: "some-bucket",
# public_endpoint: "https://s3.amazonaws.com"
#
# Configure S3 credentials:
# config :ex_aws, :s3,
# access_key_id: "xxxxxxxxxxxxx",
# secret_access_key: "yyyyyyyyyyyy",
# region: "us-east-1",
# scheme: "https://"
#
# For using third-party S3 clones like wasabi, also do:
# config :ex_aws, :s3,
# host: "s3.wasabisys.com"

View file

@ -3,12 +3,17 @@ defmodule Pleroma.Upload do
alias Pleroma.Web
def store(%Plug.Upload{} = file, should_dedupe) do
settings = Application.get_env(:pleroma, Pleroma.Upload)
use_s3 = Keyword.fetch!(settings, :use_s3)
content_type = get_content_type(file.path)
uuid = get_uuid(file, should_dedupe)
name = get_name(file, uuid, content_type, should_dedupe)
upload_folder = get_upload_path(uuid, should_dedupe)
url_path = get_url(name, uuid, should_dedupe)
strip_exif_data(content_type, file.path)
File.mkdir_p!(upload_folder)
result_file = Path.join(upload_folder, name)
@ -18,7 +23,12 @@ def store(%Plug.Upload{} = file, should_dedupe) do
File.cp!(file.path, result_file)
end
strip_exif_data(content_type, result_file)
url_path =
if use_s3 do
put_s3_file(name, uuid, result_file, content_type)
else
url_path
end
%{
"type" => "Document",
@ -33,7 +43,11 @@ def store(%Plug.Upload{} = file, should_dedupe) do
}
end
# XXX: does this code actually work? i am skeptical. --kaniini
def store(%{"img" => "data:image/" <> image_data}, should_dedupe) do
settings = Application.get_env(:pleroma, Pleroma.Upload)
use_s3 = Keyword.fetch!(settings, :use_s3)
parsed = Regex.named_captures(~r/(?<filetype>jpeg|png|gif);base64,(?<data>.*)/, image_data)
data = Base.decode64!(parsed["data"], ignore: :whitespace)
uuid = UUID.generate()
@ -71,6 +85,13 @@ def store(%{"img" => "data:image/" <> image_data}, should_dedupe) do
strip_exif_data(content_type, result_file)
url_path =
if use_s3 do
put_s3_file(name, uuid, result_file, content_type)
else
url_path
end
%{
"type" => "Image",
"url" => [
@ -203,4 +224,25 @@ def get_content_type(file) do
_e -> "application/octet-stream"
end
end
defp put_s3_file(name, uuid, path, content_type) do
settings = Application.get_env(:pleroma, Pleroma.Upload)
bucket = Keyword.fetch!(settings, :bucket)
public_endpoint = Keyword.fetch!(settings, :public_endpoint)
{:ok, file_data} = File.read(path)
File.rm!(path)
s3_name = "#{uuid}/#{name}"
{:ok, result} =
ExAws.S3.put_object(bucket, s3_name, file_data, [
{:acl, :public_read},
{:content_type, content_type}
])
|> ExAws.request()
"#{public_endpoint}/#{bucket}/#{s3_name}"
end
end

View file

@ -46,6 +46,8 @@ defp deps do
{:httpoison, "~> 1.2.0"},
{:jason, "~> 1.0"},
{:mogrify, "~> 0.6.1"},
{:ex_aws, "~> 2.0"},
{:ex_aws_s3, "~> 2.0"},
{:ex_machina, "~> 2.2", only: :test},
{:credo, "~> 0.9.3", only: [:dev, :test]},
{:mock, "~> 0.3.1", only: :test}

View file

@ -12,6 +12,8 @@
"decimal": {:hex, :decimal, "1.5.0", "b0433a36d0e2430e3d50291b1c65f53c37d56f83665b43d79963684865beab68", [:mix], [], "hexpm"},
"ecto": {:hex, :ecto, "2.2.10", "e7366dc82f48f8dd78fcbf3ab50985ceeb11cb3dc93435147c6e13f2cda0992e", [:mix], [{:db_connection, "~> 1.1", [hex: :db_connection, repo: "hexpm", optional: true]}, {:decimal, "~> 1.2", [hex: :decimal, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.8.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}, {:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.13.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, repo: "hexpm", optional: true]}], "hexpm"},
"eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"},
"ex_aws": {:hex, :ex_aws, "2.1.0", "b92651527d6c09c479f9013caa9c7331f19cba38a650590d82ebf2c6c16a1d8a", [:mix], [{:configparser_ex, "~> 2.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:xml_builder, "~> 0.1.0", [hex: :xml_builder, repo: "hexpm", optional: true]}], "hexpm"},
"ex_aws_s3": {:hex, :ex_aws_s3, "2.0.1", "9e09366e77f25d3d88c5393824e613344631be8db0d1839faca49686e99b6704", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm"},
"ex_machina": {:hex, :ex_machina, "2.2.0", "fec496331e04fc2db2a1a24fe317c12c0c4a50d2beb8ebb3531ed1f0d84be0ed", [:mix], [{:ecto, "~> 2.1", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"},
"gettext": {:hex, :gettext, "0.15.0", "40a2b8ce33a80ced7727e36768499fc9286881c43ebafccae6bab731e2b2b8ce", [:mix], [], "hexpm"},
"hackney": {:hex, :hackney, "1.13.0", "24edc8cd2b28e1c652593833862435c80661834f6c9344e84b6a2255e7aeef03", [:rebar3], [{:certifi, "2.3.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "5.1.2", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},