Merge branch 'media-proxy' into 'develop'

Media proxy

See merge request pleroma/pleroma!34

config/config.exs

@@ -47,6 +47,11 @@
   limit: 5000,
   registrations_open: true
 
+config :pleroma, :media_proxy,
+  enabled: false,
+  redirect_on_failure: true
+  #base_url: "https://cache.pleroma.social"
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env}.exs"

installation/pleroma.nginx

@@ -1,3 +1,6 @@
+proxy_cache_path /tmp/pleroma-media-cache levels=1:2 keys_zone=pleroma_media_cache:10m max_size=10g
+                 inactive=720m use_temp_path=off;
+
 server {
        listen         80;
        server_name    example.tld;
@@ -24,6 +27,13 @@ server {
         proxy_set_header Connection "upgrade";
         proxy_pass http://localhost:4000;
     }
+
+    location /proxy {
+        proxy_cache pleroma_media_cache;
+        proxy_cache_lock on;
+        proxy_pass http://localhost:4000;
+    }
+
     include snippets/well-known.conf;
 
-}
+}

lib/pleroma/formatter.ex

@@ -1,5 +1,6 @@
 defmodule Pleroma.Formatter do
   alias Pleroma.User
+  alias Pleroma.Web.MediaProxy
 
   @link_regex ~r/https?:\/\/[\w\.\/?=\-#%&@~\(\)]+[\w\/]/u
   def linkify(text) do
@@ -125,7 +126,7 @@ def emojify(text, additional \\ nil) do
     end
 
     Enum.reduce(all_emoji, text, fn ({emoji, file}, text) ->
-      String.replace(text, ":#{emoji}:", "<img height='32px' width='32px' alt='#{emoji}' title='#{emoji}' src='#{file}' />")
+      String.replace(text, ":#{emoji}:", "<img height='32px' width='32px' alt='#{emoji}' title='#{emoji}' src='#{MediaProxy.url(file)}' />")
     end)
   end
 

lib/pleroma/web/mastodon_api/views/account_view.ex

@@ -3,6 +3,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
   alias Pleroma.User
   alias Pleroma.Web.MastodonAPI.AccountView
   alias Pleroma.Web.CommonAPI.Utils
+  alias Pleroma.Web.MediaProxy
 
   defp image_url(%{"url" => [ %{ "href" => href } | _ ]}), do: href
   defp image_url(_), do: nil
@@ -12,10 +13,10 @@ def render("accounts.json", %{users: users} = opts) do
   end
 
   def render("account.json", %{user: user}) do
-    image = User.avatar_url(user)
+    image = User.avatar_url(user) |> MediaProxy.url()
     user_info = User.user_info(user)
 
-    header = image_url(user.info["banner"]) || "https://placehold.it/700x335"
+    header = (image_url(user.info["banner"]) || "https://placehold.it/700x335") |> MediaProxy.url()
 
     %{
       id: to_string(user.id),

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -3,6 +3,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
   alias Pleroma.Web.MastodonAPI.{AccountView, StatusView}
   alias Pleroma.{User, Activity}
   alias Pleroma.Web.CommonAPI.Utils
+  alias Pleroma.Web.MediaProxy
 
   def render("index.json", opts) do
     render_many(opts.activities, StatusView, "status.json", opts)
@@ -121,9 +122,9 @@ def render("attachment.json", %{attachment: attachment}) do
 
     %{
       id: to_string(attachment["id"] || hash_id),
-      url: href,
+      url: MediaProxy.url(href),
       remote_url: href,
-      preview_url: href,
+      preview_url: MediaProxy.url(href),
       text_url: href,
       type: type
     }

lib/pleroma/web/media_proxy/controller.ex

@@ -0,0 +1,83 @@
+defmodule Pleroma.Web.MediaProxy.MediaProxyController do
+  use Pleroma.Web, :controller
+  require Logger
+
+  @max_body_length 25 * 1048576
+
+  @cache_control %{
+    default: "public, max-age=1209600",
+    error:   "public, must-revalidate, max-age=160",
+  }
+
+  def remote(conn, %{"sig" => sig, "url" => url}) do
+    config = Application.get_env(:pleroma, :media_proxy, [])
+    with \
+      true <- Keyword.get(config, :enabled, false),
+      {:ok, url} <- Pleroma.Web.MediaProxy.decode_url(sig, url),
+      url = URI.encode(url),
+      {:ok, content_type, body} <- proxy_request(url)
+    do
+      conn
+      |> put_resp_content_type(content_type)
+      |> set_cache_header(:default)
+      |> send_resp(200, body)
+    else
+      false -> send_error(conn, 404)
+      {:error, :invalid_signature} -> send_error(conn, 403)
+      {:error, {:http, _, url}} -> redirect_or_error(conn, url, Keyword.get(config, :redirect_on_failure, true))
+    end
+  end
+
+  defp proxy_request(link) do
+    headers = [{"user-agent", "Pleroma/MediaProxy; #{Pleroma.Web.base_url()} <#{Application.get_env(:pleroma, :instance)[:email]}>"}]
+    options = [:insecure, {:follow_redirect, true}]
+    with \
+      {:ok, 200, headers, client} <- :hackney.request(:get, link, headers, "", options),
+      headers = Enum.into(headers, Map.new),
+      {:ok, body} <- proxy_request_body(client),
+      content_type <- proxy_request_content_type(headers, body)
+    do
+      {:ok, content_type, body}
+    else
+      {:ok, status, _, _} ->
+        Logger.warn "MediaProxy: request failed, status #{status}, link: #{link}"
+        {:error, {:http, :bad_status, link}}
+      {:error, error} ->
+        Logger.warn "MediaProxy: request failed, error #{inspect error}, link: #{link}"
+        {:error, {:http, error, link}}
+    end
+  end
+
+  defp set_cache_header(conn, key) do
+    Plug.Conn.put_resp_header(conn, "cache-control", @cache_control[key])
+  end
+
+  defp redirect_or_error(conn, url, true), do: redirect(conn, external: url)
+  defp redirect_or_error(conn, url, _), do: send_error(conn, 502, "Media proxy error: " <> url)
+
+  defp send_error(conn, code, body \\ "") do
+    conn
+    |> set_cache_header(:error)
+    |> send_resp(code, body)
+  end
+
+  defp proxy_request_body(client), do: proxy_request_body(client, <<>>)
+  defp proxy_request_body(client, body) when byte_size(body) < @max_body_length do
+    case :hackney.stream_body(client) do
+      {:ok, data} -> proxy_request_body(client, <<body :: binary, data :: binary>>)
+      :done -> {:ok, body}
+      {:error, reason} -> {:error, reason}
+    end
+  end
+  defp proxy_request_body(client, _) do
+    :hackney.close(client)
+    {:error, :body_too_large}
+  end
+
+  # TODO: the body is passed here as well because some hosts do not provide a content-type.
+  # At some point we may want to use magic numbers to discover the content-type and reply a proper one.
+  defp proxy_request_content_type(headers, _body) do
+    headers["Content-Type"] || headers["content-type"] || "image/jpeg"
+  end
+
+end

lib/pleroma/web/media_proxy/media_proxy.ex

@@ -0,0 +1,32 @@
+defmodule Pleroma.Web.MediaProxy do
+  @base64_opts [padding: false]
+
+  def url(nil), do: nil
+
+  def url(url = "/" <> _), do: url
+
+  def url(url) do
+    config = Application.get_env(:pleroma, :media_proxy, [])
+    if !Keyword.get(config, :enabled, false) or String.starts_with?(url, Pleroma.Web.base_url) do
+      url
+    else
+      secret = Application.get_env(:pleroma, Pleroma.Web.Endpoint)[:secret_key_base]
+      base64 = Base.url_encode64(url, @base64_opts)
+      sig = :crypto.hmac(:sha, secret, base64)
+      sig64 = sig |> Base.url_encode64(@base64_opts)
+      Keyword.get(config, :base_url, Pleroma.Web.base_url) <> "/proxy/#{sig64}/#{base64}"
+    end
+  end
+
+  def decode_url(sig, url) do
+    secret = Application.get_env(:pleroma, Pleroma.Web.Endpoint)[:secret_key_base]
+    sig = Base.url_decode64!(sig, @base64_opts)
+    local_sig = :crypto.hmac(:sha, secret, url)
+    if local_sig == sig do
+      {:ok, Base.url_decode64!(url, @base64_opts)}
+    else
+      {:error, :invalid_signature}
+    end
+  end
+
+end

lib/pleroma/web/router.ex

@@ -238,6 +238,14 @@ def user_fetcher(username) do
     delete "/auth/sign_out", MastodonAPIController, :logout
   end
 
+  pipeline :remote_media do
+    plug :accepts, ["html"]
+  end
+  scope "/proxy/", Pleroma.Web.MediaProxy do
+    pipe_through :remote_media
+    get "/:sig/:url", MediaProxyController, :remote
+  end
+
   scope "/", Fallback do
     get "/*path", RedirectController, :redirector
   end

lib/pleroma/web/twitter_api/representers/object_representer.ex

@@ -6,7 +6,7 @@ def to_map(%Object{} = object, _opts) do
     data = object.data
     url = List.first(data["url"])
     %{
-      url: url["href"],
+      url: url["href"] |> Pleroma.Web.MediaProxy.url(),
       mimetype: url["mediaType"],
       id: data["uuid"],
       oembed: false

lib/pleroma/web/twitter_api/views/user_view.ex

@@ -2,6 +2,7 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
   use Pleroma.Web, :view
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI.Utils
+  alias Pleroma.Web.MediaProxy
 
   def render("show.json", %{user: user = %User{}} = assigns) do
     render_one(user, Pleroma.Web.TwitterAPI.UserView, "user.json", assigns)
@@ -12,7 +13,7 @@ def render("index.json", %{users: users, for: user}) do
   end
 
   def render("user.json", %{user: user = %User{}} = assigns) do
-    image = User.avatar_url(user)
+    image = User.avatar_url(user) |> MediaProxy.url()
     {following, follows_you, statusnet_blocking} = if assigns[:for] do
       {
         User.following?(assigns[:for], user),
@@ -44,8 +45,8 @@ def render("user.json", %{user: user = %User{}} = assigns) do
       "screen_name" => user.nickname,
       "statuses_count" => user_info[:note_count],
       "statusnet_profile_url" => user.ap_id,
-      "cover_photo" => image_url(user.info["banner"]),
-      "background_image" => image_url(user.info["background"])
+      "cover_photo" => image_url(user.info["banner"]) |> MediaProxy.url(),
+      "background_image" => image_url(user.info["background"]) |> MediaProxy.url(),
     }
 
     if assigns[:token] do