Merge branch 'media-preview-proxy-nostream' into 'develop'
Media preview proxy See merge request pleroma/pleroma!3001
This commit is contained in:
feld
commit
6c052bd5b6
25 changed files with 982 additions and 127 deletions
|
|
@ -15,6 +15,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
|
|||
- The `discoverable` field in the `User` struct will now add a NOINDEX metatag to profile pages when false.
|
||||
- Users with the `discoverable` field set to false will not show up in searches.
|
||||
- Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option).
|
||||
|
||||
### Added
|
||||
- Media preview proxy (requires media proxy be enabled; see `:media_preview_proxy` config for more details).
|
||||
|
||||
### Removed
|
||||
|
||||
- **Breaking:** `Pleroma.Workers.Cron.StatsWorker` setting from Oban `:crontab` (moved to a simpler implementation).
|
||||
|
|
|
|||
|
|
@ -434,6 +434,8 @@
|
|||
proxy_opts: [
|
||||
redirect_on_failure: false,
|
||||
max_body_length: 25 * 1_048_576,
|
||||
# Note: max_read_duration defaults to Pleroma.ReverseProxy.max_read_duration_default/1
|
||||
max_read_duration: 30_000,
|
||||
http: [
|
||||
follow_redirect: true,
|
||||
pool: :media
|
||||
|
|
@ -448,6 +450,14 @@
|
|||
|
||||
config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script, script_path: nil
|
||||
|
||||
# Note: media preview proxy depends on media proxy to be enabled
|
||||
config :pleroma, :media_preview_proxy,
|
||||
enabled: false,
|
||||
thumbnail_max_width: 600,
|
||||
thumbnail_max_height: 600,
|
||||
image_quality: 85,
|
||||
min_content_length: 100 * 1024
|
||||
|
||||
config :pleroma, :chat, enabled: true
|
||||
|
||||
config :phoenix, :format_encoders, json: Jason
|
||||
|
|
@ -753,8 +763,8 @@
|
|||
],
|
||||
media: [
|
||||
size: 50,
|
||||
max_waiting: 10,
|
||||
recv_timeout: 10_000
|
||||
max_waiting: 20,
|
||||
recv_timeout: 15_000
|
||||
],
|
||||
upload: [
|
||||
size: 25,
|
||||
|
|
|
|||
|
|
@ -1887,6 +1887,7 @@
|
|||
suggestions: [
|
||||
redirect_on_failure: false,
|
||||
max_body_length: 25 * 1_048_576,
|
||||
max_read_duration: 30_000,
|
||||
http: [
|
||||
follow_redirect: true,
|
||||
pool: :media
|
||||
|
|
@ -1907,6 +1908,11 @@
|
|||
"Limits the content length to be approximately the " <>
|
||||
"specified length. It is validated with the `content-length` header and also verified when proxying."
|
||||
},
|
||||
%{
|
||||
key: :max_read_duration,
|
||||
type: :integer,
|
||||
description: "Timeout (in milliseconds) of GET request to remote URI."
|
||||
},
|
||||
%{
|
||||
key: :http,
|
||||
label: "HTTP",
|
||||
|
|
@ -1953,6 +1959,43 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
%{
|
||||
group: :pleroma,
|
||||
key: :media_preview_proxy,
|
||||
type: :group,
|
||||
description: "Media preview proxy",
|
||||
children: [
|
||||
%{
|
||||
key: :enabled,
|
||||
type: :boolean,
|
||||
description:
|
||||
"Enables proxying of remote media preview to the instance's proxy. Requires enabled media proxy."
|
||||
},
|
||||
%{
|
||||
key: :thumbnail_max_width,
|
||||
type: :integer,
|
||||
description:
|
||||
"Max width of preview thumbnail for images (video preview always has original dimensions)."
|
||||
},
|
||||
%{
|
||||
key: :thumbnail_max_height,
|
||||
type: :integer,
|
||||
description:
|
||||
"Max height of preview thumbnail for images (video preview always has original dimensions)."
|
||||
},
|
||||
%{
|
||||
key: :image_quality,
|
||||
type: :integer,
|
||||
description: "Quality of the output. Ranges from 0 (min quality) to 100 (max quality)."
|
||||
},
|
||||
%{
|
||||
key: :min_content_length,
|
||||
type: :integer,
|
||||
description:
|
||||
"Min content length to perform preview, in bytes. If greater than 0, media smaller in size will be served as is, without thumbnailing."
|
||||
}
|
||||
]
|
||||
},
|
||||
%{
|
||||
group: :pleroma,
|
||||
key: Pleroma.Web.MediaProxy.Invalidation.Http,
|
||||
|
|
|
|||
|
|
@ -324,6 +324,14 @@ This section describe PWA manifest instance-specific values. Currently this opti
|
|||
* `enabled`: Enables purge cache
|
||||
* `provider`: Which one of the [purge cache strategy](#purge-cache-strategy) to use.
|
||||
|
||||
## :media_preview_proxy
|
||||
|
||||
* `enabled`: Enables proxying of remote media preview to the instance’s proxy. Requires enabled media proxy (`media_proxy/enabled`).
|
||||
* `thumbnail_max_width`: Max width of preview thumbnail for images (video preview always has original dimensions).
|
||||
* `thumbnail_max_height`: Max height of preview thumbnail for images (video preview always has original dimensions).
|
||||
* `image_quality`: Quality of the output. Ranges from 0 (min quality) to 100 (max quality).
|
||||
* `min_content_length`: Min content length to perform preview, in bytes. If greater than 0, media smaller in size will be served as is, without thumbnailing.
|
||||
|
||||
### Purge cache strategy
|
||||
|
||||
#### Pleroma.Web.MediaProxy.Invalidation.Script
|
||||
|
|
|
|||
150
lib/pleroma/helpers/media_helper.ex
Normal file
150
lib/pleroma/helpers/media_helper.ex
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Helpers.MediaHelper do
|
||||
@moduledoc """
|
||||
Handles common media-related operations.
|
||||
"""
|
||||
|
||||
alias Pleroma.HTTP
|
||||
|
||||
def image_resize(url, options) do
|
||||
with executable when is_binary(executable) <- System.find_executable("convert"),
|
||||
{:ok, args} <- prepare_image_resize_args(options),
|
||||
{:ok, env} <- HTTP.get(url, [], pool: :media),
|
||||
{:ok, fifo_path} <- mkfifo() do
|
||||
args = List.flatten([fifo_path, args])
|
||||
run_fifo(fifo_path, env, executable, args)
|
||||
else
|
||||
nil -> {:error, {:convert, :command_not_found}}
|
||||
{:error, _} = error -> error
|
||||
end
|
||||
end
|
||||
|
||||
defp prepare_image_resize_args(
|
||||
%{max_width: max_width, max_height: max_height, format: "png"} = options
|
||||
) do
|
||||
quality = options[:quality] || 85
|
||||
resize = Enum.join([max_width, "x", max_height, ">"])
|
||||
|
||||
args = [
|
||||
"-resize",
|
||||
resize,
|
||||
"-quality",
|
||||
to_string(quality),
|
||||
"png:-"
|
||||
]
|
||||
|
||||
{:ok, args}
|
||||
end
|
||||
|
||||
defp prepare_image_resize_args(%{max_width: max_width, max_height: max_height} = options) do
|
||||
quality = options[:quality] || 85
|
||||
resize = Enum.join([max_width, "x", max_height, ">"])
|
||||
|
||||
args = [
|
||||
"-interlace",
|
||||
"Plane",
|
||||
"-resize",
|
||||
resize,
|
||||
"-quality",
|
||||
to_string(quality),
|
||||
"jpg:-"
|
||||
]
|
||||
|
||||
{:ok, args}
|
||||
end
|
||||
|
||||
defp prepare_image_resize_args(_), do: {:error, :missing_options}
|
||||
|
||||
# Note: video thumbnail is intentionally not resized (always has original dimensions)
|
||||
def video_framegrab(url) do
|
||||
with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
|
||||
{:ok, env} <- HTTP.get(url, [], pool: :media),
|
||||
{:ok, fifo_path} <- mkfifo(),
|
||||
args = [
|
||||
"-y",
|
||||
"-i",
|
||||
fifo_path,
|
||||
"-vframes",
|
||||
"1",
|
||||
"-f",
|
||||
"mjpeg",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-"
|
||||
] do
|
||||
run_fifo(fifo_path, env, executable, args)
|
||||
else
|
||||
nil -> {:error, {:ffmpeg, :command_not_found}}
|
||||
{:error, _} = error -> error
|
||||
end
|
||||
end
|
||||
|
||||
defp run_fifo(fifo_path, env, executable, args) do
|
||||
pid =
|
||||
Port.open({:spawn_executable, executable}, [
|
||||
:use_stdio,
|
||||
:stream,
|
||||
:exit_status,
|
||||
:binary,
|
||||
args: args
|
||||
])
|
||||
|
||||
fifo = Port.open(to_charlist(fifo_path), [:eof, :binary, :stream, :out])
|
||||
fix = Pleroma.Helpers.QtFastStart.fix(env.body)
|
||||
true = Port.command(fifo, fix)
|
||||
:erlang.port_close(fifo)
|
||||
loop_recv(pid)
|
||||
after
|
||||
File.rm(fifo_path)
|
||||
end
|
||||
|
||||
defp mkfifo do
|
||||
path = Path.join(System.tmp_dir!(), "pleroma-media-preview-pipe-#{Ecto.UUID.generate()}")
|
||||
|
||||
case System.cmd("mkfifo", [path]) do
|
||||
{_, 0} ->
|
||||
spawn(fifo_guard(path))
|
||||
{:ok, path}
|
||||
|
||||
{_, err} ->
|
||||
{:error, {:fifo_failed, err}}
|
||||
end
|
||||
end
|
||||
|
||||
defp fifo_guard(path) do
|
||||
pid = self()
|
||||
|
||||
fn ->
|
||||
ref = Process.monitor(pid)
|
||||
|
||||
receive do
|
||||
{:DOWN, ^ref, :process, ^pid, _} ->
|
||||
File.rm(path)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp loop_recv(pid) do
|
||||
loop_recv(pid, <<>>)
|
||||
end
|
||||
|
||||
defp loop_recv(pid, acc) do
|
||||
receive do
|
||||
{^pid, {:data, data}} ->
|
||||
loop_recv(pid, acc <> data)
|
||||
|
||||
{^pid, {:exit_status, 0}} ->
|
||||
{:ok, acc}
|
||||
|
||||
{^pid, {:exit_status, status}} ->
|
||||
{:error, status}
|
||||
after
|
||||
5000 ->
|
||||
:erlang.port_close(pid)
|
||||
{:error, :timeout}
|
||||
end
|
||||
end
|
||||
end
|
||||
131
lib/pleroma/helpers/qt_fast_start.ex
Normal file
131
lib/pleroma/helpers/qt_fast_start.ex
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Helpers.QtFastStart do
|
||||
@moduledoc """
|
||||
(WIP) Converts a "slow start" (data before metadatas) mov/mp4 file to a "fast start" one (metadatas before data).
|
||||
"""
|
||||
|
||||
# TODO: Cleanup and optimizations
|
||||
# Inspirations: https://www.ffmpeg.org/doxygen/3.4/qt-faststart_8c_source.html
|
||||
# https://github.com/danielgtaylor/qtfaststart/blob/master/qtfaststart/processor.py
|
||||
# ISO/IEC 14496-12:2015, ISO/IEC 15444-12:2015
|
||||
# Paracetamol
|
||||
|
||||
def fix(<<0x00, 0x00, 0x00, _, 0x66, 0x74, 0x79, 0x70, _::bits>> = binary) do
|
||||
index = fix(binary, 0, nil, nil, [])
|
||||
|
||||
case index do
|
||||
:abort -> binary
|
||||
[{"ftyp", _, _, _, _}, {"mdat", _, _, _, _} | _] -> faststart(index)
|
||||
[{"ftyp", _, _, _, _}, {"free", _, _, _, _}, {"mdat", _, _, _, _} | _] -> faststart(index)
|
||||
_ -> binary
|
||||
end
|
||||
end
|
||||
|
||||
def fix(binary) do
|
||||
binary
|
||||
end
|
||||
|
||||
# MOOV have been seen before MDAT- abort
|
||||
defp fix(<<_::bits>>, _, true, false, _) do
|
||||
:abort
|
||||
end
|
||||
|
||||
defp fix(
|
||||
<<size::integer-big-size(32), fourcc::bits-size(32), rest::bits>>,
|
||||
pos,
|
||||
got_moov,
|
||||
got_mdat,
|
||||
acc
|
||||
) do
|
||||
full_size = (size - 8) * 8
|
||||
<<data::bits-size(full_size), rest::bits>> = rest
|
||||
|
||||
acc = [
|
||||
{fourcc, pos, pos + size, size,
|
||||
<<size::integer-big-size(32), fourcc::bits-size(32), data::bits>>}
|
||||
| acc
|
||||
]
|
||||
|
||||
fix(rest, pos + size, got_moov || fourcc == "moov", got_mdat || fourcc == "mdat", acc)
|
||||
end
|
||||
|
||||
defp fix(<<>>, _pos, _, _, acc) do
|
||||
:lists.reverse(acc)
|
||||
end
|
||||
|
||||
defp faststart(index) do
|
||||
{{_ftyp, _, _, _, ftyp}, index} = List.keytake(index, "ftyp", 0)
|
||||
|
||||
# Skip re-writing the free fourcc as it's kind of useless.
|
||||
# Why stream useless bytes when you can do without?
|
||||
{free_size, index} =
|
||||
case List.keytake(index, "free", 0) do
|
||||
{{_, _, _, size, _}, index} -> {size, index}
|
||||
_ -> {0, index}
|
||||
end
|
||||
|
||||
{{_moov, _, _, moov_size, moov}, index} = List.keytake(index, "moov", 0)
|
||||
offset = -free_size + moov_size
|
||||
rest = for {_, _, _, _, data} <- index, do: data, into: []
|
||||
<<moov_head::bits-size(64), moov_data::bits>> = moov
|
||||
[ftyp, moov_head, fix_moov(moov_data, offset, []), rest]
|
||||
end
|
||||
|
||||
defp fix_moov(
|
||||
<<size::integer-big-size(32), fourcc::bits-size(32), rest::bits>>,
|
||||
offset,
|
||||
acc
|
||||
) do
|
||||
full_size = (size - 8) * 8
|
||||
<<data::bits-size(full_size), rest::bits>> = rest
|
||||
|
||||
data =
|
||||
cond do
|
||||
fourcc in ["trak", "mdia", "minf", "stbl"] ->
|
||||
# Theses contains sto or co64 part
|
||||
[<<size::integer-big-size(32), fourcc::bits-size(32)>>, fix_moov(data, offset, [])]
|
||||
|
||||
fourcc in ["stco", "co64"] ->
|
||||
# fix the damn thing
|
||||
<<version::integer-big-size(32), count::integer-big-size(32), rest::bits>> = data
|
||||
|
||||
entry_size =
|
||||
case fourcc do
|
||||
"stco" -> 32
|
||||
"co64" -> 64
|
||||
end
|
||||
|
||||
[
|
||||
<<size::integer-big-size(32), fourcc::bits-size(32), version::integer-big-size(32),
|
||||
count::integer-big-size(32)>>,
|
||||
rewrite_entries(entry_size, offset, rest, [])
|
||||
]
|
||||
|
||||
true ->
|
||||
[<<size::integer-big-size(32), fourcc::bits-size(32)>>, data]
|
||||
end
|
||||
|
||||
acc = [acc | data]
|
||||
fix_moov(rest, offset, acc)
|
||||
end
|
||||
|
||||
defp fix_moov(<<>>, _, acc), do: acc
|
||||
|
||||
for size <- [32, 64] do
|
||||
defp rewrite_entries(
|
||||
unquote(size),
|
||||
offset,
|
||||
<<pos::integer-big-size(unquote(size)), rest::bits>>,
|
||||
acc
|
||||
) do
|
||||
rewrite_entries(unquote(size), offset, rest, [
|
||||
acc | <<pos + offset::integer-big-size(unquote(size))>>
|
||||
])
|
||||
end
|
||||
end
|
||||
|
||||
defp rewrite_entries(_, _, <<>>, acc), do: acc
|
||||
end
|
||||
|
|
@ -3,18 +3,22 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Helpers.UriHelper do
|
||||
def append_uri_params(uri, appended_params) do
|
||||
def modify_uri_params(uri, overridden_params, deleted_params \\ []) do
|
||||
uri = URI.parse(uri)
|
||||
appended_params = for {k, v} <- appended_params, into: %{}, do: {to_string(k), v}
|
||||
existing_params = URI.query_decoder(uri.query || "") |> Enum.into(%{})
|
||||
updated_params_keys = Enum.uniq(Map.keys(existing_params) ++ Map.keys(appended_params))
|
||||
|
||||
existing_params = URI.query_decoder(uri.query || "") |> Map.new()
|
||||
overridden_params = Map.new(overridden_params, fn {k, v} -> {to_string(k), v} end)
|
||||
deleted_params = Enum.map(deleted_params, &to_string/1)
|
||||
|
||||
updated_params =
|
||||
for k <- updated_params_keys, do: {k, appended_params[k] || existing_params[k]}
|
||||
existing_params
|
||||
|> Map.merge(overridden_params)
|
||||
|> Map.drop(deleted_params)
|
||||
|
||||
uri
|
||||
|> Map.put(:query, URI.encode_query(updated_params))
|
||||
|> URI.to_string()
|
||||
|> String.replace_suffix("?", "")
|
||||
end
|
||||
|
||||
def maybe_add_base("/" <> uri, base), do: Path.join([base, uri])
|
||||
|
|
|
|||
|
|
@ -156,9 +156,7 @@ def get_or_update_favicon(%URI{host: host} = instance_uri) do
|
|||
defp scrape_favicon(%URI{} = instance_uri) do
|
||||
try do
|
||||
with {:ok, %Tesla.Env{body: html}} <-
|
||||
Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}],
|
||||
adapter: [pool: :media]
|
||||
),
|
||||
Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], pool: :media),
|
||||
{_, [favicon_rel | _]} when is_binary(favicon_rel) <-
|
||||
{:parse,
|
||||
html |> Floki.parse_document!() |> Floki.attribute("link[rel=icon]", "href")},
|
||||
|
|
|
|||
|
|
@ -17,6 +17,9 @@ defmodule Pleroma.ReverseProxy do
|
|||
@failed_request_ttl :timer.seconds(60)
|
||||
@methods ~w(GET HEAD)
|
||||
|
||||
def max_read_duration_default, do: @max_read_duration
|
||||
def default_cache_control_header, do: @default_cache_control_header
|
||||
|
||||
@moduledoc """
|
||||
A reverse proxy.
|
||||
|
||||
|
|
@ -391,6 +394,8 @@ defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and
|
|||
|
||||
defp body_size_constraint(_, _), do: :ok
|
||||
|
||||
defp check_read_duration(nil = _duration, max), do: check_read_duration(@max_read_duration, max)
|
||||
|
||||
defp check_read_duration(duration, max)
|
||||
when is_integer(duration) and is_integer(max) and max > 0 do
|
||||
if duration > max do
|
||||
|
|
|
|||
|
|
@ -12,17 +12,21 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
|
|||
|
||||
require Logger
|
||||
|
||||
@options [
|
||||
@adapter_options [
|
||||
pool: :media,
|
||||
recv_timeout: 10_000
|
||||
]
|
||||
|
||||
def perform(:prefetch, url) do
|
||||
Logger.debug("Prefetching #{inspect(url)}")
|
||||
# Fetching only proxiable resources
|
||||
if MediaProxy.enabled?() and MediaProxy.url_proxiable?(url) do
|
||||
# If preview proxy is enabled, it'll also hit media proxy (so we're caching both requests)
|
||||
prefetch_url = MediaProxy.preview_url(url)
|
||||
|
||||
url
|
||||
|> MediaProxy.url()
|
||||
|> HTTP.get([], @options)
|
||||
Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}")
|
||||
|
||||
HTTP.get(prefetch_url, [], @adapter_options)
|
||||
end
|
||||
end
|
||||
|
||||
def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do
|
||||
|
|
|
|||
|
|
@ -181,8 +181,10 @@ defp do_render("show.json", %{user: user} = opts) do
|
|||
user = User.sanitize_html(user, User.html_filter_policy(opts[:for]))
|
||||
display_name = user.name || user.nickname
|
||||
|
||||
image = User.avatar_url(user) |> MediaProxy.url()
|
||||
avatar = User.avatar_url(user) |> MediaProxy.url()
|
||||
avatar_static = User.avatar_url(user) |> MediaProxy.preview_url(static: true)
|
||||
header = User.banner_url(user) |> MediaProxy.url()
|
||||
header_static = User.banner_url(user) |> MediaProxy.preview_url(static: true)
|
||||
|
||||
following_count =
|
||||
if !user.hide_follows_count or !user.hide_follows or opts[:for] == user do
|
||||
|
|
@ -247,10 +249,10 @@ defp do_render("show.json", %{user: user} = opts) do
|
|||
statuses_count: user.note_count,
|
||||
note: user.bio,
|
||||
url: user.uri || user.ap_id,
|
||||
avatar: image,
|
||||
avatar_static: image,
|
||||
avatar: avatar,
|
||||
avatar_static: avatar_static,
|
||||
header: header,
|
||||
header_static: header,
|
||||
header_static: header_static,
|
||||
emojis: emojis,
|
||||
fields: user.fields,
|
||||
bot: bot,
|
||||
|
|
|
|||
|
|
@ -415,6 +415,7 @@ def render("attachment.json", %{attachment: attachment}) do
|
|||
[attachment_url | _] = attachment["url"]
|
||||
media_type = attachment_url["mediaType"] || attachment_url["mimeType"] || "image"
|
||||
href = attachment_url["href"] |> MediaProxy.url()
|
||||
href_preview = attachment_url["href"] |> MediaProxy.preview_url()
|
||||
|
||||
type =
|
||||
cond do
|
||||
|
|
@ -430,7 +431,7 @@ def render("attachment.json", %{attachment: attachment}) do
|
|||
id: to_string(attachment["id"] || hash_id),
|
||||
url: href,
|
||||
remote_url: href,
|
||||
preview_url: href,
|
||||
preview_url: href_preview,
|
||||
text_url: href,
|
||||
type: type,
|
||||
description: attachment["name"],
|
||||
|
|
|
|||
|
|
@ -33,6 +33,8 @@ defp do_purge(urls) do
|
|||
def prepare_urls(urls) do
|
||||
urls
|
||||
|> List.wrap()
|
||||
|> Enum.map(&MediaProxy.url/1)
|
||||
|> Enum.map(fn url -> [MediaProxy.url(url), MediaProxy.preview_url(url)] end)
|
||||
|> List.flatten()
|
||||
|> Enum.uniq()
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
|
||||
defmodule Pleroma.Web.MediaProxy do
|
||||
alias Pleroma.Config
|
||||
alias Pleroma.Helpers.UriHelper
|
||||
alias Pleroma.Upload
|
||||
alias Pleroma.Web
|
||||
alias Pleroma.Web.MediaProxy.Invalidation
|
||||
|
|
@ -40,27 +41,35 @@ def url(url) when is_nil(url) or url == "", do: nil
|
|||
def url("/" <> _ = url), do: url
|
||||
|
||||
def url(url) do
|
||||
if disabled?() or not url_proxiable?(url) do
|
||||
url
|
||||
else
|
||||
if enabled?() and url_proxiable?(url) do
|
||||
encode_url(url)
|
||||
else
|
||||
url
|
||||
end
|
||||
end
|
||||
|
||||
@spec url_proxiable?(String.t()) :: boolean()
|
||||
def url_proxiable?(url) do
|
||||
if local?(url) or whitelisted?(url) do
|
||||
false
|
||||
not local?(url) and not whitelisted?(url)
|
||||
end
|
||||
|
||||
def preview_url(url, preview_params \\ []) do
|
||||
if preview_enabled?() do
|
||||
encode_preview_url(url, preview_params)
|
||||
else
|
||||
true
|
||||
url(url)
|
||||
end
|
||||
end
|
||||
|
||||
defp disabled?, do: !Config.get([:media_proxy, :enabled], false)
|
||||
def enabled?, do: Config.get([:media_proxy, :enabled], false)
|
||||
|
||||
defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())
|
||||
# Note: media proxy must be enabled for media preview proxy in order to load all
|
||||
# non-local non-whitelisted URLs through it and be sure that body size constraint is preserved.
|
||||
def preview_enabled?, do: enabled?() and !!Config.get([:media_preview_proxy, :enabled])
|
||||
|
||||
defp whitelisted?(url) do
|
||||
def local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())
|
||||
|
||||
def whitelisted?(url) do
|
||||
%{host: domain} = URI.parse(url)
|
||||
|
||||
mediaproxy_whitelist_domains =
|
||||
|
|
@ -85,17 +94,29 @@ defp maybe_get_domain_from_url("http" <> _ = url) do
|
|||
|
||||
defp maybe_get_domain_from_url(domain), do: domain
|
||||
|
||||
def encode_url(url) do
|
||||
defp base64_sig64(url) do
|
||||
base64 = Base.url_encode64(url, @base64_opts)
|
||||
|
||||
sig64 =
|
||||
base64
|
||||
|> signed_url
|
||||
|> signed_url()
|
||||
|> Base.url_encode64(@base64_opts)
|
||||
|
||||
{base64, sig64}
|
||||
end
|
||||
|
||||
def encode_url(url) do
|
||||
{base64, sig64} = base64_sig64(url)
|
||||
|
||||
build_url(sig64, base64, filename(url))
|
||||
end
|
||||
|
||||
def encode_preview_url(url, preview_params \\ []) do
|
||||
{base64, sig64} = base64_sig64(url)
|
||||
|
||||
build_preview_url(sig64, base64, filename(url), preview_params)
|
||||
end
|
||||
|
||||
def decode_url(sig, url) do
|
||||
with {:ok, sig} <- Base.url_decode64(sig, @base64_opts),
|
||||
signature when signature == sig <- signed_url(url) do
|
||||
|
|
@ -113,10 +134,14 @@ def filename(url_or_path) do
|
|||
if path = URI.parse(url_or_path).path, do: Path.basename(path)
|
||||
end
|
||||
|
||||
def build_url(sig_base64, url_base64, filename \\ nil) do
|
||||
def base_url do
|
||||
Config.get([:media_proxy, :base_url], Web.base_url())
|
||||
end
|
||||
|
||||
defp proxy_url(path, sig_base64, url_base64, filename) do
|
||||
[
|
||||
Config.get([:media_proxy, :base_url], Web.base_url()),
|
||||
"proxy",
|
||||
base_url(),
|
||||
path,
|
||||
sig_base64,
|
||||
url_base64,
|
||||
filename
|
||||
|
|
@ -124,4 +149,38 @@ def build_url(sig_base64, url_base64, filename \\ nil) do
|
|||
|> Enum.filter(& &1)
|
||||
|> Path.join()
|
||||
end
|
||||
|
||||
def build_url(sig_base64, url_base64, filename \\ nil) do
|
||||
proxy_url("proxy", sig_base64, url_base64, filename)
|
||||
end
|
||||
|
||||
def build_preview_url(sig_base64, url_base64, filename \\ nil, preview_params \\ []) do
|
||||
uri = proxy_url("proxy/preview", sig_base64, url_base64, filename)
|
||||
|
||||
UriHelper.modify_uri_params(uri, preview_params)
|
||||
end
|
||||
|
||||
def verify_request_path_and_url(
|
||||
%Plug.Conn{params: %{"filename" => _}, request_path: request_path},
|
||||
url
|
||||
) do
|
||||
verify_request_path_and_url(request_path, url)
|
||||
end
|
||||
|
||||
def verify_request_path_and_url(request_path, url) when is_binary(request_path) do
|
||||
filename = filename(url)
|
||||
|
||||
if filename && not basename_matches?(request_path, filename) do
|
||||
{:wrong_filename, filename}
|
||||
else
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
def verify_request_path_and_url(_, _), do: :ok
|
||||
|
||||
defp basename_matches?(path, filename) do
|
||||
basename = Path.basename(path)
|
||||
basename == filename or URI.decode(basename) == filename or URI.encode(basename) == filename
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -5,44 +5,201 @@
|
|||
defmodule Pleroma.Web.MediaProxy.MediaProxyController do
|
||||
use Pleroma.Web, :controller
|
||||
|
||||
alias Pleroma.Config
|
||||
alias Pleroma.Helpers.MediaHelper
|
||||
alias Pleroma.Helpers.UriHelper
|
||||
alias Pleroma.ReverseProxy
|
||||
alias Pleroma.Web.MediaProxy
|
||||
alias Plug.Conn
|
||||
|
||||
@default_proxy_opts [max_body_length: 25 * 1_048_576, http: [follow_redirect: true]]
|
||||
|
||||
def remote(conn, %{"sig" => sig64, "url" => url64} = params) do
|
||||
with config <- Pleroma.Config.get([:media_proxy], []),
|
||||
true <- Keyword.get(config, :enabled, false),
|
||||
def remote(conn, %{"sig" => sig64, "url" => url64}) do
|
||||
with {_, true} <- {:enabled, MediaProxy.enabled?()},
|
||||
{:ok, url} <- MediaProxy.decode_url(sig64, url64),
|
||||
{_, false} <- {:in_banned_urls, MediaProxy.in_banned_urls(url)},
|
||||
:ok <- filename_matches(params, conn.request_path, url) do
|
||||
ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts))
|
||||
:ok <- MediaProxy.verify_request_path_and_url(conn, url) do
|
||||
ReverseProxy.call(conn, url, media_proxy_opts())
|
||||
else
|
||||
error when error in [false, {:in_banned_urls, true}] ->
|
||||
send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404))
|
||||
{:enabled, false} ->
|
||||
send_resp(conn, 404, Conn.Status.reason_phrase(404))
|
||||
|
||||
{:in_banned_urls, true} ->
|
||||
send_resp(conn, 404, Conn.Status.reason_phrase(404))
|
||||
|
||||
{:error, :invalid_signature} ->
|
||||
send_resp(conn, 403, Plug.Conn.Status.reason_phrase(403))
|
||||
send_resp(conn, 403, Conn.Status.reason_phrase(403))
|
||||
|
||||
{:wrong_filename, filename} ->
|
||||
redirect(conn, external: MediaProxy.build_url(sig64, url64, filename))
|
||||
end
|
||||
end
|
||||
|
||||
def filename_matches(%{"filename" => _} = _, path, url) do
|
||||
filename = MediaProxy.filename(url)
|
||||
|
||||
if filename && does_not_match(path, filename) do
|
||||
{:wrong_filename, filename}
|
||||
def preview(%Conn{} = conn, %{"sig" => sig64, "url" => url64}) do
|
||||
with {_, true} <- {:enabled, MediaProxy.preview_enabled?()},
|
||||
{:ok, url} <- MediaProxy.decode_url(sig64, url64),
|
||||
:ok <- MediaProxy.verify_request_path_and_url(conn, url) do
|
||||
handle_preview(conn, url)
|
||||
else
|
||||
:ok
|
||||
{:enabled, false} ->
|
||||
send_resp(conn, 404, Conn.Status.reason_phrase(404))
|
||||
|
||||
{:error, :invalid_signature} ->
|
||||
send_resp(conn, 403, Conn.Status.reason_phrase(403))
|
||||
|
||||
{:wrong_filename, filename} ->
|
||||
redirect(conn, external: MediaProxy.build_preview_url(sig64, url64, filename))
|
||||
end
|
||||
end
|
||||
|
||||
def filename_matches(_, _, _), do: :ok
|
||||
defp handle_preview(conn, url) do
|
||||
media_proxy_url = MediaProxy.url(url)
|
||||
|
||||
defp does_not_match(path, filename) do
|
||||
basename = Path.basename(path)
|
||||
basename != filename and URI.decode(basename) != filename and URI.encode(basename) != filename
|
||||
with {:ok, %{status: status} = head_response} when status in 200..299 <-
|
||||
Pleroma.HTTP.request("head", media_proxy_url, [], [], pool: :media) do
|
||||
content_type = Tesla.get_header(head_response, "content-type")
|
||||
content_length = Tesla.get_header(head_response, "content-length")
|
||||
content_length = content_length && String.to_integer(content_length)
|
||||
static = conn.params["static"] in ["true", true]
|
||||
|
||||
cond do
|
||||
static and content_type == "image/gif" ->
|
||||
handle_jpeg_preview(conn, media_proxy_url)
|
||||
|
||||
static ->
|
||||
drop_static_param_and_redirect(conn)
|
||||
|
||||
content_type == "image/gif" ->
|
||||
redirect(conn, external: media_proxy_url)
|
||||
|
||||
min_content_length_for_preview() > 0 and content_length > 0 and
|
||||
content_length < min_content_length_for_preview() ->
|
||||
redirect(conn, external: media_proxy_url)
|
||||
|
||||
true ->
|
||||
handle_preview(content_type, conn, media_proxy_url)
|
||||
end
|
||||
else
|
||||
# If HEAD failed, redirecting to media proxy URI doesn't make much sense; returning an error
|
||||
{_, %{status: status}} ->
|
||||
send_resp(conn, :failed_dependency, "Can't fetch HTTP headers (HTTP #{status}).")
|
||||
|
||||
{:error, :recv_response_timeout} ->
|
||||
send_resp(conn, :failed_dependency, "HEAD request timeout.")
|
||||
|
||||
_ ->
|
||||
send_resp(conn, :failed_dependency, "Can't fetch HTTP headers.")
|
||||
end
|
||||
end
|
||||
|
||||
defp handle_preview("image/png" <> _ = _content_type, conn, media_proxy_url) do
|
||||
handle_png_preview(conn, media_proxy_url)
|
||||
end
|
||||
|
||||
defp handle_preview("image/" <> _ = _content_type, conn, media_proxy_url) do
|
||||
handle_jpeg_preview(conn, media_proxy_url)
|
||||
end
|
||||
|
||||
defp handle_preview("video/" <> _ = _content_type, conn, media_proxy_url) do
|
||||
handle_video_preview(conn, media_proxy_url)
|
||||
end
|
||||
|
||||
defp handle_preview(_unsupported_content_type, conn, media_proxy_url) do
|
||||
fallback_on_preview_error(conn, media_proxy_url)
|
||||
end
|
||||
|
||||
defp handle_png_preview(conn, media_proxy_url) do
|
||||
quality = Config.get!([:media_preview_proxy, :image_quality])
|
||||
{thumbnail_max_width, thumbnail_max_height} = thumbnail_max_dimensions()
|
||||
|
||||
with {:ok, thumbnail_binary} <-
|
||||
MediaHelper.image_resize(
|
||||
media_proxy_url,
|
||||
%{
|
||||
max_width: thumbnail_max_width,
|
||||
max_height: thumbnail_max_height,
|
||||
quality: quality,
|
||||
format: "png"
|
||||
}
|
||||
) do
|
||||
conn
|
||||
|> put_preview_response_headers(["image/png", "preview.png"])
|
||||
|> send_resp(200, thumbnail_binary)
|
||||
else
|
||||
_ ->
|
||||
fallback_on_preview_error(conn, media_proxy_url)
|
||||
end
|
||||
end
|
||||
|
||||
defp handle_jpeg_preview(conn, media_proxy_url) do
|
||||
quality = Config.get!([:media_preview_proxy, :image_quality])
|
||||
{thumbnail_max_width, thumbnail_max_height} = thumbnail_max_dimensions()
|
||||
|
||||
with {:ok, thumbnail_binary} <-
|
||||
MediaHelper.image_resize(
|
||||
media_proxy_url,
|
||||
%{max_width: thumbnail_max_width, max_height: thumbnail_max_height, quality: quality}
|
||||
) do
|
||||
conn
|
||||
|> put_preview_response_headers()
|
||||
|> send_resp(200, thumbnail_binary)
|
||||
else
|
||||
_ ->
|
||||
fallback_on_preview_error(conn, media_proxy_url)
|
||||
end
|
||||
end
|
||||
|
||||
defp handle_video_preview(conn, media_proxy_url) do
|
||||
with {:ok, thumbnail_binary} <-
|
||||
MediaHelper.video_framegrab(media_proxy_url) do
|
||||
conn
|
||||
|> put_preview_response_headers()
|
||||
|> send_resp(200, thumbnail_binary)
|
||||
else
|
||||
_ ->
|
||||
fallback_on_preview_error(conn, media_proxy_url)
|
||||
end
|
||||
end
|
||||
|
||||
defp drop_static_param_and_redirect(conn) do
|
||||
uri_without_static_param =
|
||||
conn
|
||||
|> current_url()
|
||||
|> UriHelper.modify_uri_params(%{}, ["static"])
|
||||
|
||||
redirect(conn, external: uri_without_static_param)
|
||||
end
|
||||
|
||||
defp fallback_on_preview_error(conn, media_proxy_url) do
|
||||
redirect(conn, external: media_proxy_url)
|
||||
end
|
||||
|
||||
defp put_preview_response_headers(
|
||||
conn,
|
||||
[content_type, filename] = _content_info \\ ["image/jpeg", "preview.jpg"]
|
||||
) do
|
||||
conn
|
||||
|> put_resp_header("content-type", content_type)
|
||||
|> put_resp_header("content-disposition", "inline; filename=\"#{filename}\"")
|
||||
|> put_resp_header("cache-control", ReverseProxy.default_cache_control_header())
|
||||
end
|
||||
|
||||
defp thumbnail_max_dimensions do
|
||||
config = media_preview_proxy_config()
|
||||
|
||||
thumbnail_max_width = Keyword.fetch!(config, :thumbnail_max_width)
|
||||
thumbnail_max_height = Keyword.fetch!(config, :thumbnail_max_height)
|
||||
|
||||
{thumbnail_max_width, thumbnail_max_height}
|
||||
end
|
||||
|
||||
defp min_content_length_for_preview do
|
||||
Keyword.get(media_preview_proxy_config(), :min_content_length, 0)
|
||||
end
|
||||
|
||||
defp media_preview_proxy_config do
|
||||
Config.get!([:media_preview_proxy])
|
||||
end
|
||||
|
||||
defp media_proxy_opts do
|
||||
Config.get([:media_proxy, :proxy_opts], [])
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ def scrub_html(content) when is_binary(content) do
|
|||
def scrub_html(content), do: content
|
||||
|
||||
def attachment_url(url) do
|
||||
MediaProxy.url(url)
|
||||
MediaProxy.preview_url(url)
|
||||
end
|
||||
|
||||
def user_name_string(user) do
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ defp handle_existing_authorization(
|
|||
redirect_uri = redirect_uri(conn, redirect_uri)
|
||||
url_params = %{access_token: token.token}
|
||||
url_params = Maps.put_if_present(url_params, :state, params["state"])
|
||||
url = UriHelper.append_uri_params(redirect_uri, url_params)
|
||||
url = UriHelper.modify_uri_params(redirect_uri, url_params)
|
||||
redirect(conn, external: url)
|
||||
else
|
||||
conn
|
||||
|
|
@ -161,7 +161,7 @@ def after_create_authorization(%Plug.Conn{} = conn, %Authorization{} = auth, %{
|
|||
redirect_uri = redirect_uri(conn, redirect_uri)
|
||||
url_params = %{code: auth.token}
|
||||
url_params = Maps.put_if_present(url_params, :state, auth_attrs["state"])
|
||||
url = UriHelper.append_uri_params(redirect_uri, url_params)
|
||||
url = UriHelper.modify_uri_params(redirect_uri, url_params)
|
||||
redirect(conn, external: url)
|
||||
else
|
||||
conn
|
||||
|
|
|
|||
|
|
@ -679,6 +679,8 @@ defmodule Pleroma.Web.Router do
|
|||
end
|
||||
|
||||
scope "/proxy/", Pleroma.Web.MediaProxy do
|
||||
get("/preview/:sig/:url", MediaProxyController, :preview)
|
||||
get("/preview/:sig/:url/:filename", MediaProxyController, :preview)
|
||||
get("/:sig/:url", MediaProxyController, :remote)
|
||||
get("/:sig/:url/:filename", MediaProxyController, :remote)
|
||||
end
|
||||
|
|
|
|||
2
mix.lock
2
mix.lock
|
|
@ -31,6 +31,7 @@
|
|||
"ecto": {:hex, :ecto, "3.4.5", "2bcd262f57b2c888b0bd7f7a28c8a48aa11dc1a2c6a858e45dd8f8426d504265", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8c6d1d4d524559e9b7a062f0498e2c206122552d63eacff0a6567ffe7a8e8691"},
|
||||
"ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"},
|
||||
"ecto_sql": {:hex, :ecto_sql, "3.4.5", "30161f81b167d561a9a2df4329c10ae05ff36eca7ccc84628f2c8b9fa1e43323", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0 or ~> 0.4.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.0", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "31990c6a3579b36a3c0841d34a94c275e727de8b84f58509da5f1b2032c98ac2"},
|
||||
"eimp": {:hex, :eimp, "1.0.14", "fc297f0c7e2700457a95a60c7010a5f1dcb768a083b6d53f49cd94ab95a28f22", [:rebar3], [{:p1_utils, "1.0.18", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm", "501133f3112079b92d9e22da8b88bf4f0e13d4d67ae9c15c42c30bd25ceb83b6"},
|
||||
"elixir_make": {:hex, :elixir_make, "0.6.0", "38349f3e29aff4864352084fc736fa7fa0f2995a819a737554f7ebd28b85aaab", [:mix], [], "hexpm", "d522695b93b7f0b4c0fcb2dfe73a6b905b1c301226a5a55cb42e5b14d509e050"},
|
||||
"esshd": {:hex, :esshd, "0.1.1", "d4dd4c46698093a40a56afecce8a46e246eb35463c457c246dacba2e056f31b5", [:mix], [], "hexpm", "d73e341e3009d390aa36387dc8862860bf9f874c94d9fd92ade2926376f49981"},
|
||||
"eternal": {:hex, :eternal, "1.2.1", "d5b6b2499ba876c57be2581b5b999ee9bdf861c647401066d3eeed111d096bc4", [:mix], [], "hexpm", "b14f1dc204321429479c569cfbe8fb287541184ed040956c8862cb7a677b8406"},
|
||||
|
|
@ -80,6 +81,7 @@
|
|||
"nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]},
|
||||
"oban": {:hex, :oban, "2.0.0", "e6ce70d94dd46815ec0882a1ffb7356df9a9d5b8a40a64ce5c2536617a447379", [:mix], [{:ecto_sql, ">= 3.4.3", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cf574813bd048b98a698aa587c21367d2e06842d4e1b1993dcd6a696e9e633bd"},
|
||||
"open_api_spex": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git", "f296ac0924ba3cf79c7a588c4c252889df4c2edd", [ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"]},
|
||||
"p1_utils": {:hex, :p1_utils, "1.0.18", "3fe224de5b2e190d730a3c5da9d6e8540c96484cf4b4692921d1e28f0c32b01c", [:rebar3], [], "hexpm", "1fc8773a71a15553b179c986b22fbeead19b28fe486c332d4929700ffeb71f88"},
|
||||
"parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"},
|
||||
"pbkdf2_elixir": {:hex, :pbkdf2_elixir, "1.2.1", "9cbe354b58121075bd20eb83076900a3832324b7dd171a6895fab57b6bb2752c", [:mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}], "hexpm", "d3b40a4a4630f0b442f19eca891fcfeeee4c40871936fed2f68e1c4faa30481f"},
|
||||
"phoenix": {:hex, :phoenix, "1.4.17", "1b1bd4cff7cfc87c94deaa7d60dd8c22e04368ab95499483c50640ef3bd838d8", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3a8e5d7a3d76d452bb5fb86e8b7bd115f737e4f8efe202a463d4aeb4a5809611"},
|
||||
|
|
|
|||
BIN
test/fixtures/image.gif
vendored
Executable file
BIN
test/fixtures/image.gif
vendored
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 978 KiB |
BIN
test/fixtures/image.png
vendored
Executable file
BIN
test/fixtures/image.png
vendored
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 102 KiB |
|
|
@ -22,6 +22,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicyTest do
|
|||
}
|
||||
}
|
||||
|
||||
setup do: clear_config([:media_proxy, :enabled], true)
|
||||
|
||||
test "it prefetches media proxy URIs" do
|
||||
with_mock HTTP, get: fn _, _, _ -> {:ok, []} end do
|
||||
MediaProxyWarmingPolicy.filter(@message)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
defmodule Pleroma.Web.MastodonAPI.AccountViewTest do
|
||||
use Pleroma.DataCase
|
||||
|
||||
alias Pleroma.Config
|
||||
alias Pleroma.User
|
||||
alias Pleroma.UserRelationship
|
||||
alias Pleroma.Web.CommonAPI
|
||||
|
|
@ -540,8 +541,9 @@ test "shows non-zero when historical unapproved requests are present" do
|
|||
end
|
||||
end
|
||||
|
||||
test "uses mediaproxy urls when it's enabled" do
|
||||
test "uses mediaproxy urls when it's enabled (regardless of media preview proxy state)" do
|
||||
clear_config([:media_proxy, :enabled], true)
|
||||
clear_config([:media_preview_proxy, :enabled])
|
||||
|
||||
user =
|
||||
insert(:user,
|
||||
|
|
@ -550,20 +552,24 @@ test "uses mediaproxy urls when it's enabled" do
|
|||
emoji: %{"joker_smile" => "https://evil.website/society.png"}
|
||||
)
|
||||
|
||||
AccountView.render("show.json", %{user: user, skip_visibility_check: true})
|
||||
|> Enum.all?(fn
|
||||
{key, url} when key in [:avatar, :avatar_static, :header, :header_static] ->
|
||||
String.starts_with?(url, Pleroma.Web.base_url())
|
||||
with media_preview_enabled <- [false, true] do
|
||||
Config.put([:media_preview_proxy, :enabled], media_preview_enabled)
|
||||
|
||||
{:emojis, emojis} ->
|
||||
Enum.all?(emojis, fn %{url: url, static_url: static_url} ->
|
||||
String.starts_with?(url, Pleroma.Web.base_url()) &&
|
||||
String.starts_with?(static_url, Pleroma.Web.base_url())
|
||||
end)
|
||||
AccountView.render("show.json", %{user: user, skip_visibility_check: true})
|
||||
|> Enum.all?(fn
|
||||
{key, url} when key in [:avatar, :avatar_static, :header, :header_static] ->
|
||||
String.starts_with?(url, Pleroma.Web.base_url())
|
||||
|
||||
_ ->
|
||||
true
|
||||
end)
|
||||
|> assert()
|
||||
{:emojis, emojis} ->
|
||||
Enum.all?(emojis, fn %{url: url, static_url: static_url} ->
|
||||
String.starts_with?(url, Pleroma.Web.base_url()) &&
|
||||
String.starts_with?(static_url, Pleroma.Web.base_url())
|
||||
end)
|
||||
|
||||
_ ->
|
||||
true
|
||||
end)
|
||||
|> assert()
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -8,34 +8,34 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do
|
|||
import Mock
|
||||
|
||||
alias Pleroma.Web.MediaProxy
|
||||
alias Pleroma.Web.MediaProxy.MediaProxyController
|
||||
alias Plug.Conn
|
||||
|
||||
setup do
|
||||
on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
|
||||
end
|
||||
|
||||
test "it returns 404 when MediaProxy disabled", %{conn: conn} do
|
||||
clear_config([:media_proxy, :enabled], false)
|
||||
|
||||
assert %Conn{
|
||||
status: 404,
|
||||
resp_body: "Not Found"
|
||||
} = get(conn, "/proxy/hhgfh/eeeee")
|
||||
|
||||
assert %Conn{
|
||||
status: 404,
|
||||
resp_body: "Not Found"
|
||||
} = get(conn, "/proxy/hhgfh/eeee/fff")
|
||||
end
|
||||
|
||||
describe "" do
|
||||
describe "Media Proxy" do
|
||||
setup do
|
||||
clear_config([:media_proxy, :enabled], true)
|
||||
clear_config([Pleroma.Web.Endpoint, :secret_key_base], "00000000000")
|
||||
|
||||
[url: MediaProxy.encode_url("https://google.fn/test.png")]
|
||||
end
|
||||
|
||||
test "it returns 404 when disabled", %{conn: conn} do
|
||||
clear_config([:media_proxy, :enabled], false)
|
||||
|
||||
assert %Conn{
|
||||
status: 404,
|
||||
resp_body: "Not Found"
|
||||
} = get(conn, "/proxy/hhgfh/eeeee")
|
||||
|
||||
assert %Conn{
|
||||
status: 404,
|
||||
resp_body: "Not Found"
|
||||
} = get(conn, "/proxy/hhgfh/eeee/fff")
|
||||
end
|
||||
|
||||
test "it returns 403 for invalid signature", %{conn: conn, url: url} do
|
||||
Pleroma.Config.put([Pleroma.Web.Endpoint, :secret_key_base], "000")
|
||||
%{path: path} = URI.parse(url)
|
||||
|
|
@ -56,7 +56,7 @@ test "it returns 403 for invalid signature", %{conn: conn, url: url} do
|
|||
} = get(conn, "/proxy/hhgfh/eeee/fff")
|
||||
end
|
||||
|
||||
test "redirects on valid url when filename is invalidated", %{conn: conn, url: url} do
|
||||
test "redirects to valid url when filename is invalidated", %{conn: conn, url: url} do
|
||||
invalid_url = String.replace(url, "test.png", "test-file.png")
|
||||
response = get(conn, invalid_url)
|
||||
assert response.status == 302
|
||||
|
|
@ -80,42 +80,248 @@ test "it returns 404 when url is in banned_urls cache", %{conn: conn, url: url}
|
|||
end
|
||||
end
|
||||
|
||||
describe "filename_matches/3" do
|
||||
test "preserves the encoded or decoded path" do
|
||||
assert MediaProxyController.filename_matches(
|
||||
%{"filename" => "/Hello world.jpg"},
|
||||
"/Hello world.jpg",
|
||||
"http://pleroma.social/Hello world.jpg"
|
||||
) == :ok
|
||||
describe "Media Preview Proxy" do
|
||||
setup do
|
||||
clear_config([:media_proxy, :enabled], true)
|
||||
clear_config([:media_preview_proxy, :enabled], true)
|
||||
clear_config([Pleroma.Web.Endpoint, :secret_key_base], "00000000000")
|
||||
|
||||
assert MediaProxyController.filename_matches(
|
||||
%{"filename" => "/Hello%20world.jpg"},
|
||||
"/Hello%20world.jpg",
|
||||
"http://pleroma.social/Hello%20world.jpg"
|
||||
) == :ok
|
||||
original_url = "https://google.fn/test.png"
|
||||
|
||||
assert MediaProxyController.filename_matches(
|
||||
%{"filename" => "/my%2Flong%2Furl%2F2019%2F07%2FS.jpg"},
|
||||
"/my%2Flong%2Furl%2F2019%2F07%2FS.jpg",
|
||||
"http://pleroma.social/my%2Flong%2Furl%2F2019%2F07%2FS.jpg"
|
||||
) == :ok
|
||||
|
||||
assert MediaProxyController.filename_matches(
|
||||
%{"filename" => "/my%2Flong%2Furl%2F2019%2F07%2FS.jp"},
|
||||
"/my%2Flong%2Furl%2F2019%2F07%2FS.jp",
|
||||
"http://pleroma.social/my%2Flong%2Furl%2F2019%2F07%2FS.jpg"
|
||||
) == {:wrong_filename, "my%2Flong%2Furl%2F2019%2F07%2FS.jpg"}
|
||||
[
|
||||
url: MediaProxy.encode_preview_url(original_url),
|
||||
media_proxy_url: MediaProxy.encode_url(original_url)
|
||||
]
|
||||
end
|
||||
|
||||
test "encoded url are tried to match for proxy as `conn.request_path` encodes the url" do
|
||||
# conn.request_path will return encoded url
|
||||
request_path = "/ANALYSE-DAI-_-LE-STABLECOIN-100-D%C3%89CENTRALIS%C3%89-BQ.jpg"
|
||||
test "returns 404 when media proxy is disabled", %{conn: conn} do
|
||||
clear_config([:media_proxy, :enabled], false)
|
||||
|
||||
assert MediaProxyController.filename_matches(
|
||||
true,
|
||||
request_path,
|
||||
"https://mydomain.com/uploads/2019/07/ANALYSE-DAI-_-LE-STABLECOIN-100-DÉCENTRALISÉ-BQ.jpg"
|
||||
) == :ok
|
||||
assert %Conn{
|
||||
status: 404,
|
||||
resp_body: "Not Found"
|
||||
} = get(conn, "/proxy/preview/hhgfh/eeeee")
|
||||
|
||||
assert %Conn{
|
||||
status: 404,
|
||||
resp_body: "Not Found"
|
||||
} = get(conn, "/proxy/preview/hhgfh/fff")
|
||||
end
|
||||
|
||||
test "returns 404 when disabled", %{conn: conn} do
|
||||
clear_config([:media_preview_proxy, :enabled], false)
|
||||
|
||||
assert %Conn{
|
||||
status: 404,
|
||||
resp_body: "Not Found"
|
||||
} = get(conn, "/proxy/preview/hhgfh/eeeee")
|
||||
|
||||
assert %Conn{
|
||||
status: 404,
|
||||
resp_body: "Not Found"
|
||||
} = get(conn, "/proxy/preview/hhgfh/fff")
|
||||
end
|
||||
|
||||
test "it returns 403 for invalid signature", %{conn: conn, url: url} do
|
||||
Pleroma.Config.put([Pleroma.Web.Endpoint, :secret_key_base], "000")
|
||||
%{path: path} = URI.parse(url)
|
||||
|
||||
assert %Conn{
|
||||
status: 403,
|
||||
resp_body: "Forbidden"
|
||||
} = get(conn, path)
|
||||
|
||||
assert %Conn{
|
||||
status: 403,
|
||||
resp_body: "Forbidden"
|
||||
} = get(conn, "/proxy/preview/hhgfh/eeee")
|
||||
|
||||
assert %Conn{
|
||||
status: 403,
|
||||
resp_body: "Forbidden"
|
||||
} = get(conn, "/proxy/preview/hhgfh/eeee/fff")
|
||||
end
|
||||
|
||||
test "redirects to valid url when filename is invalidated", %{conn: conn, url: url} do
|
||||
invalid_url = String.replace(url, "test.png", "test-file.png")
|
||||
response = get(conn, invalid_url)
|
||||
assert response.status == 302
|
||||
assert redirected_to(response) == url
|
||||
end
|
||||
|
||||
test "responds with 424 Failed Dependency if HEAD request to media proxy fails", %{
|
||||
conn: conn,
|
||||
url: url,
|
||||
media_proxy_url: media_proxy_url
|
||||
} do
|
||||
Tesla.Mock.mock(fn
|
||||
%{method: "head", url: ^media_proxy_url} ->
|
||||
%Tesla.Env{status: 500, body: ""}
|
||||
end)
|
||||
|
||||
response = get(conn, url)
|
||||
assert response.status == 424
|
||||
assert response.resp_body == "Can't fetch HTTP headers (HTTP 500)."
|
||||
end
|
||||
|
||||
test "redirects to media proxy URI on unsupported content type", %{
|
||||
conn: conn,
|
||||
url: url,
|
||||
media_proxy_url: media_proxy_url
|
||||
} do
|
||||
Tesla.Mock.mock(fn
|
||||
%{method: "head", url: ^media_proxy_url} ->
|
||||
%Tesla.Env{status: 200, body: "", headers: [{"content-type", "application/pdf"}]}
|
||||
end)
|
||||
|
||||
response = get(conn, url)
|
||||
assert response.status == 302
|
||||
assert redirected_to(response) == media_proxy_url
|
||||
end
|
||||
|
||||
test "with `static=true` and GIF image preview requested, responds with JPEG image", %{
|
||||
conn: conn,
|
||||
url: url,
|
||||
media_proxy_url: media_proxy_url
|
||||
} do
|
||||
# Setting a high :min_content_length to ensure this scenario is not affected by its logic
|
||||
clear_config([:media_preview_proxy, :min_content_length], 1_000_000_000)
|
||||
|
||||
Tesla.Mock.mock(fn
|
||||
%{method: "head", url: ^media_proxy_url} ->
|
||||
%Tesla.Env{
|
||||
status: 200,
|
||||
body: "",
|
||||
headers: [{"content-type", "image/gif"}, {"content-length", "1001718"}]
|
||||
}
|
||||
|
||||
%{method: :get, url: ^media_proxy_url} ->
|
||||
%Tesla.Env{status: 200, body: File.read!("test/fixtures/image.gif")}
|
||||
end)
|
||||
|
||||
response = get(conn, url <> "?static=true")
|
||||
|
||||
assert response.status == 200
|
||||
assert Conn.get_resp_header(response, "content-type") == ["image/jpeg"]
|
||||
assert response.resp_body != ""
|
||||
end
|
||||
|
||||
test "with GIF image preview requested and no `static` param, redirects to media proxy URI",
|
||||
%{
|
||||
conn: conn,
|
||||
url: url,
|
||||
media_proxy_url: media_proxy_url
|
||||
} do
|
||||
Tesla.Mock.mock(fn
|
||||
%{method: "head", url: ^media_proxy_url} ->
|
||||
%Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/gif"}]}
|
||||
end)
|
||||
|
||||
response = get(conn, url)
|
||||
|
||||
assert response.status == 302
|
||||
assert redirected_to(response) == media_proxy_url
|
||||
end
|
||||
|
||||
test "with `static` param and non-GIF image preview requested, " <>
|
||||
"redirects to media preview proxy URI without `static` param",
|
||||
%{
|
||||
conn: conn,
|
||||
url: url,
|
||||
media_proxy_url: media_proxy_url
|
||||
} do
|
||||
Tesla.Mock.mock(fn
|
||||
%{method: "head", url: ^media_proxy_url} ->
|
||||
%Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/jpeg"}]}
|
||||
end)
|
||||
|
||||
response = get(conn, url <> "?static=true")
|
||||
|
||||
assert response.status == 302
|
||||
assert redirected_to(response) == url
|
||||
end
|
||||
|
||||
test "with :min_content_length setting not matched by Content-Length header, " <>
|
||||
"redirects to media proxy URI",
|
||||
%{
|
||||
conn: conn,
|
||||
url: url,
|
||||
media_proxy_url: media_proxy_url
|
||||
} do
|
||||
clear_config([:media_preview_proxy, :min_content_length], 100_000)
|
||||
|
||||
Tesla.Mock.mock(fn
|
||||
%{method: "head", url: ^media_proxy_url} ->
|
||||
%Tesla.Env{
|
||||
status: 200,
|
||||
body: "",
|
||||
headers: [{"content-type", "image/gif"}, {"content-length", "5000"}]
|
||||
}
|
||||
end)
|
||||
|
||||
response = get(conn, url)
|
||||
|
||||
assert response.status == 302
|
||||
assert redirected_to(response) == media_proxy_url
|
||||
end
|
||||
|
||||
test "thumbnails PNG images into PNG", %{
|
||||
conn: conn,
|
||||
url: url,
|
||||
media_proxy_url: media_proxy_url
|
||||
} do
|
||||
Tesla.Mock.mock(fn
|
||||
%{method: "head", url: ^media_proxy_url} ->
|
||||
%Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/png"}]}
|
||||
|
||||
%{method: :get, url: ^media_proxy_url} ->
|
||||
%Tesla.Env{status: 200, body: File.read!("test/fixtures/image.png")}
|
||||
end)
|
||||
|
||||
response = get(conn, url)
|
||||
|
||||
assert response.status == 200
|
||||
assert Conn.get_resp_header(response, "content-type") == ["image/png"]
|
||||
assert response.resp_body != ""
|
||||
end
|
||||
|
||||
test "thumbnails JPEG images into JPEG", %{
|
||||
conn: conn,
|
||||
url: url,
|
||||
media_proxy_url: media_proxy_url
|
||||
} do
|
||||
Tesla.Mock.mock(fn
|
||||
%{method: "head", url: ^media_proxy_url} ->
|
||||
%Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/jpeg"}]}
|
||||
|
||||
%{method: :get, url: ^media_proxy_url} ->
|
||||
%Tesla.Env{status: 200, body: File.read!("test/fixtures/image.jpg")}
|
||||
end)
|
||||
|
||||
response = get(conn, url)
|
||||
|
||||
assert response.status == 200
|
||||
assert Conn.get_resp_header(response, "content-type") == ["image/jpeg"]
|
||||
assert response.resp_body != ""
|
||||
end
|
||||
|
||||
test "redirects to media proxy URI in case of thumbnailing error", %{
|
||||
conn: conn,
|
||||
url: url,
|
||||
media_proxy_url: media_proxy_url
|
||||
} do
|
||||
Tesla.Mock.mock(fn
|
||||
%{method: "head", url: ^media_proxy_url} ->
|
||||
%Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/jpeg"}]}
|
||||
|
||||
%{method: :get, url: ^media_proxy_url} ->
|
||||
%Tesla.Env{status: 200, body: "<html><body>error</body></html>"}
|
||||
end)
|
||||
|
||||
response = get(conn, url)
|
||||
|
||||
assert response.status == 302
|
||||
assert redirected_to(response) == media_proxy_url
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -6,9 +6,16 @@ defmodule Pleroma.Web.MediaProxyTest do
|
|||
use ExUnit.Case
|
||||
use Pleroma.Tests.Helpers
|
||||
|
||||
alias Pleroma.Config
|
||||
alias Pleroma.Web.Endpoint
|
||||
alias Pleroma.Web.MediaProxy
|
||||
|
||||
defp decode_result(encoded) do
|
||||
[_, "proxy", sig, base64 | _] = URI.parse(encoded).path |> String.split("/")
|
||||
{:ok, decoded} = MediaProxy.decode_url(sig, base64)
|
||||
decoded
|
||||
end
|
||||
|
||||
describe "when enabled" do
|
||||
setup do: clear_config([:media_proxy, :enabled], true)
|
||||
|
||||
|
|
@ -35,7 +42,7 @@ test "encodes and decodes URL" do
|
|||
|
||||
assert String.starts_with?(
|
||||
encoded,
|
||||
Pleroma.Config.get([:media_proxy, :base_url], Pleroma.Web.base_url())
|
||||
Config.get([:media_proxy, :base_url], Pleroma.Web.base_url())
|
||||
)
|
||||
|
||||
assert String.ends_with?(encoded, "/logo.png")
|
||||
|
|
@ -75,6 +82,64 @@ test "validates signature" do
|
|||
assert MediaProxy.decode_url(sig, base64) == {:error, :invalid_signature}
|
||||
end
|
||||
|
||||
def test_verify_request_path_and_url(request_path, url, expected_result) do
|
||||
assert MediaProxy.verify_request_path_and_url(request_path, url) == expected_result
|
||||
|
||||
assert MediaProxy.verify_request_path_and_url(
|
||||
%Plug.Conn{
|
||||
params: %{"filename" => Path.basename(request_path)},
|
||||
request_path: request_path
|
||||
},
|
||||
url
|
||||
) == expected_result
|
||||
end
|
||||
|
||||
test "if first arg of `verify_request_path_and_url/2` is a Plug.Conn without \"filename\" " <>
|
||||
"parameter, `verify_request_path_and_url/2` returns :ok " do
|
||||
assert MediaProxy.verify_request_path_and_url(
|
||||
%Plug.Conn{params: %{}, request_path: "/some/path"},
|
||||
"https://instance.com/file.jpg"
|
||||
) == :ok
|
||||
|
||||
assert MediaProxy.verify_request_path_and_url(
|
||||
%Plug.Conn{params: %{}, request_path: "/path/to/file.jpg"},
|
||||
"https://instance.com/file.jpg"
|
||||
) == :ok
|
||||
end
|
||||
|
||||
test "`verify_request_path_and_url/2` preserves the encoded or decoded path" do
|
||||
test_verify_request_path_and_url(
|
||||
"/Hello world.jpg",
|
||||
"http://pleroma.social/Hello world.jpg",
|
||||
:ok
|
||||
)
|
||||
|
||||
test_verify_request_path_and_url(
|
||||
"/Hello%20world.jpg",
|
||||
"http://pleroma.social/Hello%20world.jpg",
|
||||
:ok
|
||||
)
|
||||
|
||||
test_verify_request_path_and_url(
|
||||
"/my%2Flong%2Furl%2F2019%2F07%2FS.jpg",
|
||||
"http://pleroma.social/my%2Flong%2Furl%2F2019%2F07%2FS.jpg",
|
||||
:ok
|
||||
)
|
||||
|
||||
test_verify_request_path_and_url(
|
||||
# Note: `conn.request_path` returns encoded url
|
||||
"/ANALYSE-DAI-_-LE-STABLECOIN-100-D%C3%89CENTRALIS%C3%89-BQ.jpg",
|
||||
"https://mydomain.com/uploads/2019/07/ANALYSE-DAI-_-LE-STABLECOIN-100-DÉCENTRALISÉ-BQ.jpg",
|
||||
:ok
|
||||
)
|
||||
|
||||
test_verify_request_path_and_url(
|
||||
"/my%2Flong%2Furl%2F2019%2F07%2FS",
|
||||
"http://pleroma.social/my%2Flong%2Furl%2F2019%2F07%2FS.jpg",
|
||||
{:wrong_filename, "my%2Flong%2Furl%2F2019%2F07%2FS.jpg"}
|
||||
)
|
||||
end
|
||||
|
||||
test "uses the configured base_url" do
|
||||
base_url = "https://cache.pleroma.social"
|
||||
clear_config([:media_proxy, :base_url], base_url)
|
||||
|
|
@ -124,12 +189,6 @@ test "does not encode remote urls" do
|
|||
end
|
||||
end
|
||||
|
||||
defp decode_result(encoded) do
|
||||
[_, "proxy", sig, base64 | _] = URI.parse(encoded).path |> String.split("/")
|
||||
{:ok, decoded} = MediaProxy.decode_url(sig, base64)
|
||||
decoded
|
||||
end
|
||||
|
||||
describe "whitelist" do
|
||||
setup do: clear_config([:media_proxy, :enabled], true)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue