add finch outbound proxy support (#158)

Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/158
This commit is contained in:
floatingghost 2022-08-14 23:13:49 +00:00
parent 5796d81d98
commit 37a1001b97
11 changed files with 98 additions and 262 deletions

View file

@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
## [Unreleased]
### Removed
- Non-finch HTTP adapters. `:tesla, :adapter` is now highly recommended to be set to the default.
## 2022.08
### Added
- extended runtime module support, see config cheatsheet
- quote posting; quotes are limited to public posts

View file

@ -2598,9 +2598,10 @@
%{
key: :proxy_url,
label: "Proxy URL",
type: [:string, :tuple],
description: "Proxy URL",
suggestions: ["localhost:9020", {:socks5, :localhost, 3090}]
type: :string,
description:
"Proxy URL - of the format http://host:port. Advise setting in .exs instead of admin-fe due to this being set at boot-time.",
suggestions: ["http://localhost:3128"]
},
%{
key: :user_agent,

View file

@ -521,7 +521,7 @@ Available caches:
### :http
* `proxy_url`: an upstream proxy to fetch posts and/or media with, (default: `nil`)
* `proxy_url`: an upstream proxy to fetch posts and/or media with, (default: `nil`); for example `http://127.0.0.1:3192`. Does not support SOCKS5 proxy, only http(s).
* `send_user_agent`: should we include a user agent with HTTP requests? (default: `true`)
* `user_agent`: what user agent should we use? (default: `:default`), must be string or `:default`
* `adapter`: array of adapter options

View file

@ -248,9 +248,13 @@ def limiters_setup do
end
defp http_children do
proxy_url = Config.get([:http, :proxy_url])
proxy = Pleroma.HTTP.AdapterHelper.format_proxy(proxy_url)
config =
[:http, :adapter]
|> Config.get([])
|> Pleroma.HTTP.AdapterHelper.maybe_add_proxy_pool(proxy)
|> Keyword.put(:name, MyFinch)
[{Finch, config}]

View file

@ -6,7 +6,7 @@ defmodule Pleroma.HTTP.AdapterHelper do
@moduledoc """
Configure Tesla.Client with default and customized adapter options.
"""
@defaults [name: MyFinch, connect_timeout: 5_000, recv_timeout: 5_000]
@defaults [name: MyFinch, pool_timeout: 5_000, receive_timeout: 5_000]
@type proxy_type() :: :socks4 | :socks5
@type host() :: charlist() | :inet.ip_address()
@ -25,15 +25,58 @@ def format_proxy(nil), do: nil
def format_proxy(proxy_url) do
case parse_proxy(proxy_url) do
{:ok, host, port} -> {host, port}
{:ok, type, host, port} -> {type, host, port}
{:ok, host, port} -> {:http, host, port, []}
{:ok, type, host, port} -> {type, host, port, []}
_ -> nil
end
end
@spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword()
def maybe_add_proxy(opts, nil), do: opts
def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy)
def maybe_add_proxy(opts, proxy) do
Keyword.put(opts, :proxy, proxy)
end
def maybe_add_proxy_pool(opts, nil), do: opts
def maybe_add_proxy_pool(opts, proxy) do
Logger.info("Using HTTP Proxy: #{inspect(proxy)}")
opts
|> maybe_add_pools()
|> maybe_add_default_pool()
|> maybe_add_conn_opts()
|> put_in([:pools, :default, :conn_opts, :proxy], proxy)
end
defp maybe_add_pools(opts) do
if Keyword.has_key?(opts, :pools) do
opts
else
Keyword.put(opts, :pools, %{})
end
end
defp maybe_add_default_pool(opts) do
pools = Keyword.get(opts, :pools)
if Map.has_key?(pools, :default) do
opts
else
put_in(opts, [:pools, :default], [])
end
end
defp maybe_add_conn_opts(opts) do
conn_opts = get_in(opts, [:pools, :default, :conn_opts])
unless is_nil(conn_opts) do
opts
else
put_in(opts, [:pools, :default, :conn_opts], [])
end
end
@doc """
Merge default connection & adapter options with received ones.
@ -46,36 +89,31 @@ def options(%URI{} = uri, opts \\ []) do
|> AdapterHelper.Default.options(uri)
end
defp proxy_type("http"), do: {:ok, :http}
defp proxy_type("https"), do: {:ok, :https}
defp proxy_type(_), do: {:error, :unknown}
@spec parse_proxy(String.t() | tuple() | nil) ::
{:ok, host(), pos_integer()}
| {:ok, proxy_type(), host(), pos_integer()}
| {:error, atom()}
| nil
def parse_proxy(nil), do: nil
def parse_proxy(proxy) when is_binary(proxy) do
with [host, port] <- String.split(proxy, ":"),
{port, ""} <- Integer.parse(port) do
{:ok, parse_host(host), port}
with %URI{} = uri <- URI.parse(proxy),
{:ok, type} <- proxy_type(uri.scheme) do
{:ok, type, uri.host, uri.port}
else
{_, _} ->
Logger.warn("Parsing port failed #{inspect(proxy)}")
{:error, :invalid_proxy_port}
:error ->
Logger.warn("Parsing port failed #{inspect(proxy)}")
{:error, :invalid_proxy_port}
_ ->
Logger.warn("Parsing proxy failed #{inspect(proxy)}")
e ->
Logger.warn("Parsing proxy failed #{inspect(proxy)}, #{inspect(e)}")
{:error, :invalid_proxy}
end
end
def parse_proxy(proxy) when is_tuple(proxy) do
with {type, host, port} <- proxy do
{:ok, type, parse_host(host), port}
{:ok, type, host, port}
else
_ ->
Logger.warn("Parsing proxy failed #{inspect(proxy)}")

View file

@ -9,7 +9,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Default do
@spec options(keyword(), URI.t()) :: keyword()
def options(opts, _uri) do
proxy = Pleroma.Config.get([:http, :proxy_url], nil)
proxy = Pleroma.Config.get([:http, :proxy_url])
AdapterHelper.maybe_add_proxy(opts, AdapterHelper.format_proxy(proxy))
end

View file

@ -1,82 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.AdapterHelper.Gun do
@behaviour Pleroma.HTTP.AdapterHelper
alias Pleroma.Config
alias Pleroma.HTTP.AdapterHelper
require Logger
@defaults [
retry: 1,
retry_timeout: 1_000
]
@type pool() :: :federation | :upload | :media | :default
@spec options(keyword(), URI.t()) :: keyword()
def options(incoming_opts \\ [], %URI{} = uri) do
proxy =
[:http, :proxy_url]
|> Config.get()
|> AdapterHelper.format_proxy()
config_opts = Config.get([:http, :adapter], [])
@defaults
|> Keyword.merge(config_opts)
|> add_scheme_opts(uri)
|> AdapterHelper.maybe_add_proxy(proxy)
|> Keyword.merge(incoming_opts)
|> put_timeout()
end
defp add_scheme_opts(opts, %{scheme: "http"}), do: opts
defp add_scheme_opts(opts, %{scheme: "https"}) do
Keyword.put(opts, :certificates_verification, true)
end
defp put_timeout(opts) do
{recv_timeout, opts} = Keyword.pop(opts, :recv_timeout, pool_timeout(opts[:pool]))
# this is the timeout to receive a message from Gun
# `:timeout` key is used in Tesla
Keyword.put(opts, :timeout, recv_timeout)
end
@spec pool_timeout(pool()) :: non_neg_integer()
def pool_timeout(pool) do
default = Config.get([:pools, :default, :recv_timeout], 5_000)
Config.get([:pools, pool, :recv_timeout], default)
end
def limiter_setup do
prefix = Pleroma.Gun.ConnectionPool
wait = Config.get([:connections_pool, :connection_acquisition_wait])
retries = Config.get([:connections_pool, :connection_acquisition_retries])
:pools
|> Config.get([])
|> Enum.each(fn {name, opts} ->
max_running = Keyword.get(opts, :size, 50)
max_waiting = Keyword.get(opts, :max_waiting, 10)
result =
ConcurrentLimiter.new(:"#{prefix}.#{name}", max_running, max_waiting,
wait: wait,
max_retries: retries
)
case result do
:ok -> :ok
{:error, :existing} -> :ok
end
end)
:ok
end
end

View file

@ -1,40 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.AdapterHelper.Hackney do
@behaviour Pleroma.HTTP.AdapterHelper
@defaults [
follow_redirect: true,
force_redirect: true
]
@spec options(keyword(), URI.t()) :: keyword()
def options(connection_opts \\ [], %URI{} = uri) do
proxy = Pleroma.Config.get([:http, :proxy_url])
config_opts = Pleroma.Config.get([:http, :adapter], [])
@defaults
|> Keyword.merge(config_opts)
|> Keyword.merge(connection_opts)
|> add_scheme_opts(uri)
|> maybe_add_with_body()
|> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy)
end
defp add_scheme_opts(opts, %URI{scheme: "https"}) do
Keyword.put(opts, :ssl_options, versions: [:"tlsv1.3", :"tlsv1.2", :"tlsv1.1", :tlsv1])
end
defp add_scheme_opts(opts, _), do: opts
defp maybe_add_with_body(opts) do
if opts[:max_body] do
Keyword.put(opts, :with_body, true)
else
opts
end
end
end

View file

@ -1,77 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.AdapterHelper.GunTest do
use ExUnit.Case
use Pleroma.Tests.Helpers
import Mox
alias Pleroma.HTTP.AdapterHelper.Gun
setup :verify_on_exit!
describe "options/1" do
setup do: clear_config([:http, :adapter], a: 1, b: 2)
test "https url with default port" do
uri = URI.parse("https://example.com")
opts = Gun.options([receive_conn: false], uri)
assert opts[:certificates_verification]
end
test "https ipv4 with default port" do
uri = URI.parse("https://127.0.0.1")
opts = Gun.options([receive_conn: false], uri)
assert opts[:certificates_verification]
end
test "https ipv6 with default port" do
uri = URI.parse("https://[2a03:2880:f10c:83:face:b00c:0:25de]")
opts = Gun.options([receive_conn: false], uri)
assert opts[:certificates_verification]
end
test "https url with non standart port" do
uri = URI.parse("https://example.com:115")
opts = Gun.options([receive_conn: false], uri)
assert opts[:certificates_verification]
end
test "merges with defaul http adapter config" do
defaults = Gun.options([receive_conn: false], URI.parse("https://example.com"))
assert Keyword.has_key?(defaults, :a)
assert Keyword.has_key?(defaults, :b)
end
test "parses string proxy host & port" do
clear_config([:http, :proxy_url], "localhost:8123")
uri = URI.parse("https://some-domain.com")
opts = Gun.options([receive_conn: false], uri)
assert opts[:proxy] == {'localhost', 8123}
end
test "parses tuple proxy scheme host and port" do
clear_config([:http, :proxy_url], {:socks, 'localhost', 1234})
uri = URI.parse("https://some-domain.com")
opts = Gun.options([receive_conn: false], uri)
assert opts[:proxy] == {:socks, 'localhost', 1234}
end
test "passed opts have more weight than defaults" do
clear_config([:http, :proxy_url], {:socks5, 'localhost', 1234})
uri = URI.parse("https://some-domain.com")
opts = Gun.options([receive_conn: false, proxy: {'example.com', 4321}], uri)
assert opts[:proxy] == {'example.com', 4321}
end
end
end

View file

@ -1,35 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.AdapterHelper.HackneyTest do
use ExUnit.Case, async: true
use Pleroma.Tests.Helpers
alias Pleroma.HTTP.AdapterHelper.Hackney
setup_all do
uri = URI.parse("http://domain.com")
{:ok, uri: uri}
end
describe "options/2" do
setup do: clear_config([:http, :adapter], a: 1, b: 2)
test "add proxy and opts from config", %{uri: uri} do
opts = Hackney.options([proxy: "localhost:8123"], uri)
assert opts[:a] == 1
assert opts[:b] == 2
assert opts[:proxy] == "localhost:8123"
end
test "respect connection opts and no proxy", %{uri: uri} do
opts = Hackney.options([a: 2, b: 1], uri)
assert opts[:a] == 2
assert opts[:b] == 1
refute Keyword.has_key?(opts, :proxy)
end
end
end

View file

@ -13,16 +13,38 @@ test "with nil" do
end
test "with string" do
assert AdapterHelper.format_proxy("127.0.0.1:8123") == {{127, 0, 0, 1}, 8123}
assert AdapterHelper.format_proxy("http://127.0.0.1:8123") == {:http, "127.0.0.1", 8123, []}
end
test "localhost with port" do
assert AdapterHelper.format_proxy("localhost:8123") == {'localhost', 8123}
assert AdapterHelper.format_proxy("https://localhost:8123") ==
{:https, "localhost", 8123, []}
end
test "tuple" do
assert AdapterHelper.format_proxy({:socks4, :localhost, 9050}) ==
{:socks4, 'localhost', 9050}
assert AdapterHelper.format_proxy({:http, "localhost", 9050}) ==
{:http, "localhost", 9050, []}
end
end
describe "maybe_add_proxy_pool/1" do
test "should do nothing with nil" do
assert AdapterHelper.maybe_add_proxy_pool([], nil) == []
end
test "should create pools" do
assert AdapterHelper.maybe_add_proxy_pool([], "proxy") == [
pools: %{default: [conn_opts: [proxy: "proxy"]]}
]
end
test "should not override conn_opts if set" do
assert AdapterHelper.maybe_add_proxy_pool(
[pools: %{default: [conn_opts: [already: "set"]]}],
"proxy"
) == [
pools: %{default: [conn_opts: [proxy: "proxy", already: "set"]]}
]
end
end
end