From 4aff4efa8d53988d00381b1346241359cf787e87 Mon Sep 17 00:00:00 2001 From: href Date: Wed, 30 Jan 2019 12:38:38 +0100 Subject: [PATCH] Use multiple hackney pools * federation (ap, salmon) * media (rich media, media proxy) * upload (uploader proxy) Each "part" will stop fighting others ones -- a huge federation outbound could before make the media proxy fail to checkout a connection in time. splitted media and uploaded media for the good reason than an upload pool will have all connections to the same host (the uploader upstream). it also has a longer default retention period for connections. --- config/config.exs | 34 +++++++++++++++++-- docs/config.md | 17 ++++++++++ lib/pleroma/application.ex | 34 ++++++++++++++++--- lib/pleroma/http/connection.ex | 3 +- lib/pleroma/uploaders/mdii.ex | 3 +- .../mastodon_api/mastodon_api_controller.ex | 3 +- lib/pleroma/web/rich_media/parser.ex | 2 +- .../web/rich_media/parsers/oembed_parser.ex | 2 +- 8 files changed, 86 insertions(+), 12 deletions(-) diff --git a/config/config.exs b/config/config.exs index 1180a50bc..4dc7a62d7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -15,6 +15,20 @@ seconds_valid: 60, method: Pleroma.Captcha.Kocaptcha +config :pleroma, :hackney_pools, + federation: [ + max_connections: 50, + timeout: 150_000 + ], + media: [ + max_connections: 50, + timeout: 150_000 + ], + upload: [ + max_connections: 25, + timeout: 300_000 + ] + config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch" # Upload configuration @@ -22,7 +36,14 @@ uploader: Pleroma.Uploaders.Local, filters: [], proxy_remote: false, - proxy_opts: [] + proxy_opts: [ + redirect_on_failure: false, + max_body_length: 25 * 1_048_576, + http: [ + follow_redirect: true, + pool: :upload + ] + ] config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads" @@ -214,7 +235,16 @@ reject: [], accept: [] -config :pleroma, :media_proxy, enabled: false +config :pleroma, :media_proxy, + enabled: false, + proxy_opts: [ + redirect_on_failure: false, + max_body_length: 25 * 1_048_576, + http: [ + follow_redirect: true, + pool: :media + ] + ] config :pleroma, :chat, enabled: true diff --git a/docs/config.md b/docs/config.md index 8740c3fae..a00532d16 100644 --- a/docs/config.md +++ b/docs/config.md @@ -234,3 +234,20 @@ curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerando * Pleroma.Web.Metadata.Providers.OpenGraph * Pleroma.Web.Metadata.Providers.TwitterCard * `unfurl_nsfw`: If set to `true` nsfw attachments will be shown in previews + +## :hackney_pools + +Advanced. Tweaks Hackney (http client) connections pools. + +There's three pools used: + +* `:federation` for the federation jobs. + You may want this pool max_connections to be at least equal to the number of federator jobs + retry queue jobs. +* `:media` for rich media, media proxy +* `:upload` for uploaded media (if using a remote uploader and `proxy_remote: true`) + +For each pool, the options are: + +* `max_connections` - how much connections a pool can hold +* `timeout` - retention duration for connections + diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index c65bebb3b..f775cd53d 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -101,12 +101,15 @@ def start(_type, _args) do ], id: :cachex_idem ), - worker(Pleroma.FlakeId, []), - worker(Pleroma.Web.Federator.RetryQueue, []), - worker(Pleroma.Web.Federator, []), - worker(Pleroma.Stats, []), - worker(Pleroma.Web.Push, []) + worker(Pleroma.FlakeId, []) ] ++ + hackney_pool_children() ++ + [ + worker(Pleroma.Web.Federator.RetryQueue, []), + worker(Pleroma.Web.Federator, []), + worker(Pleroma.Stats, []), + worker(Pleroma.Web.Push, []) + ] ++ streamer_child() ++ chat_child() ++ [ @@ -121,6 +124,20 @@ def start(_type, _args) do Supervisor.start_link(children, opts) end + def enabled_hackney_pools() do + [:media] ++ + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do + [:federation] + else + [] + end ++ + if Pleroma.Config.get([Pleroma.Uploader, :proxy_remote]) do + [:uploadproxy] + else + [] + end + end + if Mix.env() == :test do defp streamer_child(), do: [] defp chat_child(), do: [] @@ -137,4 +154,11 @@ defp chat_child() do end end end + + defp hackney_pool_children() do + for pool <- enabled_hackney_pools() do + options = Pleroma.Config.get([:hackney_pools, pool]) + :hackney_pool.child_spec(pool, options) + end + end end diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 699d80cd7..b798eaa5a 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -10,7 +10,8 @@ defmodule Pleroma.HTTP.Connection do @hackney_options [ timeout: 10000, recv_timeout: 20000, - follow_redirect: true + follow_redirect: true, + pool: :federation ] @adapter Application.get_env(:tesla, :adapter) diff --git a/lib/pleroma/uploaders/mdii.ex b/lib/pleroma/uploaders/mdii.ex index 530b34362..320b07abd 100644 --- a/lib/pleroma/uploaders/mdii.ex +++ b/lib/pleroma/uploaders/mdii.ex @@ -24,7 +24,8 @@ def put_file(upload) do extension = String.split(upload.name, ".") |> List.last() query = "#{cgi}?#{extension}" - with {:ok, %{status: 200, body: body}} <- @httpoison.post(query, file_data) do + with {:ok, %{status: 200, body: body}} <- + @httpoison.post(query, file_data, adapter: [pool: :default]) do remote_file_name = String.split(body) |> List.first() public_url = "#{files}/#{remote_file_name}.#{extension}" {:ok, {:url, public_url}} diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 65b612026..a92645ca3 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1311,7 +1311,8 @@ def suggestions(%{assigns: %{user: user}} = conn, _) do [], adapter: [ timeout: timeout, - recv_timeout: timeout + recv_timeout: timeout, + pool: :default ] ), {:ok, data} <- Jason.decode(body) do diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index 76d977ac2..874e8c5e6 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -28,7 +28,7 @@ def parse(url) do defp parse_url(url) do try do - {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url) + {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], pool: :media) html |> maybe_parse() |> get_parsed_data() rescue diff --git a/lib/pleroma/web/rich_media/parsers/oembed_parser.ex b/lib/pleroma/web/rich_media/parsers/oembed_parser.ex index efa98bc2c..c3adc4962 100644 --- a/lib/pleroma/web/rich_media/parsers/oembed_parser.ex +++ b/lib/pleroma/web/rich_media/parsers/oembed_parser.ex @@ -20,7 +20,7 @@ defp get_oembed_url(nodes) do end defp get_oembed_data(url) do - {:ok, %Tesla.Env{body: json}} = Pleroma.HTTP.get(url) + {:ok, %Tesla.Env{body: json}} = Pleroma.HTTP.get(url, [], pool: :media) {:ok, data} = Jason.decode(json)