From 92213fb87c7996caf9d1188a94907d2231ba25c8 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 6 Jun 2019 23:59:51 +0300 Subject: [PATCH 01/33] Replace Mix.env with Pleroma.Config.get(:env) Mix.env/0 is not availible in release environments such as distillery or elixir's built-in releases. --- config/config.exs | 2 ++ lib/pleroma/application.ex | 2 +- lib/pleroma/plugs/http_security_plug.ex | 4 ++-- lib/pleroma/web/federator/retry_queue.ex | 6 ++++-- lib/pleroma/web/rel_me.ex | 2 +- lib/pleroma/web/rich_media/parser.ex | 2 +- lib/pleroma/web/router.ex | 2 +- lib/pleroma/web/views/error_view.ex | 2 +- 8 files changed, 13 insertions(+), 9 deletions(-) diff --git a/config/config.exs b/config/config.exs index 4e2b1703b..c6bf71fc8 100644 --- a/config/config.exs +++ b/config/config.exs @@ -500,6 +500,8 @@ config :pleroma, :database, rum_enabled: false +config :pleroma, :env, Mix.env() + config :http_signatures, adapter: Pleroma.Signature diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 69a8a5761..5627d20af 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -194,7 +194,7 @@ def enabled_hackney_pools do end end - if Mix.env() == :test do + if Pleroma.Config.get(:env) == :test do defp streamer_child, do: [] defp chat_child, do: [] else diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex index 485ddfbc7..a7cc22831 100644 --- a/lib/pleroma/plugs/http_security_plug.ex +++ b/lib/pleroma/plugs/http_security_plug.ex @@ -56,14 +56,14 @@ defp csp_string do connect_src = "connect-src 'self' #{static_url} #{websocket_url}" connect_src = - if Mix.env() == :dev do + if Pleroma.Config.get(:env) == :dev do connect_src <> " http://localhost:3035/" else connect_src end script_src = - if Mix.env() == :dev do + if Pleroma.Config.get(:env) == :dev do "script-src 'self' 'unsafe-eval'" else "script-src 'self'" diff --git a/lib/pleroma/web/federator/retry_queue.ex b/lib/pleroma/web/federator/retry_queue.ex index 71e49494f..3db948c2e 100644 --- a/lib/pleroma/web/federator/retry_queue.ex +++ b/lib/pleroma/web/federator/retry_queue.ex @@ -15,7 +15,9 @@ def init(args) do def start_link do enabled = - if Mix.env() == :test, do: true, else: Pleroma.Config.get([__MODULE__, :enabled], false) + if Pleroma.Config.get(:env) == :test, + do: true, + else: Pleroma.Config.get([__MODULE__, :enabled], false) if enabled do Logger.info("Starting retry queue") @@ -219,7 +221,7 @@ def handle_info(unknown, state) do {:noreply, state} end - if Mix.env() == :test do + if Pleroma.Config.get(:env) == :test do defp growth_function(_retries) do _shutit = Pleroma.Config.get([__MODULE__, :initial_timeout]) DateTime.to_unix(DateTime.utc_now()) - 1 diff --git a/lib/pleroma/web/rel_me.ex b/lib/pleroma/web/rel_me.ex index 26eb614a6..d376e2069 100644 --- a/lib/pleroma/web/rel_me.ex +++ b/lib/pleroma/web/rel_me.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.RelMe do with_body: true ] - if Mix.env() == :test do + if Pleroma.Config.get(:env) == :test do def parse(url) when is_binary(url), do: parse_url(url) else def parse(url) when is_binary(url) do diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index e4595800c..21cd47890 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -18,7 +18,7 @@ defmodule Pleroma.Web.RichMedia.Parser do def parse(nil), do: {:error, "No URL provided"} - if Mix.env() == :test do + if Pleroma.Config.get(:env) == :test do def parse(url), do: parse_url(url) else def parse(url) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e699f6ae2..1b37d6a93 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -699,7 +699,7 @@ defmodule Pleroma.Web.Router do get("/:sig/:url/:filename", MediaProxyController, :remote) end - if Mix.env() == :dev do + if Pleroma.Config.get(:env) == :dev do scope "/dev" do pipe_through([:mailbox_preview]) diff --git a/lib/pleroma/web/views/error_view.ex b/lib/pleroma/web/views/error_view.ex index f4c04131c..5cb8669fe 100644 --- a/lib/pleroma/web/views/error_view.ex +++ b/lib/pleroma/web/views/error_view.ex @@ -13,7 +13,7 @@ def render("404.json", _assigns) do def render("500.json", assigns) do Logger.error("Internal server error: #{inspect(assigns[:reason])}") - if Mix.env() != :prod do + if Pleroma.Config.get(:env) != :prod do %{errors: %{detail: "Internal server error", reason: inspect(assigns[:reason])}} else %{errors: %{detail: "Internal server error"}} From bc597d888c95ed7d91534cc8083152f3282393e3 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 7 Jun 2019 12:37:20 +0300 Subject: [PATCH 02/33] Mix Tasks: Switch to Application.ensure_all_started instead of Mix.Task.run and ensure serve_endpoints is set to false In release environments there is no Mix.Task.run and serve_endpoints must be set to true for the endpoints to start, so we need to ensure it is set to false before starting Pleroma for executing a mix task. --- lib/mix/tasks/pleroma/common.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/common.ex b/lib/mix/tasks/pleroma/common.ex index 48c0c1346..25977f656 100644 --- a/lib/mix/tasks/pleroma/common.ex +++ b/lib/mix/tasks/pleroma/common.ex @@ -5,7 +5,8 @@ defmodule Mix.Tasks.Pleroma.Common do @doc "Common functions to be reused in mix tasks" def start_pleroma do - Mix.Task.run("app.start") + Application.put_env(:phoenix, :serve_endpoints, false, persistent: true) + {:ok, _} = Application.ensure_all_started(:pleroma) end def get_option(options, opt, prompt, defval \\ nil, defname \\ nil) do From 593b8b1e6a8502cca9bf5559b8bec86f172bbecb Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 7 Jun 2019 14:28:14 +0200 Subject: [PATCH 03/33] Configuration: Skip thread containment by default In my tests the interaction between thread containment and other restrictions makes postgresql create some very bad query plans. This caused direct messages to time out on soykaf, for example. --- CHANGELOG.md | 1 + config/config.exs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf2232b09..da5a9aa99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer +- Thread containment / test for complete visibility will be skipped by default. - Enforcement of OAuth scopes - Add multiple use/time expiring invite token - Restyled OAuth pages to fit with Pleroma's default theme diff --git a/config/config.exs b/config/config.exs index c6bf71fc8..95a0e9972 100644 --- a/config/config.exs +++ b/config/config.exs @@ -244,7 +244,7 @@ safe_dm_mentions: false, healthcheck: false, remote_post_retention_days: 90, - skip_thread_containment: false, + skip_thread_containment: true, limit_unauthenticated_to_local_content: true config :pleroma, :app_account_creation, enabled: true, max_requests: 25, interval: 1800 From 76fc4c92bb39dd5d6f4349482cb57419c0f0f93e Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 7 Jun 2019 17:16:56 +0200 Subject: [PATCH 04/33] Fix tests. --- config/test.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/test.exs b/config/test.exs index 7861b9598..6d4fcf7d0 100644 --- a/config/test.exs +++ b/config/test.exs @@ -27,7 +27,8 @@ config :pleroma, :instance, email: "admin@example.com", - notify_email: "noreply@example.com" + notify_email: "noreply@example.com", + skip_thread_containment: false # Configure your database config :pleroma, Pleroma.Repo, From cb3258c863f7485551eb28474d60f12547019d34 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 7 Jun 2019 17:31:21 +0200 Subject: [PATCH 05/33] Emoji: Use full path to check if a file is a directory. --- lib/pleroma/emoji.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index de7fcc1ce..b77b26f7f 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -98,7 +98,9 @@ defp load do Logger.error("Could not access the custom emoji directory #{emoji_dir_path}: #{e}") {:ok, results} -> - grouped = Enum.group_by(results, &File.dir?/1) + grouped = + Enum.group_by(results, fn file -> File.dir?(Path.join(emoji_dir_path, file)) end) + packs = grouped[true] || [] files = grouped[false] || [] From 970f71e222136a3c01a38ffe6c1c44704828434b Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 7 Jun 2019 17:51:47 +0200 Subject: [PATCH 06/33] Conversations: Fetch users in one query. --- lib/pleroma/conversation/participation.ex | 4 ++-- test/conversation/participation_test.exs | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index 2c13c4b40..5883e4183 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -59,10 +59,10 @@ def mark_as_unread(participation) do def for_user(user, params \\ %{}) do from(p in __MODULE__, where: p.user_id == ^user.id, - order_by: [desc: p.updated_at] + order_by: [desc: p.updated_at], + preload: [conversation: [:users]] ) |> Pleroma.Pagination.fetch_paginated(params) - |> Repo.preload(conversation: [:users]) end def for_user_with_last_activity_id(user, params \\ %{}) do diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs index 0e60bfca5..2a03e5d67 100644 --- a/test/conversation/participation_test.exs +++ b/test/conversation/participation_test.exs @@ -72,8 +72,11 @@ test "gets all the participations for a user, ordered by updated at descending" object2 = Pleroma.Object.normalize(activity_two) object3 = Pleroma.Object.normalize(activity_three) + user = Repo.get(Pleroma.User, user.id) + assert participation_one.conversation.ap_id == object3.data["context"] assert participation_two.conversation.ap_id == object2.data["context"] + assert participation_one.conversation.users == [user] # Pagination assert [participation_one] = Participation.for_user(user, %{"limit" => 1}) From d020f68e87decca850904b76c9053a4de024be8d Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 7 Jun 2019 20:40:38 +0300 Subject: [PATCH 07/33] Transmogrifier: Do not crash if inReplyTo does not exist and can't be fetched --- lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +- test/web/activity_pub/transmogrifier_test.exs | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index ff031a16e..3bb8b40b5 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -339,7 +339,7 @@ def fix_content_map(object), do: object def fix_type(%{"inReplyTo" => reply_id} = object) when is_binary(reply_id) do reply = Object.normalize(reply_id) - if reply.data["type"] == "Question" and object["name"] do + if reply && (reply.data["type"] == "Question" and object["name"]) do Map.put(object, "type", "Answer") else object diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 28971ae45..26e8d60fe 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -60,6 +60,22 @@ test "it fetches replied-to activities if we don't have them" do assert returned_object.data["inReplyToAtomUri"] == "https://shitposter.club/notice/2827873" end + test "it does not crash if the object in inReplyTo can't be fetched" do + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + + object = + data["object"] + |> Map.put("inReplyTo", "https://nonexistent.space/whatever") + + data = + data + |> Map.put("object", object) + + {:ok, _returned_activity} = Transmogrifier.handle_incoming(data) + end + test "it works for incoming notices" do data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() From dffc9f060adf43a4faaa5790dc8a01b3d7cb5e34 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 7 Jun 2019 20:48:25 +0300 Subject: [PATCH 08/33] replace missing mock with a 404 --- test/web/activity_pub/transmogrifier_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 26e8d60fe..cc1781403 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -67,7 +67,7 @@ test "it does not crash if the object in inReplyTo can't be fetched" do object = data["object"] - |> Map.put("inReplyTo", "https://nonexistent.space/whatever") + |> Map.put("inReplyTo", "https://404.site/whatever") data = data From 3d374bf7df6a5cc4bacefefcb133387fe4c5265f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 8 Jun 2019 17:15:49 +0300 Subject: [PATCH 09/33] Basic release skeleton --- config/releases.exs | 1 + rel/env.sh.eex | 12 ++++++++++++ rel/pleroma_ctl | 5 +++++ rel/vm.args.eex | 11 +++++++++++ 4 files changed, 29 insertions(+) create mode 100644 config/releases.exs create mode 100644 rel/env.sh.eex create mode 100755 rel/pleroma_ctl create mode 100644 rel/vm.args.eex diff --git a/config/releases.exs b/config/releases.exs new file mode 100644 index 000000000..becde7693 --- /dev/null +++ b/config/releases.exs @@ -0,0 +1 @@ +import Config diff --git a/rel/env.sh.eex b/rel/env.sh.eex new file mode 100644 index 000000000..a4ce25295 --- /dev/null +++ b/rel/env.sh.eex @@ -0,0 +1,12 @@ +#!/bin/sh + +# Sets and enables heart (recommended only in daemon mode) +# if [ "$RELEASE_COMMAND" = "daemon" ] || [ "$RELEASE_COMMAND" = "daemon_iex" ]; then +# HEART_COMMAND="$RELEASE_ROOT/bin/$RELEASE_NAME $RELEASE_COMMAND" +# export HEART_COMMAND +# export ELIXIR_ERL_OPTIONS="-heart" +# fi + +# Set the release to work across nodes +export RELEASE_DISTRIBUTION=name +export RELEASE_NODE=<%= @release.name %>@127.0.0.1 diff --git a/rel/pleroma_ctl b/rel/pleroma_ctl new file mode 100755 index 000000000..543b742b9 --- /dev/null +++ b/rel/pleroma_ctl @@ -0,0 +1,5 @@ +#!/bin/sh +# XXX: This should be removed when elixir's releases get custom command support +SCRIPT=$(readlink -f "$0") +SCRIPTPATH=$(dirname "$SCRIPT") +$SCRIPTPATH/pleroma eval 'Pleroma.ReleaseTasks.mix_task("'"$*"'")' diff --git a/rel/vm.args.eex b/rel/vm.args.eex new file mode 100644 index 000000000..71e803264 --- /dev/null +++ b/rel/vm.args.eex @@ -0,0 +1,11 @@ +## Customize flags given to the VM: http://erlang.org/doc/man/erl.html +## -mode/-name/-sname/-setcookie are configured via env vars, do not set them here + +## Number of dirty schedulers doing IO work (file, sockets, etc) +##+SDio 5 + +## Increase number of concurrent ports/sockets +##+Q 65536 + +## Tweak GC to run more often +##-env ERL_FULLSWEEP_AFTER 10 From c47dc0de2c567195206523a057c7df067d6fb076 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 8 Jun 2019 17:17:10 +0300 Subject: [PATCH 10/33] Load ex_syslog and copy pleroma_ctl --- mix.exs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mix.exs b/mix.exs index 9447a2e4f..a6481bab6 100644 --- a/mix.exs +++ b/mix.exs @@ -32,10 +32,22 @@ def project do ], main: "readme", output: "priv/static/doc" + ], + releases: [ + pleroma: [ + include_executables_for: [:unix], + applications: [ex_syslogger: :load, syslog: :load], + steps: [:assemble, ©_pleroma_ctl/1] + ] ] ] end + def copy_pleroma_ctl(%{path: target_path} = release) do + File.cp!("./rel/pleroma_ctl", Path.join([target_path, "bin", "pleroma_ctl"])) + release + end + # Configuration for the OTP application. # # Type `mix help compile.app` for more information. From 4b98a7ce4ef4b4e7b74f533e6d6ed343e1b34c48 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 8 Jun 2019 17:17:28 +0300 Subject: [PATCH 11/33] Set serve_endpoints to true in prod config as setting it in runtime config would cause issues with mix tasks --- config/prod.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/prod.exs b/config/prod.exs index d0cfd1ac2..cd5cdb087 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -15,7 +15,8 @@ # which you typically run after static files are built. config :pleroma, Pleroma.Web.Endpoint, http: [port: 4000], - protocol: "http" + protocol: "http", + serve_endpoints: true # Do not print debug messages in production config :logger, level: :info From d7ec0898e5aa7acae463760fd85d1ebf8307b4f9 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 8 Jun 2019 17:40:40 +0300 Subject: [PATCH 12/33] Make mix tasks work in a release --- lib/mix/tasks/pleroma/common.ex | 54 ++++++++++++++++++++------ lib/mix/tasks/pleroma/instance.ex | 12 +++--- lib/mix/tasks/pleroma/relay.ex | 4 +- lib/mix/tasks/pleroma/uploads.ex | 12 +++--- lib/mix/tasks/pleroma/user.ex | 64 +++++++++++++++---------------- lib/pleroma/release_tasks.ex | 38 ++++++++++++++++++ rel/pleroma_ctl | 2 +- 7 files changed, 128 insertions(+), 58 deletions(-) create mode 100644 lib/pleroma/release_tasks.ex diff --git a/lib/mix/tasks/pleroma/common.ex b/lib/mix/tasks/pleroma/common.ex index 25977f656..0e03a7872 100644 --- a/lib/mix/tasks/pleroma/common.ex +++ b/lib/mix/tasks/pleroma/common.ex @@ -10,19 +10,51 @@ def start_pleroma do end def get_option(options, opt, prompt, defval \\ nil, defname \\ nil) do - Keyword.get(options, opt) || - case Mix.shell().prompt("#{prompt} [#{defname || defval}]") do - "\n" -> - case defval do - nil -> get_option(options, opt, prompt, defval) - defval -> defval - end - - opt -> - opt |> String.trim() - end + Keyword.get(options, opt) || shell_prompt(prompt, defval, defname) end + def shell_prompt(prompt, defval \\ nil, defname \\ nil) do + prompt_message = "#{prompt} [#{defname || defval}]" + + input = + if mix_shell?(), + do: Mix.shell().prompt(prompt_message), + else: :io.get_line(prompt_message) + + case input do + "\n" -> + case defval do + nil -> + shell_prompt(prompt, defval, defname) + + defval -> + defval + end + + input -> + String.trim(input) + end + end + + def shell_yes?(message) do + shell_prompt(message, "Yn") in ~w(Yn Y y) + end + + def shell_info(message) do + if mix_shell?(), + do: Mix.shell().info(message), + else: IO.puts(message) + end + + def shell_error(message) do + if mix_shell?(), + do: Mix.shell().error(message), + else: IO.puts(:stderr, message) + end + + @doc "Performs a safe check whether `Mix.shell/0` is available (does not raise if Mix is not loaded)" + def mix_shell?, do: :erlang.function_exported(Mix, :shell, 0) + def escape_sh_path(path) do ~S(') <> String.replace(path, ~S('), ~S(\')) <> ~S(') end diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index 6cee8d630..88925dbaf 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -155,17 +155,17 @@ def run(["gen" | rest]) do dbpass: dbpass ) - Mix.shell().info( + Common.shell_info( "Writing config to #{config_path}. You should rename it to config/prod.secret.exs or config/dev.secret.exs." ) File.write(config_path, result_config) - Mix.shell().info("Writing #{psql_path}.") + Common.shell_info("Writing #{psql_path}.") File.write(psql_path, result_psql) write_robots_txt(indexable) - Mix.shell().info( + Common.shell_info( "\n" <> """ To get started: @@ -179,7 +179,7 @@ def run(["gen" | rest]) do end ) else - Mix.shell().error( + Common.shell_error( "The task would have overwritten the following files:\n" <> (Enum.map(paths, &"- #{&1}\n") |> Enum.join("")) <> "Rerun with `--force` to overwrite them." @@ -204,10 +204,10 @@ defp write_robots_txt(indexable) do if File.exists?(robots_txt_path) do File.cp!(robots_txt_path, "#{robots_txt_path}.bak") - Mix.shell().info("Backing up existing robots.txt to #{robots_txt_path}.bak") + Common.shell_info("Backing up existing robots.txt to #{robots_txt_path}.bak") end File.write(robots_txt_path, robots_txt) - Mix.shell().info("Writing #{robots_txt_path}.") + Common.shell_info("Writing #{robots_txt_path}.") end end diff --git a/lib/mix/tasks/pleroma/relay.ex b/lib/mix/tasks/pleroma/relay.ex index fbec473c5..213ae24d2 100644 --- a/lib/mix/tasks/pleroma/relay.ex +++ b/lib/mix/tasks/pleroma/relay.ex @@ -30,7 +30,7 @@ def run(["follow", target]) do # put this task to sleep to allow the genserver to push out the messages :timer.sleep(500) else - {:error, e} -> Mix.shell().error("Error while following #{target}: #{inspect(e)}") + {:error, e} -> Common.shell_error("Error while following #{target}: #{inspect(e)}") end end @@ -41,7 +41,7 @@ def run(["unfollow", target]) do # put this task to sleep to allow the genserver to push out the messages :timer.sleep(500) else - {:error, e} -> Mix.shell().error("Error while following #{target}: #{inspect(e)}") + {:error, e} -> Common.shell_error("Error while following #{target}: #{inspect(e)}") end end end diff --git a/lib/mix/tasks/pleroma/uploads.ex b/lib/mix/tasks/pleroma/uploads.ex index 106fcf443..8855b5538 100644 --- a/lib/mix/tasks/pleroma/uploads.ex +++ b/lib/mix/tasks/pleroma/uploads.ex @@ -38,10 +38,10 @@ def run(["migrate_local", target_uploader | args]) do Pleroma.Config.put([Upload, :uploader], uploader) end - Mix.shell().info("Migrating files from local #{local_path} to #{to_string(uploader)}") + Common.shell_info("Migrating files from local #{local_path} to #{to_string(uploader)}") if delete? do - Mix.shell().info( + Common.shell_info( "Attention: uploaded files will be deleted, hope you have backups! (--delete ; cancel with ^C)" ) @@ -78,7 +78,7 @@ def run(["migrate_local", target_uploader | args]) do |> Enum.filter(& &1) total_count = length(uploads) - Mix.shell().info("Found #{total_count} uploads") + Common.shell_info("Found #{total_count} uploads") uploads |> Task.async_stream( @@ -90,7 +90,7 @@ def run(["migrate_local", target_uploader | args]) do :ok error -> - Mix.shell().error("failed to upload #{inspect(upload.path)}: #{inspect(error)}") + Common.shell_error("failed to upload #{inspect(upload.path)}: #{inspect(error)}") end end, timeout: 150_000 @@ -99,10 +99,10 @@ def run(["migrate_local", target_uploader | args]) do # credo:disable-for-next-line Credo.Check.Warning.UnusedEnumOperation |> Enum.reduce(0, fn done, count -> count = count + length(done) - Mix.shell().info("Uploaded #{count}/#{total_count} files") + Common.shell_info("Uploaded #{count}/#{total_count} files") count end) - Mix.shell().info("Done!") + Common.shell_info("Done!") end end diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index 25fc40ea7..7eaa49836 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -115,7 +115,7 @@ def run(["new", nickname, email | rest]) do admin? = Keyword.get(options, :admin, false) assume_yes? = Keyword.get(options, :assume_yes, false) - Mix.shell().info(""" + Common.shell_info(""" A user will be created with the following information: - nickname: #{nickname} - email: #{email} @@ -128,7 +128,7 @@ def run(["new", nickname, email | rest]) do - admin: #{if(admin?, do: "true", else: "false")} """) - proceed? = assume_yes? or Mix.shell().yes?("Continue?") + proceed? = assume_yes? or Common.shell_yes?("Continue?") if proceed? do Common.start_pleroma() @@ -145,7 +145,7 @@ def run(["new", nickname, email | rest]) do changeset = User.register_changeset(%User{}, params, need_confirmation: false) {:ok, _user} = User.register(changeset) - Mix.shell().info("User #{nickname} created") + Common.shell_info("User #{nickname} created") if moderator? do run(["set", nickname, "--moderator"]) @@ -159,7 +159,7 @@ def run(["new", nickname, email | rest]) do run(["reset_password", nickname]) end else - Mix.shell().info("User will not be created.") + Common.shell_info("User will not be created.") end end @@ -168,10 +168,10 @@ def run(["rm", nickname]) do with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do User.perform(:delete, user) - Mix.shell().info("User #{nickname} deleted.") + Common.shell_info("User #{nickname} deleted.") else _ -> - Mix.shell().error("No local user #{nickname}") + Common.shell_error("No local user #{nickname}") end end @@ -181,12 +181,12 @@ def run(["toggle_activated", nickname]) do with %User{} = user <- User.get_cached_by_nickname(nickname) do {:ok, user} = User.deactivate(user, !user.info.deactivated) - Mix.shell().info( + Common.shell_info( "Activation status of #{nickname}: #{if(user.info.deactivated, do: "de", else: "")}activated" ) else _ -> - Mix.shell().error("No user #{nickname}") + Common.shell_error("No user #{nickname}") end end @@ -195,7 +195,7 @@ def run(["reset_password", nickname]) do with %User{local: true} = user <- User.get_cached_by_nickname(nickname), {:ok, token} <- Pleroma.PasswordResetToken.create_token(user) do - Mix.shell().info("Generated password reset token for #{user.nickname}") + Common.shell_info("Generated password reset token for #{user.nickname}") IO.puts( "URL: #{ @@ -208,7 +208,7 @@ def run(["reset_password", nickname]) do ) else _ -> - Mix.shell().error("No local user #{nickname}") + Common.shell_error("No local user #{nickname}") end end @@ -216,7 +216,7 @@ def run(["unsubscribe", nickname]) do Common.start_pleroma() with %User{} = user <- User.get_cached_by_nickname(nickname) do - Mix.shell().info("Deactivating #{user.nickname}") + Common.shell_info("Deactivating #{user.nickname}") User.deactivate(user) {:ok, friends} = User.get_friends(user) @@ -224,7 +224,7 @@ def run(["unsubscribe", nickname]) do Enum.each(friends, fn friend -> user = User.get_cached_by_id(user.id) - Mix.shell().info("Unsubscribing #{friend.nickname} from #{user.nickname}") + Common.shell_info("Unsubscribing #{friend.nickname} from #{user.nickname}") User.unfollow(user, friend) end) @@ -233,11 +233,11 @@ def run(["unsubscribe", nickname]) do user = User.get_cached_by_id(user.id) if Enum.empty?(user.following) do - Mix.shell().info("Successfully unsubscribed all followers from #{user.nickname}") + Common.shell_info("Successfully unsubscribed all followers from #{user.nickname}") end else _ -> - Mix.shell().error("No user #{nickname}") + Common.shell_error("No user #{nickname}") end end @@ -274,7 +274,7 @@ def run(["set", nickname | rest]) do end else _ -> - Mix.shell().error("No local user #{nickname}") + Common.shell_error("No local user #{nickname}") end end @@ -284,10 +284,10 @@ def run(["tag", nickname | tags]) do with %User{} = user <- User.get_cached_by_nickname(nickname) do user = user |> User.tag(tags) - Mix.shell().info("Tags of #{user.nickname}: #{inspect(tags)}") + Common.shell_info("Tags of #{user.nickname}: #{inspect(tags)}") else _ -> - Mix.shell().error("Could not change user tags for #{nickname}") + Common.shell_error("Could not change user tags for #{nickname}") end end @@ -297,10 +297,10 @@ def run(["untag", nickname | tags]) do with %User{} = user <- User.get_cached_by_nickname(nickname) do user = user |> User.untag(tags) - Mix.shell().info("Tags of #{user.nickname}: #{inspect(tags)}") + Common.shell_info("Tags of #{user.nickname}: #{inspect(tags)}") else _ -> - Mix.shell().error("Could not change user tags for #{nickname}") + Common.shell_error("Could not change user tags for #{nickname}") end end @@ -326,7 +326,7 @@ def run(["invite" | rest]) do with {:ok, val} <- options[:expires_at], options = Map.put(options, :expires_at, val), {:ok, invite} <- UserInviteToken.create_invite(options) do - Mix.shell().info( + Common.shell_info( "Generated user invite token " <> String.replace(invite.invite_type, "_", " ") ) @@ -340,14 +340,14 @@ def run(["invite" | rest]) do IO.puts(url) else error -> - Mix.shell().error("Could not create invite token: #{inspect(error)}") + Common.shell_error("Could not create invite token: #{inspect(error)}") end end def run(["invites"]) do Common.start_pleroma() - Mix.shell().info("Invites list:") + Common.shell_info("Invites list:") UserInviteToken.list_invites() |> Enum.each(fn invite -> @@ -361,7 +361,7 @@ def run(["invites"]) do " | Max use: #{max_use} Left use: #{max_use - invite.uses}" end - Mix.shell().info( + Common.shell_info( "ID: #{invite.id} | Token: #{invite.token} | Token type: #{invite.invite_type} | Used: #{ invite.used }#{expire_info}#{using_info}" @@ -374,9 +374,9 @@ def run(["revoke_invite", token]) do with {:ok, invite} <- UserInviteToken.find_by_token(token), {:ok, _} <- UserInviteToken.update_invite(invite, %{used: true}) do - Mix.shell().info("Invite for token #{token} was revoked.") + Common.shell_info("Invite for token #{token} was revoked.") else - _ -> Mix.shell().error("No invite found with token #{token}") + _ -> Common.shell_error("No invite found with token #{token}") end end @@ -385,10 +385,10 @@ def run(["delete_activities", nickname]) do with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do {:ok, _} = User.delete_user_activities(user) - Mix.shell().info("User #{nickname} statuses deleted.") + Common.shell_info("User #{nickname} statuses deleted.") else _ -> - Mix.shell().error("No local user #{nickname}") + Common.shell_error("No local user #{nickname}") end end @@ -400,10 +400,10 @@ def run(["toggle_confirmed", nickname]) do message = if user.info.confirmation_pending, do: "needs", else: "doesn't need" - Mix.shell().info("#{nickname} #{message} confirmation.") + Common.shell_info("#{nickname} #{message} confirmation.") else _ -> - Mix.shell().error("No local user #{nickname}") + Common.shell_error("No local user #{nickname}") end end @@ -416,7 +416,7 @@ defp set_moderator(user, value) do {:ok, user} = User.update_and_set_cache(user_cng) - Mix.shell().info("Moderator status of #{user.nickname}: #{user.info.is_moderator}") + Common.shell_info("Moderator status of #{user.nickname}: #{user.info.is_moderator}") user end @@ -429,7 +429,7 @@ defp set_admin(user, value) do {:ok, user} = User.update_and_set_cache(user_cng) - Mix.shell().info("Admin status of #{user.nickname}: #{user.info.is_admin}") + Common.shell_info("Admin status of #{user.nickname}: #{user.info.is_admin}") user end @@ -442,7 +442,7 @@ defp set_locked(user, value) do {:ok, user} = User.update_and_set_cache(user_cng) - Mix.shell().info("Locked status of #{user.nickname}: #{user.info.locked}") + Common.shell_info("Locked status of #{user.nickname}: #{user.info.locked}") user end end diff --git a/lib/pleroma/release_tasks.ex b/lib/pleroma/release_tasks.ex new file mode 100644 index 000000000..66a8b627f --- /dev/null +++ b/lib/pleroma/release_tasks.ex @@ -0,0 +1,38 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ReleaseTasks do + def run(args) do + [task | args] = String.split(args) + + case task do + "migrate" -> migrate(args) + task -> mix_task(task, args) + end + end + + defp mix_task(task, args) do + # Modules are not loaded before application starts + Mix.Tasks.Pleroma.Common.start_pleroma() + {:ok, modules} = :application.get_key(:pleroma, :modules) + + module = + Enum.find(modules, fn module -> + module = Module.split(module) + + match?(["Mix", "Tasks", "Pleroma" | _], module) and + String.downcase(List.last(module)) == task + end) + + if module do + module.run(args) + else + IO.puts("The task #{task} does not exist") + end + end + + defp migrate(_args) do + :noop + end +end diff --git a/rel/pleroma_ctl b/rel/pleroma_ctl index 543b742b9..16526af44 100755 --- a/rel/pleroma_ctl +++ b/rel/pleroma_ctl @@ -2,4 +2,4 @@ # XXX: This should be removed when elixir's releases get custom command support SCRIPT=$(readlink -f "$0") SCRIPTPATH=$(dirname "$SCRIPT") -$SCRIPTPATH/pleroma eval 'Pleroma.ReleaseTasks.mix_task("'"$*"'")' +$SCRIPTPATH/pleroma eval 'Pleroma.ReleaseTasks.run("'"$*"'")' From 4f5149c93be8d0abffca2aa74984b03d8e27739d Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 8 Jun 2019 18:02:57 +0300 Subject: [PATCH 13/33] Set default loglevel to `warn` in prod It's rare that info logs are needed to debug the issue, so I would suggest setting them to warn in prod by default to make finding the relevant parts easier and potentially even decrease cpu usage on bigger instances Closes #962 --- CHANGELOG.md | 1 + config/prod.exs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf2232b09..510ad7ff5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Federation: Expand the audience of delete activities to all recipients of the deleted object - Federation: Removed `inReplyToStatusId` from objects - Configuration: Dedupe enabled by default +- Configuration: Default log level in `prod` environment is now set to `warn` - Configuration: Added `extra_cookie_attrs` for setting non-standard cookie attributes. Defaults to ["SameSite=Lax"] so that remote follows work. - Timelines: Messages involving people you have blocked will be excluded from the timeline in all cases instead of just repeats. - Admin API: Move the user related API to `api/pleroma/admin/users` diff --git a/config/prod.exs b/config/prod.exs index d0cfd1ac2..1179cf3b0 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -18,7 +18,7 @@ protocol: "http" # Do not print debug messages in production -config :logger, level: :info +config :logger, level: :warn # ## SSL Support # From 7223c1b643874f368937969be441c42f7eb55d14 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 8 Jun 2019 20:10:25 +0300 Subject: [PATCH 14/33] Use Mix.shell().yes? if available --- lib/mix/tasks/pleroma/common.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/common.ex b/lib/mix/tasks/pleroma/common.ex index 0e03a7872..7d50605af 100644 --- a/lib/mix/tasks/pleroma/common.ex +++ b/lib/mix/tasks/pleroma/common.ex @@ -37,7 +37,9 @@ def shell_prompt(prompt, defval \\ nil, defname \\ nil) do end def shell_yes?(message) do - shell_prompt(message, "Yn") in ~w(Yn Y y) + if mix_shell?(), + do: Mix.shell().yes?("Continue?"), + else: shell_prompt(message, "Continue?") in ~w(Yn Y y) end def shell_info(message) do From b6d2db42a759354bb21e2385021dfb6acfe29ef2 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 8 Jun 2019 21:26:00 +0300 Subject: [PATCH 15/33] Fix wrong placement of serve_endpoints --- config/prod.exs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/prod.exs b/config/prod.exs index cd5cdb087..adc1c4bb7 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -15,8 +15,9 @@ # which you typically run after static files are built. config :pleroma, Pleroma.Web.Endpoint, http: [port: 4000], - protocol: "http", - serve_endpoints: true + protocol: "http" + +config :phoenix, serve_endpoints: true # Do not print debug messages in production config :logger, level: :info From 2a659b35f16cacb39ea6dae2aefd3572f4be6783 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 9 Jun 2019 13:33:44 +0300 Subject: [PATCH 16/33] Add migrate/rollback to release tasks --- lib/pleroma/release_tasks.ex | 36 +++++++++++++++++++++++++++++++----- rel/pleroma_ctl | 20 +++++++++++++++++--- 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/release_tasks.ex b/lib/pleroma/release_tasks.ex index 66a8b627f..7726bc635 100644 --- a/lib/pleroma/release_tasks.ex +++ b/lib/pleroma/release_tasks.ex @@ -3,18 +3,21 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReleaseTasks do + @repo Pleroma.Repo + def run(args) do + Mix.Tasks.Pleroma.Common.start_pleroma() [task | args] = String.split(args) case task do - "migrate" -> migrate(args) + "migrate" -> migrate() + "create" -> create() + "rollback" -> rollback(String.to_integer(Enum.at(args, 0))) task -> mix_task(task, args) end end defp mix_task(task, args) do - # Modules are not loaded before application starts - Mix.Tasks.Pleroma.Common.start_pleroma() {:ok, modules} = :application.get_key(:pleroma, :modules) module = @@ -32,7 +35,30 @@ defp mix_task(task, args) do end end - defp migrate(_args) do - :noop + def migrate do + {:ok, _, _} = Ecto.Migrator.with_repo(@repo, &Ecto.Migrator.run(&1, :up, all: true)) + end + + def rollback(version) do + {:ok, _, _} = Ecto.Migrator.with_repo(@repo, &Ecto.Migrator.run(&1, :down, to: version)) + end + + def create do + case @repo.__adapter__.storage_up(@repo.config) do + :ok -> + IO.puts("The database for #{inspect(@repo)} has been created") + + {:error, :already_up} -> + IO.puts("The database for #{inspect(@repo)} has already been created") + + {:error, term} when is_binary(term) -> + IO.puts(:stderr, "The database for #{inspect(@repo)} couldn't be created: #{term}") + + {:error, term} -> + IO.puts( + :stderr, + "The database for #{inspect(@repo)} couldn't be created: #{inspect(term)}" + ) + end end end diff --git a/rel/pleroma_ctl b/rel/pleroma_ctl index 16526af44..6137f19d3 100755 --- a/rel/pleroma_ctl +++ b/rel/pleroma_ctl @@ -1,5 +1,19 @@ #!/bin/sh # XXX: This should be removed when elixir's releases get custom command support -SCRIPT=$(readlink -f "$0") -SCRIPTPATH=$(dirname "$SCRIPT") -$SCRIPTPATH/pleroma eval 'Pleroma.ReleaseTasks.run("'"$*"'")' +if [ -z "$1" ] || [ "$1" == "help" ]; then + echo "Usage: $(basename "$0") COMMAND [ARGS] + + The known commands are: + + create Create database schema (needs to be executed only once) + migrate Execute database migrations (needs to be done after updates) + rollback Rollback database migrations (needs to be done before downgrading) + + and any mix tasks under Pleroma namespace, for example \`mix pleroma.user COMMAND\` is + equialent to \`$(basename "$0") user COMMAND\` +" +else + SCRIPT=$(readlink -f "$0") + SCRIPTPATH=$(dirname "$SCRIPT") + $SCRIPTPATH/pleroma eval 'Pleroma.ReleaseTasks.run("'"$*"'")' +fi From bf391569cf83c3dec75fe1a6870ae0b9f228400b Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 9 Jun 2019 13:34:58 +0300 Subject: [PATCH 17/33] specify that a version is needed for rollback --- rel/pleroma_ctl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rel/pleroma_ctl b/rel/pleroma_ctl index 6137f19d3..1276bf339 100755 --- a/rel/pleroma_ctl +++ b/rel/pleroma_ctl @@ -5,9 +5,9 @@ if [ -z "$1" ] || [ "$1" == "help" ]; then The known commands are: - create Create database schema (needs to be executed only once) - migrate Execute database migrations (needs to be done after updates) - rollback Rollback database migrations (needs to be done before downgrading) + create Create database schema (needs to be executed only once) + migrate Execute database migrations (needs to be done after updates) + rollback [VERSION] Rollback database migrations (needs to be done before downgrading) and any mix tasks under Pleroma namespace, for example \`mix pleroma.user COMMAND\` is equialent to \`$(basename "$0") user COMMAND\` From cfcc0c87763f460f4f3329f4de64890fbf23e795 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 9 Jun 2019 13:56:41 +0300 Subject: [PATCH 18/33] Add a changelog entry for releases --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 510ad7ff5..4dafa0df5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - [MongooseIM](https://github.com/esl/MongooseIM) http authentication support. - LDAP authentication - External OAuth provider authentication +- Support for building a release using [`mix release`](https://hexdocs.pm/mix/master/Mix.Tasks.Release.html) - A [job queue](https://git.pleroma.social/pleroma/pleroma_job_queue) for federation, emails, web push, etc. - [Prometheus](https://prometheus.io/) metrics - Support for Mastodon's remote interaction From 365268d522323c6dcdf0ed977263cbf0f01c984b Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Mon, 10 Jun 2019 11:34:11 +0000 Subject: [PATCH 19/33] Add more tests for using media_proxy and whitelists --- test/media_proxy_test.exs | 15 +++ .../mastodon_api_controller_test.exs | 102 +++++++++++++----- 2 files changed, 91 insertions(+), 26 deletions(-) diff --git a/test/media_proxy_test.exs b/test/media_proxy_test.exs index 0a02039a6..b23aeb88b 100644 --- a/test/media_proxy_test.exs +++ b/test/media_proxy_test.exs @@ -149,6 +149,21 @@ test "encoding S3 links (must preserve `%2F`)" do encoded = url(url) assert decode_result(encoded) == url end + + test "does not change whitelisted urls" do + upload_config = Pleroma.Config.get([Pleroma.Upload]) + media_url = "https://media.pleroma.social" + Pleroma.Config.put([Pleroma.Upload, :base_url], media_url) + Pleroma.Config.put([:media_proxy, :whitelist], ["media.pleroma.social"]) + Pleroma.Config.put([:media_proxy, :base_url], "https://cache.pleroma.social") + + url = "#{media_url}/static/logo.png" + encoded = url(url) + + assert String.starts_with?(encoded, media_url) + + Pleroma.Config.put([Pleroma.Upload], upload_config) + end end describe "when disabled" do diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 33c8e209a..de32084bd 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1421,6 +1421,82 @@ test "returns the relationships for the current user", %{conn: conn} do end end + describe "media upload" do + setup do + upload_config = Pleroma.Config.get([Pleroma.Upload]) + proxy_config = Pleroma.Config.get([:media_proxy]) + + on_exit(fn -> + Pleroma.Config.put([Pleroma.Upload], upload_config) + Pleroma.Config.put([:media_proxy], proxy_config) + end) + + user = insert(:user) + + conn = + build_conn() + |> assign(:user, user) + + image = %Plug.Upload{ + content_type: "image/jpg", + path: Path.absname("test/fixtures/image.jpg"), + filename: "an_image.jpg" + } + + [conn: conn, image: image] + end + + test "returns uploaded image", %{conn: conn, image: image} do + desc = "Description of the image" + + media = + conn + |> post("/api/v1/media", %{"file" => image, "description" => desc}) + |> json_response(:ok) + + assert media["type"] == "image" + assert media["description"] == desc + assert media["id"] + + object = Repo.get(Object, media["id"]) + assert object.data["actor"] == User.ap_id(conn.assigns[:user]) + end + + test "returns proxied url when media proxy is enabled", %{conn: conn, image: image} do + Pleroma.Config.put([Pleroma.Upload, :base_url], "https://media.pleroma.social") + + proxy_url = "https://cache.pleroma.social" + Pleroma.Config.put([:media_proxy, :enabled], true) + Pleroma.Config.put([:media_proxy, :base_url], proxy_url) + + media = + conn + |> post("/api/v1/media", %{"file" => image}) + |> json_response(:ok) + + assert String.starts_with?(media["url"], proxy_url) + end + + test "returns media url when proxy is enabled but media url is whitelisted", %{ + conn: conn, + image: image + } do + media_url = "https://media.pleroma.social" + Pleroma.Config.put([Pleroma.Upload, :base_url], media_url) + + Pleroma.Config.put([:media_proxy, :enabled], true) + Pleroma.Config.put([:media_proxy, :base_url], "https://cache.pleroma.social") + Pleroma.Config.put([:media_proxy, :whitelist], ["media.pleroma.social"]) + + media = + conn + |> post("/api/v1/media", %{"file" => image}) + |> json_response(:ok) + + assert String.starts_with?(media["url"], media_url) + end + end + describe "locked accounts" do test "/api/v1/follow_requests works" do user = insert(:user, %{info: %User.Info{locked: true}}) @@ -1530,32 +1606,6 @@ test "account fetching also works nickname", %{conn: conn} do assert id == user.id end - test "media upload", %{conn: conn} do - file = %Plug.Upload{ - content_type: "image/jpg", - path: Path.absname("test/fixtures/image.jpg"), - filename: "an_image.jpg" - } - - desc = "Description of the image" - - user = insert(:user) - - conn = - conn - |> assign(:user, user) - |> post("/api/v1/media", %{"file" => file, "description" => desc}) - - assert media = json_response(conn, 200) - - assert media["type"] == "image" - assert media["description"] == desc - assert media["id"] - - object = Repo.get(Object, media["id"]) - assert object.data["actor"] == User.ap_id(user) - end - test "mascot upload", %{conn: conn} do user = insert(:user) From dbe4c2b7c8cfec4d8348de869a86c03015a7b7c5 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 10 Jun 2019 11:47:22 +0000 Subject: [PATCH 20/33] Update pleroma_ctl --- rel/pleroma_ctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rel/pleroma_ctl b/rel/pleroma_ctl index 1276bf339..ef2717c44 100755 --- a/rel/pleroma_ctl +++ b/rel/pleroma_ctl @@ -10,7 +10,7 @@ if [ -z "$1" ] || [ "$1" == "help" ]; then rollback [VERSION] Rollback database migrations (needs to be done before downgrading) and any mix tasks under Pleroma namespace, for example \`mix pleroma.user COMMAND\` is - equialent to \`$(basename "$0") user COMMAND\` + equivalent to \`$(basename "$0") user COMMAND\` " else SCRIPT=$(readlink -f "$0") From a7d956d3831f7a1dcd70f34b819084bd59eb536c Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Mon, 10 Jun 2019 13:20:37 +0000 Subject: [PATCH 21/33] Remove unused imports and aliases from migration --- ...719152213_add_follower_address_to_user.exs | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/priv/repo/migrations/20170719152213_add_follower_address_to_user.exs b/priv/repo/migrations/20170719152213_add_follower_address_to_user.exs index 4d163ce0b..591164be5 100644 --- a/priv/repo/migrations/20170719152213_add_follower_address_to_user.exs +++ b/priv/repo/migrations/20170719152213_add_follower_address_to_user.exs @@ -1,30 +1,10 @@ defmodule Pleroma.Repo.Migrations.AddFollowerAddressToUser do use Ecto.Migration - import Ecto.Query - import Supervisor.Spec - alias Pleroma.{Repo, User} def up do alter table(:users) do add :follower_address, :string, unique: true end - - # Not needed anymore for new setups. - # flush() - - # children = [ - # # Start the endpoint when the application starts - # supervisor(Pleroma.Web.Endpoint, []) - # ] - # opts = [strategy: :one_for_one, name: Pleroma.Supervisor] - # Supervisor.start_link(children, opts) - - # Enum.each(Repo.all(User), fn (user) -> - # if !user.follower_address do - # cs = Ecto.Changeset.change(user, %{follower_address: User.ap_followers(user)}) - # Repo.update!(cs) - # end - # end) end def down do From 2e5affce61a9255602d3a5d4c5caced9f09b1f5a Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 11 Jun 2019 14:27:41 +0700 Subject: [PATCH 22/33] Add RateLimiter --- docs/config.md | 11 +++ lib/pleroma/plugs/rate_limiter.ex | 87 ++++++++++++++++++++++++ mix.exs | 2 +- test/plugs/rate_limiter_test.exs | 108 ++++++++++++++++++++++++++++++ 4 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 lib/pleroma/plugs/rate_limiter.ex create mode 100644 test/plugs/rate_limiter_test.exs diff --git a/docs/config.md b/docs/config.md index c61a5d8a3..e31e2b90f 100644 --- a/docs/config.md +++ b/docs/config.md @@ -616,3 +616,14 @@ To enable them, both the `rum_enabled` flag has to be set and the following spec `mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/` This will probably take a long time. + +## :rate_limit + +A keyword list of rate limiters where a key is a limiter name and value is the limiter configuration. The basic configuration is a tuple where: + +* The first element: `scale` (Integer). The time scale in milliseconds. +* The second element: `limit` (Integer). How many requests to limit in the time scale provided. + +It is also possible to have different limits for unauthenticated and authenticated users: the keyword value must be a list of two tuples where the first one is a config for unauthenticated users and the second one is for authenticated. + +See [`Pleroma.Plugs.RateLimiter`](Pleroma.Plugs.RateLimiter.html) documentation for examples. diff --git a/lib/pleroma/plugs/rate_limiter.ex b/lib/pleroma/plugs/rate_limiter.ex new file mode 100644 index 000000000..e02ba4213 --- /dev/null +++ b/lib/pleroma/plugs/rate_limiter.ex @@ -0,0 +1,87 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Plugs.RateLimiter do + @moduledoc """ + + ## Configuration + + A keyword list of rate limiters where a key is a limiter name and value is the limiter configuration. The basic configuration is a tuple where: + + * The first element: `scale` (Integer). The time scale in milliseconds. + * The second element: `limit` (Integer). How many requests to limit in the time scale provided. + + It is also possible to have different limits for unauthenticated and authenticated users: the keyword value must be a list of two tuples where the first one is a config for unauthenticated users and the second one is for authenticated. + + ### Example + + config :pleroma, :rate_limit, + one: {1000, 10}, + two: [{10_000, 10}, {10_000, 50}] + + Here we have two limiters: `one` which is not over 10req/1s and `two` which has two limits 10req/10s for unauthenticated users and 50req/10s for authenticated users. + + ## Usage + + Inside a controller: + + plug(Pleroma.Plugs.RateLimiter, :one when action == :one) + plug(Pleroma.Plugs.RateLimiter, :two when action in [:two, :three]) + + or inside a router pipiline: + + pipeline :api do + ... + plug(Pleroma.Plugs.RateLimiter, :one) + ... + end + """ + + import Phoenix.Controller, only: [json: 2] + import Plug.Conn + + alias Pleroma.User + + def init(limiter_name) do + case Pleroma.Config.get([:rate_limit, limiter_name]) do + nil -> nil + config -> {limiter_name, config} + end + end + + # do not limit if there is no limiter configuration + def call(conn, nil), do: conn + + def call(conn, opts) do + case check_rate(conn, opts) do + {:ok, _count} -> conn + {:error, _count} -> render_error(conn) + end + end + + defp check_rate(%{assigns: %{user: %User{id: user_id}}}, {limiter_name, [_, {scale, limit}]}) do + ExRated.check_rate("#{limiter_name}:#{user_id}", scale, limit) + end + + defp check_rate(conn, {limiter_name, [{scale, limit} | _]}) do + ExRated.check_rate("#{limiter_name}:#{ip(conn)}", scale, limit) + end + + defp check_rate(conn, {limiter_name, {scale, limit}}) do + check_rate(conn, {limiter_name, [{scale, limit}]}) + end + + def ip(%{remote_ip: remote_ip}) do + remote_ip + |> Tuple.to_list() + |> Enum.join(".") + end + + defp render_error(conn) do + conn + |> put_status(:too_many_requests) + |> json(%{error: "Throttled"}) + |> halt() + end +end diff --git a/mix.exs b/mix.exs index 9447a2e4f..1b78c5ca8 100644 --- a/mix.exs +++ b/mix.exs @@ -129,7 +129,7 @@ defp deps do {:quack, "~> 0.1.1"}, {:benchee, "~> 1.0"}, {:esshd, "~> 0.1.0", runtime: Application.get_env(:esshd, :enabled, false)}, - {:ex_rated, "~> 1.2"}, + {:ex_rated, "~> 1.3"}, {:plug_static_index_html, "~> 1.0.0"}, {:excoveralls, "~> 0.11.1", only: :test} ] ++ oauth_deps() diff --git a/test/plugs/rate_limiter_test.exs b/test/plugs/rate_limiter_test.exs new file mode 100644 index 000000000..b3798bf03 --- /dev/null +++ b/test/plugs/rate_limiter_test.exs @@ -0,0 +1,108 @@ +defmodule Pleroma.Plugs.RateLimiterTest do + use ExUnit.Case, async: true + use Plug.Test + + alias Pleroma.Plugs.RateLimiter + + import Pleroma.Factory + + @limiter_name :testing + + test "init/1" do + Pleroma.Config.put([:rate_limit, @limiter_name], {1, 1}) + + assert {@limiter_name, {1, 1}} == RateLimiter.init(@limiter_name) + assert nil == RateLimiter.init(:foo) + end + + test "ip/1" do + assert "127.0.0.1" == RateLimiter.ip(%{remote_ip: {127, 0, 0, 1}}) + end + + test "it restricts by opts" do + scale = 100 + limit = 5 + + Pleroma.Config.put([:rate_limit, @limiter_name], {scale, limit}) + + opts = RateLimiter.init(@limiter_name) + conn = conn(:get, "/") + bucket_name = "#{@limiter_name}:#{RateLimiter.ip(conn)}" + + conn = RateLimiter.call(conn, opts) + assert {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {2, 3, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {3, 2, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {4, 1, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {5, 0, to_reset, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + + assert %{"error" => "Throttled"} = Phoenix.ConnTest.json_response(conn, :too_many_requests) + assert conn.halted + + Process.sleep(to_reset) + + conn = conn(:get, "/") + + conn = RateLimiter.call(conn, opts) + assert {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + refute conn.status == Plug.Conn.Status.code(:too_many_requests) + refute conn.resp_body + refute conn.halted + end + + test "optional limits for authenticated users" do + Ecto.Adapters.SQL.Sandbox.checkout(Pleroma.Repo) + + scale = 100 + limit = 5 + Pleroma.Config.put([:rate_limit, @limiter_name], [{1, 10}, {scale, limit}]) + + opts = RateLimiter.init(@limiter_name) + + user = insert(:user) + conn = conn(:get, "/") |> assign(:user, user) + bucket_name = "#{@limiter_name}:#{user.id}" + + conn = RateLimiter.call(conn, opts) + assert {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {2, 3, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {3, 2, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {4, 1, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {5, 0, to_reset, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + + assert %{"error" => "Throttled"} = Phoenix.ConnTest.json_response(conn, :too_many_requests) + assert conn.halted + + Process.sleep(to_reset) + + conn = conn(:get, "/") |> assign(:user, user) + + conn = RateLimiter.call(conn, opts) + assert {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + refute conn.status == Plug.Conn.Status.code(:too_many_requests) + refute conn.resp_body + refute conn.halted + end +end From bc8f0593670452851d5e9d97bea1ae90f10db354 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 11 Jun 2019 14:28:39 +0700 Subject: [PATCH 23/33] Add rate limiting for search endpoints --- config/config.exs | 2 ++ lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 2 ++ 2 files changed, 4 insertions(+) diff --git a/config/config.exs b/config/config.exs index 4e2b1703b..d20d4fda9 100644 --- a/config/config.exs +++ b/config/config.exs @@ -503,6 +503,8 @@ config :http_signatures, adapter: Pleroma.Signature +config :pleroma, :rate_limit, search: [{1000, 10}, {1000, 30}] + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 92cd77f62..20b08fda4 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -55,6 +55,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do when action in [:account_register] ) + plug(Pleroma.Plugs.RateLimiter, :search when action in [:search, :search2, :account_search]) + @local_mastodon_name "Mastodon-Local" action_fallback(:errors) From f26013cf2ee9c04d4b99819991aaa5936e41cba4 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 11 Jun 2019 14:36:51 +0700 Subject: [PATCH 24/33] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf2232b09..cbdaa9c6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - MRF: Support for stripping avatars and banner images from specific instances (`mrf_simple`) - MRF: Support for running subchains. - Configuration: `skip_thread_containment` option +- Configuration: `rate_limit` option. See `Pleroma.Plugs.RateLimiter` documentation for details. ### Changed - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer From ad04d12de63d559cc6398c58296afd04321adfbc Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 11 Jun 2019 16:06:03 +0700 Subject: [PATCH 25/33] Replace `MastodonAPIController.account_register/2` rate limiter --- config/config.exs | 6 +-- config/test.exs | 2 +- docs/config.md | 8 +-- lib/pleroma/plugs/rate_limit_plug.ex | 36 ------------- .../mastodon_api/mastodon_api_controller.ex | 10 +--- test/plugs/rate_limit_plug_test.exs | 50 ------------------- .../mastodon_api_controller_test.exs | 20 +------- 7 files changed, 7 insertions(+), 125 deletions(-) delete mode 100644 lib/pleroma/plugs/rate_limit_plug.ex delete mode 100644 test/plugs/rate_limit_plug_test.exs diff --git a/config/config.exs b/config/config.exs index d20d4fda9..3d2c6d48e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -247,8 +247,6 @@ skip_thread_containment: false, limit_unauthenticated_to_local_content: true -config :pleroma, :app_account_creation, enabled: true, max_requests: 25, interval: 1800 - config :pleroma, :markup, # XXX - unfortunately, inline images must be enabled by default right now, because # of custom emoji. Issue #275 discusses defanging that somehow. @@ -503,7 +501,9 @@ config :http_signatures, adapter: Pleroma.Signature -config :pleroma, :rate_limit, search: [{1000, 10}, {1000, 30}] +config :pleroma, :rate_limit, + search: [{1000, 10}, {1000, 30}], + app_account_creation: {1_800_000, 25} # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. diff --git a/config/test.exs b/config/test.exs index 7861b9598..95129f409 100644 --- a/config/test.exs +++ b/config/test.exs @@ -59,7 +59,7 @@ total_user_limit: 3, enabled: false -config :pleroma, :app_account_creation, max_requests: 5 +config :pleroma, :rate_limit, app_account_creation: {1000, 5} config :pleroma, :http_security, report_uri: "https://endpoint.com" diff --git a/docs/config.md b/docs/config.md index e31e2b90f..b62b80490 100644 --- a/docs/config.md +++ b/docs/config.md @@ -114,12 +114,6 @@ config :pleroma, Pleroma.Emails.Mailer, * `skip_thread_containment`: Skip filter out broken threads. The default is `false`. * `limit_unauthenticated_to_local_content`: Limit unauthenticated users to search for local statutes and users only. The default is `true`. -## :app_account_creation -REST API for creating an account settings -* `enabled`: Enable/disable registration -* `max_requests`: Number of requests allowed for creating accounts -* `interval`: Interval for restricting requests for one ip (seconds) - ## :logger * `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog, and `Quack.Logger` to log to Slack @@ -568,7 +562,7 @@ config :ueberauth, Ueberauth, providers: [ microsoft: {Ueberauth.Strategy.Microsoft, [callback_params: []]} ] - + # Keycloak # Note: make sure to add `keycloak:ueberauth_keycloak_strategy` entry to `OAUTH_CONSUMER_STRATEGIES` environment variable keycloak_url = "https://publicly-reachable-keycloak-instance.org:8080" diff --git a/lib/pleroma/plugs/rate_limit_plug.ex b/lib/pleroma/plugs/rate_limit_plug.ex deleted file mode 100644 index 466f64a79..000000000 --- a/lib/pleroma/plugs/rate_limit_plug.ex +++ /dev/null @@ -1,36 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Plugs.RateLimitPlug do - import Phoenix.Controller, only: [json: 2] - import Plug.Conn - - def init(opts), do: opts - - def call(conn, opts) do - enabled? = Pleroma.Config.get([:app_account_creation, :enabled]) - - case check_rate(conn, Map.put(opts, :enabled, enabled?)) do - {:ok, _count} -> conn - {:error, _count} -> render_error(conn) - %Plug.Conn{} = conn -> conn - end - end - - defp check_rate(conn, %{enabled: true} = opts) do - max_requests = opts[:max_requests] - bucket_name = conn.remote_ip |> Tuple.to_list() |> Enum.join(".") - - ExRated.check_rate(bucket_name, opts[:interval] * 1000, max_requests) - end - - defp check_rate(conn, _), do: conn - - defp render_error(conn) do - conn - |> put_status(:forbidden) - |> json(%{error: "Rate limit exceeded."}) - |> halt() - end -end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 20b08fda4..46049dd24 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -46,15 +46,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do require Logger - plug( - Pleroma.Plugs.RateLimitPlug, - %{ - max_requests: Config.get([:app_account_creation, :max_requests]), - interval: Config.get([:app_account_creation, :interval]) - } - when action in [:account_register] - ) - + plug(Pleroma.Plugs.RateLimiter, :app_account_creation when action == :account_register) plug(Pleroma.Plugs.RateLimiter, :search when action in [:search, :search2, :account_search]) @local_mastodon_name "Mastodon-Local" diff --git a/test/plugs/rate_limit_plug_test.exs b/test/plugs/rate_limit_plug_test.exs deleted file mode 100644 index 2ec9a8fb7..000000000 --- a/test/plugs/rate_limit_plug_test.exs +++ /dev/null @@ -1,50 +0,0 @@ -defmodule Pleroma.Plugs.RateLimitPlugTest do - use ExUnit.Case, async: true - use Plug.Test - - alias Pleroma.Plugs.RateLimitPlug - - @opts RateLimitPlug.init(%{max_requests: 5, interval: 1}) - - setup do - enabled = Pleroma.Config.get([:app_account_creation, :enabled]) - - Pleroma.Config.put([:app_account_creation, :enabled], true) - - on_exit(fn -> - Pleroma.Config.put([:app_account_creation, :enabled], enabled) - end) - - :ok - end - - test "it restricts by opts" do - conn = conn(:get, "/") - bucket_name = conn.remote_ip |> Tuple.to_list() |> Enum.join(".") - ms = 1000 - - conn = RateLimitPlug.call(conn, @opts) - {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, ms, 5) - conn = RateLimitPlug.call(conn, @opts) - {2, 3, _, _, _} = ExRated.inspect_bucket(bucket_name, ms, 5) - conn = RateLimitPlug.call(conn, @opts) - {3, 2, _, _, _} = ExRated.inspect_bucket(bucket_name, ms, 5) - conn = RateLimitPlug.call(conn, @opts) - {4, 1, _, _, _} = ExRated.inspect_bucket(bucket_name, ms, 5) - conn = RateLimitPlug.call(conn, @opts) - {5, 0, to_reset, _, _} = ExRated.inspect_bucket(bucket_name, ms, 5) - conn = RateLimitPlug.call(conn, @opts) - assert conn.status == 403 - assert conn.halted - assert conn.resp_body == "{\"error\":\"Rate limit exceeded.\"}" - - Process.sleep(to_reset) - - conn = conn(:get, "/") - conn = RateLimitPlug.call(conn, @opts) - {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, ms, 5) - refute conn.status == 403 - refute conn.halted - refute conn.resp_body - end -end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 33c8e209a..c569ae9dd 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -3501,24 +3501,6 @@ test "Repeated posts that are replies incorrectly have in_reply_to_id null", %{c end describe "create account by app" do - setup do - enabled = Pleroma.Config.get([:app_account_creation, :enabled]) - max_requests = Pleroma.Config.get([:app_account_creation, :max_requests]) - interval = Pleroma.Config.get([:app_account_creation, :interval]) - - Pleroma.Config.put([:app_account_creation, :enabled], true) - Pleroma.Config.put([:app_account_creation, :max_requests], 5) - Pleroma.Config.put([:app_account_creation, :interval], 1) - - on_exit(fn -> - Pleroma.Config.put([:app_account_creation, :enabled], enabled) - Pleroma.Config.put([:app_account_creation, :max_requests], max_requests) - Pleroma.Config.put([:app_account_creation, :interval], interval) - end) - - :ok - end - test "Account registration via Application", %{conn: conn} do conn = conn @@ -3621,7 +3603,7 @@ test "rate limit", %{conn: conn} do agreement: true }) - assert json_response(conn, 403) == %{"error" => "Rate limit exceeded."} + assert json_response(conn, :too_many_requests) == %{"error" => "Throttled"} end end From 7c063a898d60cadd324087999226314586b72bd3 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 11 Jun 2019 16:25:47 +0700 Subject: [PATCH 26/33] Update `ex_rated` dependency --- mix.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.lock b/mix.lock index 857bfca79..0b21d9761 100644 --- a/mix.lock +++ b/mix.lock @@ -29,7 +29,7 @@ "ex_aws_s3": {:hex, :ex_aws_s3, "2.0.1", "9e09366e77f25d3d88c5393824e613344631be8db0d1839faca49686e99b6704", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm"}, "ex_doc": {:hex, :ex_doc, "0.20.2", "1bd0dfb0304bade58beb77f20f21ee3558cc3c753743ae0ddbb0fd7ba2912331", [:mix], [{:earmark, "~> 1.3", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, "ex_machina": {:hex, :ex_machina, "2.3.0", "92a5ad0a8b10ea6314b876a99c8c9e3f25f4dde71a2a835845b136b9adaf199a", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm"}, - "ex_rated": {:hex, :ex_rated, "1.3.2", "6aeb32abb46ea6076f417a9ce8cb1cf08abf35fb2d42375beaad4dd72b550bf1", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm"}, + "ex_rated": {:hex, :ex_rated, "1.3.3", "30ecbdabe91f7eaa9d37fa4e81c85ba420f371babeb9d1910adbcd79ec798d27", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm"}, "ex_syslogger": {:git, "https://github.com/slashmili/ex_syslogger.git", "f3963399047af17e038897c69e20d552e6899e1d", [tag: "1.4.0"]}, "excoveralls": {:hex, :excoveralls, "0.11.1", "dd677fbdd49114fdbdbf445540ec735808250d56b011077798316505064edb2c", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, "floki": {:hex, :floki, "0.20.4", "be42ac911fece24b4c72f3b5846774b6e61b83fe685c2fc9d62093277fb3bc86", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}, {:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, From 581d8d3035142572eb6d46cb7cde8def7f022067 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 11 Jun 2019 16:50:35 +0700 Subject: [PATCH 27/33] Update `plug` dependency --- mix.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.lock b/mix.lock index 857bfca79..1f30d0e99 100644 --- a/mix.lock +++ b/mix.lock @@ -60,7 +60,7 @@ "phoenix_html": {:hex, :phoenix_html, "2.13.1", "fa8f034b5328e2dfa0e4131b5569379003f34bc1fafdaa84985b0b9d2f12e68b", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "1.1.1", "6668d787e602981f24f17a5fbb69cc98f8ab085114ebfac6cc36e10a90c8e93c", [:mix], [], "hexpm"}, "pleroma_job_queue": {:hex, :pleroma_job_queue, "0.2.0", "879e660aa1cebe8dc6f0aaaa6aa48b4875e89cd961d4a585fd128e0773b31a18", [:mix], [], "hexpm"}, - "plug": {:hex, :plug, "1.7.2", "d7b7db7fbd755e8283b6c0a50be71ec0a3d67d9213d74422d9372effc8e87fd1", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}], "hexpm"}, + "plug": {:hex, :plug, "1.8.2", "0bcce1daa420f189a6491f3940cc77ea7fb1919761175c9c3b59800d897440fc", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm"}, "plug_cowboy": {:hex, :plug_cowboy, "2.0.1", "d798f8ee5acc86b7d42dbe4450b8b0dadf665ce588236eb0a751a132417a980e", [:mix], [{:cowboy, "~> 2.5", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"}, "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, From 5d5a9a3a35cb06c7cbd9c29fdbf7cc0b866785fa Mon Sep 17 00:00:00 2001 From: Hakaba Hitoyo Date: Tue, 11 Jun 2019 11:34:22 +0000 Subject: [PATCH 28/33] Better default parameters for suggestion --- config/config.exs | 4 ++-- docs/config/howto_user_recomendation.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/config.exs b/config/config.exs index 0642ee3c3..b73541213 100644 --- a/config/config.exs +++ b/config/config.exs @@ -360,8 +360,8 @@ third_party_engine: "http://vinayaka.distsn.org/cgi-bin/vinayaka-user-match-suggestions-api.cgi?{{host}}+{{user}}", timeout: 300_000, - limit: 23, - web: "https://vinayaka.distsn.org/?{{host}}+{{user}}" + limit: 40, + web: "https://vinayaka.distsn.org" config :pleroma, :http_security, enabled: true, diff --git a/docs/config/howto_user_recomendation.md b/docs/config/howto_user_recomendation.md index 27c0760dd..c4d749d0c 100644 --- a/docs/config/howto_user_recomendation.md +++ b/docs/config/howto_user_recomendation.md @@ -9,8 +9,8 @@ config :pleroma, :suggestions, third_party_engine: "http://vinayaka.distsn.org/cgi-bin/vinayaka-user-match-suggestions-api.cgi?{{host}}+{{user}}", timeout: 300_000, - limit: 23, - web: "https://vinayaka.distsn.org/?{{host}}+{{user}}" + limit: 40, + web: "https://vinayaka.distsn.org" ``` @@ -26,6 +26,6 @@ config :pleroma, :suggestions, third_party_engine: "http://vinayaka.distsn.org/cgi-bin/vinayaka-user-new-suggestions-api.cgi?{{host}}+{{user}}", timeout: 60_000, - limit: 23, + limit: 40, web: "https://vinayaka.distsn.org/user-new.html" ``` From 6f29865d43f30303bc05bfb10aa28fe3ebef1bfd Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 11 Jun 2019 21:25:53 +0700 Subject: [PATCH 29/33] Add option to restrict all users to local content --- CHANGELOG.md | 2 +- config/config.exs | 2 +- docs/config.md | 3 ++- lib/pleroma/activity/search.ex | 17 +++++++------- lib/pleroma/user/search.ex | 43 +++++++++++++++++----------------- test/activity_test.exs | 13 +++++++--- test/user_test.exs | 18 +++++++++++--- 7 files changed, 59 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2900cc97..7ecdfe939 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Configuration: `notify_email` option - Configuration: Media proxy `whitelist` option - Configuration: `report_uri` option -- Configuration: `limit_unauthenticated_to_local_content` option +- Configuration: `limit_to_local_content` option - Pleroma API: User subscriptions - Pleroma API: Healthcheck endpoint - Pleroma API: `/api/v1/pleroma/mascot` per-user frontend mascot configuration endpoints diff --git a/config/config.exs b/config/config.exs index b73541213..f866e8d2b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -245,7 +245,7 @@ healthcheck: false, remote_post_retention_days: 90, skip_thread_containment: true, - limit_unauthenticated_to_local_content: true + limit_to_local_content: :unauthenticated config :pleroma, :markup, # XXX - unfortunately, inline images must be enabled by default right now, because diff --git a/docs/config.md b/docs/config.md index b62b80490..9e877fb51 100644 --- a/docs/config.md +++ b/docs/config.md @@ -112,7 +112,8 @@ config :pleroma, Pleroma.Emails.Mailer, * `healthcheck`: If set to true, system data will be shown on ``/api/pleroma/healthcheck``. * `remote_post_retention_days`: The default amount of days to retain remote posts when pruning the database. * `skip_thread_containment`: Skip filter out broken threads. The default is `false`. -* `limit_unauthenticated_to_local_content`: Limit unauthenticated users to search for local statutes and users only. The default is `true`. +* `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. + ## :logger * `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog, and `Quack.Logger` to log to Slack diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex index 9ccedcd13..8cbb64cc4 100644 --- a/lib/pleroma/activity/search.ex +++ b/lib/pleroma/activity/search.ex @@ -56,18 +56,19 @@ defp query_with(q, :rum, search_query) do ) end - # users can search everything - defp maybe_restrict_local(q, %User{}), do: q + defp maybe_restrict_local(q, user) do + limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated) - # unauthenticated users can only search local activities - defp maybe_restrict_local(q, _) do - if Pleroma.Config.get([:instance, :limit_unauthenticated_to_local_content], true) do - where(q, local: true) - else - q + case {limit, user} do + {:all, _} -> restrict_local(q) + {:unauthenticated, %User{}} -> q + {:unauthenticated, _} -> restrict_local(q) + {false, _} -> q end end + defp restrict_local(q), do: where(q, local: true) + defp maybe_fetch(activities, user, search_query) do with true <- Regex.match?(~r/https?:/, search_query), {:ok, object} <- Fetcher.fetch_object_from_id(search_query), diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index add6a0bbf..f88dffa7b 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -28,16 +28,6 @@ def search(query, opts \\ []) do results end - defp maybe_resolve(true, %User{}, query) do - User.get_or_fetch(query) - end - - defp maybe_resolve(true, _, query) do - unless restrict_local?(), do: User.get_or_fetch(query) - end - - defp maybe_resolve(_, _, _), do: :noop - defp search_query(query, for_user) do query |> union_query() @@ -49,10 +39,6 @@ defp search_query(query, for_user) do |> maybe_restrict_local(for_user) end - defp restrict_local? do - Pleroma.Config.get([:instance, :limit_unauthenticated_to_local_content], true) - end - defp union_query(query) do fts_subquery = fts_search_subquery(query) trigram_subquery = trigram_search_subquery(query) @@ -64,17 +50,30 @@ defp distinct_query(q) do from(s in subquery(q), order_by: s.search_type, distinct: s.id) end - # unauthenticated users can only search local activities - defp maybe_restrict_local(q, %User{}), do: q - - defp maybe_restrict_local(q, _) do - if restrict_local?() do - where(q, [u], u.local == true) - else - q + defp maybe_resolve(true, user, query) do + case {limit(), user} do + {:all, _} -> :noop + {:unauthenticated, %User{}} -> User.get_or_fetch(query) + {:unauthenticated, _} -> :noop + {false, _} -> User.get_or_fetch(query) end end + defp maybe_resolve(_, _, _), do: :noop + + defp maybe_restrict_local(q, user) do + case {limit(), user} do + {:all, _} -> restrict_local(q) + {:unauthenticated, %User{}} -> q + {:unauthenticated, _} -> restrict_local(q) + {false, _} -> q + end + end + + defp limit, do: Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated) + + defp restrict_local(q), do: where(q, [u], u.local == true) + defp boost_search_rank_query(query, nil), do: query defp boost_search_rank_query(query, for_user) do diff --git a/test/activity_test.exs b/test/activity_test.exs index e56e39096..7ba4363c8 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -139,18 +139,25 @@ test "find only local statuses for unauthenticated users", %{local_activity: loc assert [^local_activity] = Activity.search(nil, "find me") end - test "find all statuses for unauthenticated users when `limit_unauthenticated_to_local_content` is `false`", + test "find only local statuses for unauthenticated users when `limit_to_local_content` is `:all`", + %{local_activity: local_activity} do + Pleroma.Config.put([:instance, :limit_to_local_content], :all) + assert [^local_activity] = Activity.search(nil, "find me") + Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) + end + + test "find all statuses for unauthenticated users when `limit_to_local_content` is `false`", %{ local_activity: local_activity, remote_activity: remote_activity } do - Pleroma.Config.put([:instance, :limit_unauthenticated_to_local_content], false) + Pleroma.Config.put([:instance, :limit_to_local_content], false) activities = Enum.sort_by(Activity.search(nil, "find me"), & &1.id) assert [^local_activity, ^remote_activity] = activities - Pleroma.Config.put([:instance, :limit_unauthenticated_to_local_content], true) + Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) end end end diff --git a/test/user_test.exs b/test/user_test.exs index 8dd672173..473f545ff 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1099,8 +1099,20 @@ test "find only local users for unauthenticated users" do assert [%{id: ^id}] = User.search("lain") end - test "find all users for unauthenticated users when `limit_unauthenticated_to_local_content` is `false`" do - Pleroma.Config.put([:instance, :limit_unauthenticated_to_local_content], false) + test "find only local users for authenticated users when `limit_to_local_content` is `:all`" do + Pleroma.Config.put([:instance, :limit_to_local_content], :all) + + %{id: id} = insert(:user, %{name: "lain"}) + insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false}) + insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false}) + + assert [%{id: ^id}] = User.search("lain") + + Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) + end + + test "find all users for unauthenticated users when `limit_to_local_content` is `false`" do + Pleroma.Config.put([:instance, :limit_to_local_content], false) u1 = insert(:user, %{name: "lain"}) u2 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false}) @@ -1114,7 +1126,7 @@ test "find all users for unauthenticated users when `limit_unauthenticated_to_lo assert [u1.id, u2.id, u3.id] == results - Pleroma.Config.put([:instance, :limit_unauthenticated_to_local_content], true) + Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) end test "finds a user whose name is nil" do From f0d96534a4645f0996b3fab7ab5ba3b482fd4e23 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 11 Jun 2019 18:55:55 +0300 Subject: [PATCH 30/33] Import release config from env variable or /etc/pleroma/config.exs and warn if the file is missing --- config/releases.exs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/config/releases.exs b/config/releases.exs index becde7693..f8494dd34 100644 --- a/config/releases.exs +++ b/config/releases.exs @@ -1 +1,16 @@ import Config + +config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs" + +if File.exists?(config_path) do + import_config config_path +else + warning = [ + IO.ANSI.red(), + IO.ANSI.bright(), + "!!! #{config_path} not found! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file", + IO.ANSI.reset() + ] + + IO.puts(warning) +end From ced59be1ae8deb4dd505215062d45be3e262710e Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 11 Jun 2019 22:15:28 +0300 Subject: [PATCH 31/33] Document TagPolicy in `rewrite_policy` section --- docs/config.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/config.md b/docs/config.md index b62b80490..54632fbda 100644 --- a/docs/config.md +++ b/docs/config.md @@ -86,6 +86,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `Pleroma.Web.ActivityPub.MRF.NoOpPolicy`: Doesn’t modify activities (default) * `Pleroma.Web.ActivityPub.MRF.DropPolicy`: Drops all activities. It generally doesn’t makes sense to use in production * `Pleroma.Web.ActivityPub.MRF.SimplePolicy`: Restrict the visibility of activities from certains instances (See ``:mrf_simple`` section) + * `Pleroma.Web.ActivityPub.MRF.TagPolicy`: Applies policies to individual users based on tags, which can be set using pleroma-fe/admin-fe/any other app that supports Pleroma Admin API. For example it allows marking posts from individual users nsfw (sensitive) * `Pleroma.Web.ActivityPub.MRF.SubchainPolicy`: Selectively runs other MRF policies when messages match (see ``:mrf_subchain`` section) * `Pleroma.Web.ActivityPub.MRF.RejectNonPublic`: Drops posts with non-public visibility settings (See ``:mrf_rejectnonpublic`` section) * `Pleroma.Web.ActivityPub.MRF.EnsureRePrepended`: Rewrites posts to ensure that replies to posts with subjects do not have an identical subject and instead begin with re:. From bf22ed5fbd5d5087eb08e0bf0dc2ff6a29e90e0d Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 12 Jun 2019 15:53:33 +0700 Subject: [PATCH 32/33] Update `auto_linker` dependency --- lib/pleroma/web/rich_media/helpers.ex | 2 +- mix.exs | 2 +- mix.lock | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 9bc8f2559..f377125d7 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do alias Pleroma.Web.RichMedia.Parser defp validate_page_url(page_url) when is_binary(page_url) do - if AutoLinker.Parser.is_url?(page_url, true) do + if AutoLinker.Parser.url?(page_url, true) do URI.parse(page_url) |> validate_page_url else :error diff --git a/mix.exs b/mix.exs index bfa9bf18b..db7a30f99 100644 --- a/mix.exs +++ b/mix.exs @@ -126,7 +126,7 @@ defp deps do {:ueberauth, "~> 0.4"}, {:auto_linker, git: "https://git.pleroma.social/pleroma/auto_linker.git", - ref: "c00c4e75b35367fa42c95ffd9b8c455bf9995829"}, + ref: "e2385402bcd24fc659fee83b3eb8863b0528ad42"}, {:http_signatures, git: "https://git.pleroma.social/pleroma/http_signatures.git", ref: "9789401987096ead65646b52b5a2ca6bf52fc531"}, diff --git a/mix.lock b/mix.lock index de9d811e6..90cdf41f7 100644 --- a/mix.lock +++ b/mix.lock @@ -1,6 +1,6 @@ %{ "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm"}, - "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "c00c4e75b35367fa42c95ffd9b8c455bf9995829", [ref: "c00c4e75b35367fa42c95ffd9b8c455bf9995829"]}, + "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "e2385402bcd24fc659fee83b3eb8863b0528ad42", [ref: "e2385402bcd24fc659fee83b3eb8863b0528ad42"]}, "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"}, "bbcode": {:hex, :bbcode, "0.1.0", "400e618b640b635261611d7fb7f79d104917fc5b084aae371ab6b08477cb035b", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"}, "benchee": {:hex, :benchee, "1.0.1", "66b211f9bfd84bd97e6d1beaddf8fc2312aaabe192f776e8931cb0c16f53a521", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}], "hexpm"}, From 817c66bc3ecf26596cbbc6086a9dc9b95b88fc0a Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 12 Jun 2019 16:22:56 +0700 Subject: [PATCH 33/33] Remove search result order for non-RUM indexes --- lib/pleroma/activity/search.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex index 8cbb64cc4..0aa2aab23 100644 --- a/lib/pleroma/activity/search.ex +++ b/lib/pleroma/activity/search.ex @@ -39,8 +39,7 @@ defp query_with(q, :gin, search_query) do "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)", o.data, ^search_query - ), - order_by: [desc: :id] + ) ) end