From 76f732766ba36c3a94cf6b8b39fb745c1cf3f49a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 3 Feb 2021 12:32:44 -0600 Subject: [PATCH 01/19] Exclude reporter from receiving notifications from their own reports. Currently only works if the reporting actor is an admin, but if we include moderators with those who receive notification reports it will work for them. --- lib/pleroma/notification.ex | 4 ++-- lib/pleroma/web/activity_pub/activity_pub.ex | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 55b513212..1970fbf65 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -507,8 +507,8 @@ def get_potential_receiver_ap_ids(%{data: %{"type" => "Follow", "object" => obje [object_id] end - def get_potential_receiver_ap_ids(%{data: %{"type" => "Flag"}}) do - User.all_superusers() |> Enum.map(fn user -> user.ap_id end) + def get_potential_receiver_ap_ids(%{data: %{"type" => "Flag", "actor" => actor}}) do + (User.all_superusers() |> Enum.map(fn user -> user.ap_id end)) -- [actor] end def get_potential_receiver_ap_ids(activity) do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 1a84375fb..5b45e2ca1 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -377,6 +377,7 @@ defp do_flag( :ok <- maybe_federate(stripped_activity) do User.all_superusers() + |> Enum.filter(fn user -> user.ap_id != actor end) |> Enum.filter(fn user -> not is_nil(user.email) end) |> Enum.each(fn superuser -> superuser From 5bb5949048b6eeb236cca450c8399ac412fbd2a8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 3 Feb 2021 12:54:53 -0600 Subject: [PATCH 02/19] Validate admin making report doesn't get their own report notification, but other admins do --- test/pleroma/notification_test.exs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs index 0c6ebfb76..948587292 100644 --- a/test/pleroma/notification_test.exs +++ b/test/pleroma/notification_test.exs @@ -45,6 +45,20 @@ test "creates a notification for a report" do assert notification.type == "pleroma:report" end + test "suppresses notification to reporter if reporter is an admin" do + reporting_admin = insert(:user, is_admin: true) + reported_user = insert(:user) + other_admin = insert(:user, is_admin: true) + + {:ok, activity} = CommonAPI.report(reporting_admin, %{account_id: reported_user.id}) + + {:ok, [notification]} = Notification.create_notifications(activity) + + refute notification.user_id == reporting_admin.id + assert notification.user_id == other_admin.id + assert notification.type == "pleroma:report" + end + test "creates a notification for an emoji reaction" do user = insert(:user) other_user = insert(:user) From 000d3365c391fb3613c5365f73f5bd51d2555840 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 3 Feb 2021 14:52:49 -0600 Subject: [PATCH 03/19] Document admin actors not getting report notifications --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47009abc9..777847fa2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Deprecated `Pleroma.Uploaders.S3, :public_endpoint`. Now `Pleroma.Upload, :base_url` is the standard configuration key for all uploaders. - Improved Apache webserver support: updated sample configuration, MediaProxy cache invalidation verified with the included sample script - Improve OAuth 2.0 provider support. A missing `fqn` field was added to the response, but does not expose the user's email address. +- Admins no longer receive notifications for reports if they are the actor making the report.
API Changes From 1d8f1fe0772736dd71219d244783c9d671dd7223 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 5 Feb 2021 13:32:21 -0600 Subject: [PATCH 04/19] Add a default rule to not attempt to cache any files larger than 50MB This fixes connection failures when trying to retrieve large files. It is less common in typical Pleroma usage, but it's possible to encounter this on a cloud instance with lower memory. --- installation/pleroma.vcl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/installation/pleroma.vcl b/installation/pleroma.vcl index 13dad784c..4752510ea 100644 --- a/installation/pleroma.vcl +++ b/installation/pleroma.vcl @@ -59,6 +59,13 @@ sub vcl_backend_response { set beresp.http.CR = beresp.http.content-range; } + # Bypass cache for large files + # 50000000 ~ 50MB + if (std.integer(beresp.http.content-length, 0) > 50000000) { + set beresp.uncacheable = true; + return(deliver); + } + # Don't cache objects that require authentication if (beresp.http.Authorization && !beresp.http.Cache-Control ~ "public") { set beresp.uncacheable = true; From 8d4e0342e1b5ebbe486dc538e3c8fe81d53220e6 Mon Sep 17 00:00:00 2001 From: hyperion <8027-hyperion@users.noreply.git.pleroma.social> Date: Sat, 6 Feb 2021 09:42:17 +0000 Subject: [PATCH 05/19] Update priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs, priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs files --- docs/administration/CLI_tasks/database.md | 18 +++++++ docs/configuration/howto_search_cjk.md | 42 +++++++++++++++++ lib/mix/tasks/pleroma/database.ex | 47 +++++++++++++++++++ lib/pleroma/activity/search.ex | 8 ++-- ...1080964_add_default_text_search_config.exs | 11 +++++ ...510135645_add_fts_index_to_objects_two.exs | 2 +- 6 files changed, 123 insertions(+), 5 deletions(-) create mode 100644 docs/configuration/howto_search_cjk.md create mode 100644 priv/repo/migrations/20210121080964_add_default_text_search_config.exs diff --git a/docs/administration/CLI_tasks/database.md b/docs/administration/CLI_tasks/database.md index 6dca83167..c53c49921 100644 --- a/docs/administration/CLI_tasks/database.md +++ b/docs/administration/CLI_tasks/database.md @@ -141,3 +141,21 @@ but should only be run if necessary. **It is safe to cancel this.** ```sh mix pleroma.database ensure_expiration ``` + +## Change Text Search Configuration + +Change `default_text_search_config` for database and (if necessary) text_search_config used in index, then rebuild index (it may take time). + +=== "OTP" + + ```sh + ./bin/pleroma_ctl database set_text_search_config english + ``` + +=== "From Source" + + ```sh + mix pleroma.database set_text_search_config english + ``` + +See [PostgreSQL documentation](https://www.postgresql.org/docs/current/textsearch-configuration.html) and `docs/configuration/howto_search_cjk.md` for more detail. diff --git a/docs/configuration/howto_search_cjk.md b/docs/configuration/howto_search_cjk.md new file mode 100644 index 000000000..d3ce28077 --- /dev/null +++ b/docs/configuration/howto_search_cjk.md @@ -0,0 +1,42 @@ +# How to enable text search for Chinese, Japanese and Korean + +Pleroma's full text search feature is powered by PostgreSQL's native [text search](https://www.postgresql.org/docs/current/textsearch.html), it works well out of box for most of languages, but needs extra configurations for some asian languages like Chinese, Japanese and Korean (CJK). + + +## Setup and test the new search config + +In most cases, you would need an extension installed to support parsing CJK text. Here are a few extension you may choose from, or you are more than welcome to share additional ones you found working for you with the rest of Pleroma community. + + * [a generic n-gram parser](https://github.com/huangjimmy/pg_cjk_parser) supports Simplifed/Traditional Chinese, Japanese, and Korean + * [a Korean parser](https://github.com/i0seph/textsearch_ko) based on mecab + * [a Japanese parser](https://www.amris.co.jp/tsja/index.html) based on mecab + * [zhparser](https://github.com/amutu/zhparser/) is a PostgreSQL extension base on the Simple Chinese Word Segmentation(SCWS) + * [another Chinese parser](https://github.com/jaiminpan/pg_jieba) based on Jieba Chinese Word Segmentation + +Once you have the new search config , make sure you test it with the `pleroma` user in PostgreSQL (change `YOUR.CONFIG` to your real configuration name) +``` +SELECT ts_debug('YOUR.CONFIG', '安装和配置Nginx, ElixirとErlangをインストールします'); +``` +Check output of the query, and see if it matches your expectation. + + +## Update text search config and index in database + +=== "OTP" + + ```sh + ./bin/pleroma_ctl database set_text_search_config YOUR.CONFIG + ``` + +=== "From Source" + + ```sh + mix pleroma.database set_text_search_config YOUR.CONFIG + ``` + +Note: index update may take a while. + +## Restart database connection +Since some changes above will only apply with a new database connection, you will have to restart either Pleroma or PostgreSQL process, or use `pg_terminate_backend` SQL command without restarting either. + +Now the search results of statuses should be much more friendly for your language of choice, the results for searching users and tags were not changed, as the default parsing/matching should work for most cases. diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 6261910f0..2403ed581 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -167,4 +167,51 @@ def run(["ensure_expiration"]) do end) |> Stream.run() end + + def run(["set_text_search_config", tsconfig]) do + start_pleroma() + %{rows: [[tsc]]} = Ecto.Adapters.SQL.query!(Pleroma.Repo, "SHOW default_text_search_config;") + shell_info("Current default_text_search_config: #{tsc}") + + %{rows: [[db]]} = Ecto.Adapters.SQL.query!(Pleroma.Repo, "SELECT current_database();") + shell_info("Update default_text_search_config: #{tsconfig}") + + %{messages: msg} = + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + "ALTER DATABASE #{db} SET default_text_search_config = '#{tsconfig}';" + ) + + # non-exist config will not raise excpetion but only give >0 messages + if length(msg) > 0 do + shell_info("Error: #{inspect(msg, pretty: true)}") + else + rum_enabled = Pleroma.Config.get([:database, :rum_enabled]) + shell_info("Recreate index, RUM: #{rum_enabled}") + + # Note SQL below needs to be kept up-to-date with latest GIN or RUM index definition in future + if rum_enabled do + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + "CREATE OR REPLACE FUNCTION objects_fts_update() RETURNS trigger AS $$ BEGIN + new.fts_content := to_tsvector(new.data->>'content'); + RETURN new; + END + $$ LANGUAGE plpgsql" + ) + + shell_info("Refresh RUM index") + Ecto.Adapters.SQL.query!(Pleroma.Repo, "UPDATE objects SET updated_at = NOW();") + else + Ecto.Adapters.SQL.query!(Pleroma.Repo, "DROP INDEX IF EXISTS objects_fts;") + + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + "CREATE INDEX objects_fts ON objects USING gin(to_tsvector('#{tsconfig}', data->>'content')); " + ) + end + + shell_info('Done.') + end + end end diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex index 52e7c048d..ed898ba4f 100644 --- a/lib/pleroma/activity/search.ex +++ b/lib/pleroma/activity/search.ex @@ -64,7 +64,7 @@ defp query_with(q, :gin, search_query, :plain) do from([a, o] in q, where: fragment( - "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)", + "to_tsvector(?->>'content') @@ plainto_tsquery(?)", o.data, ^search_query ) @@ -75,7 +75,7 @@ defp query_with(q, :gin, search_query, :websearch) do from([a, o] in q, where: fragment( - "to_tsvector('english', ?->>'content') @@ websearch_to_tsquery('english', ?)", + "to_tsvector(?->>'content') @@ websearch_to_tsquery(?)", o.data, ^search_query ) @@ -86,7 +86,7 @@ defp query_with(q, :rum, search_query, :plain) do from([a, o] in q, where: fragment( - "? @@ plainto_tsquery('english', ?)", + "? @@ plainto_tsquery(?)", o.fts_content, ^search_query ), @@ -98,7 +98,7 @@ defp query_with(q, :rum, search_query, :websearch) do from([a, o] in q, where: fragment( - "? @@ websearch_to_tsquery('english', ?)", + "? @@ websearch_to_tsquery(?)", o.fts_content, ^search_query ), diff --git a/priv/repo/migrations/20210121080964_add_default_text_search_config.exs b/priv/repo/migrations/20210121080964_add_default_text_search_config.exs new file mode 100644 index 000000000..09b6cccc9 --- /dev/null +++ b/priv/repo/migrations/20210121080964_add_default_text_search_config.exs @@ -0,0 +1,11 @@ +defmodule Pleroma.Repo.Migrations.AddDefaultTextSearchConfig do + use Ecto.Migration + + def change do + execute("DO $$ + BEGIN + execute 'ALTER DATABASE '||current_database()||' SET default_text_search_config = ''english'' '; + END + $$;") + end +end diff --git a/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs index 82e02281d..88476fb57 100644 --- a/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs +++ b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs @@ -17,7 +17,7 @@ def up do execute("CREATE FUNCTION objects_fts_update() RETURNS trigger AS $$ begin - new.fts_content := to_tsvector('english', new.data->>'content'); + new.fts_content := to_tsvector(new.data->>'content'); return new; end $$ LANGUAGE plpgsql") From 9f98885388c9fad95aebddec42ad4a08f82d117a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 6 Feb 2021 11:28:55 -0600 Subject: [PATCH 06/19] No reason to suggest users try the useless "Local" mail adapter --- config/description.exs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/config/description.exs b/config/description.exs index f84b52a4f..600fa87d7 100644 --- a/config/description.exs +++ b/config/description.exs @@ -218,8 +218,7 @@ key: :adapter, type: :module, description: - "One of the mail adapters listed in [Swoosh readme](https://github.com/swoosh/swoosh#adapters)," <> - " or Swoosh.Adapters.Local for in-memory mailbox", + "One of the mail adapters listed in [Swoosh readme](https://github.com/swoosh/swoosh#adapters)", suggestions: [ Swoosh.Adapters.SMTP, Swoosh.Adapters.Sendgrid, @@ -232,8 +231,7 @@ Swoosh.Adapters.AmazonSES, Swoosh.Adapters.Dyn, Swoosh.Adapters.SocketLabs, - Swoosh.Adapters.Gmail, - Swoosh.Adapters.Local + Swoosh.Adapters.Gmail ] }, %{ From 85710b026feea51057b05d02390d4d36e5f32bb1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 6 Feb 2021 11:55:01 -0600 Subject: [PATCH 07/19] Improve SMTP adapter setting descriptions --- config/description.exs | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/config/description.exs b/config/description.exs index 600fa87d7..85f90ca3e 100644 --- a/config/description.exs +++ b/config/description.exs @@ -243,21 +243,27 @@ group: {:subgroup, Swoosh.Adapters.SMTP}, key: :relay, type: :string, - description: "`Swoosh.Adapters.SMTP` adapter specific setting", - suggestions: ["smtp.gmail.com"] + description: "Hostname or IP address", + suggestions: ["smtp.example.com"] + }, + %{ + group: {:subgroup, Swoosh.Adapters.SMTP}, + key: :port, + type: :integer, + description: "SMTP port" }, %{ group: {:subgroup, Swoosh.Adapters.SMTP}, key: :username, type: :string, - description: "`Swoosh.Adapters.SMTP` adapter specific setting", - suggestions: ["pleroma"] + description: "SMTP auth username", + suggestions: ["user@example.com"] }, %{ group: {:subgroup, Swoosh.Adapters.SMTP}, key: :password, type: :string, - description: "`Swoosh.Adapters.SMTP` adapter specific setting", + description: "SMTP auth password", suggestions: ["password"] }, %{ @@ -265,29 +271,22 @@ key: :ssl, label: "SSL", type: :boolean, - description: "`Swoosh.Adapters.SMTP` adapter specific setting" + description: "Use implicit SSL/TLS: e.g., port 465", }, %{ group: {:subgroup, Swoosh.Adapters.SMTP}, key: :tls, - label: "TLS", - type: :atom, - description: "`Swoosh.Adapters.SMTP` adapter specific setting", - suggestions: [:always, :never, :if_available] + label: "STARTTLS", + type: {:dropdown, :atom}, + description: "Explicit TLS (STARTTLS) mode", + suggestions: [:if_available, :always, :never] }, %{ group: {:subgroup, Swoosh.Adapters.SMTP}, key: :auth, - type: :atom, - description: "`Swoosh.Adapters.SMTP` adapter specific setting", - suggestions: [:always, :never, :if_available] - }, - %{ - group: {:subgroup, Swoosh.Adapters.SMTP}, - key: :port, - type: :integer, - description: "`Swoosh.Adapters.SMTP` adapter specific setting", - suggestions: [1025] + type: {:dropdown, :atom}, + description: "SMTP authentication mode", + suggestions: [:if_available, :always, :never] }, %{ group: {:subgroup, Swoosh.Adapters.SMTP}, From 6ffe15cc9feadecf5e6756cb3db3240fa9eb63c2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 6 Feb 2021 11:55:25 -0600 Subject: [PATCH 08/19] Remove No MX lookups setting This setting defaults to false so the relay host will be used in an MX query so multiple SMTP servers can be used. gen_smtp code states that all records returned from the MX query are attempted in order and only a permanent SMTP error will stop the client from attempting other servers in the list. Connection failures, TLS issues, etc will cause it to try the next host. If there is no MX record associated with the relay host, it automatically tries connecting to it directly. There is really no reason to expose this to end users. The default value is fine for everyone. --- config/description.exs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/config/description.exs b/config/description.exs index 85f90ca3e..8922a3626 100644 --- a/config/description.exs +++ b/config/description.exs @@ -295,13 +295,6 @@ description: "`Swoosh.Adapters.SMTP` adapter specific setting", suggestions: [5] }, - %{ - group: {:subgroup, Swoosh.Adapters.SMTP}, - key: :no_mx_lookups, - label: "No MX lookups", - type: :boolean, - description: "`Swoosh.Adapters.SMTP` adapter specific setting" - }, %{ group: {:subgroup, Swoosh.Adapters.Sendgrid}, key: :api_key, From cfc474c5f7e29238132948d1858e4ed0d88bb062 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 6 Feb 2021 12:01:11 -0600 Subject: [PATCH 09/19] There is no reason to expose these Local adapter settings either. --- config/description.exs | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/config/description.exs b/config/description.exs index 8922a3626..8a4410723 100644 --- a/config/description.exs +++ b/config/description.exs @@ -434,26 +434,6 @@ } ] }, - %{ - group: :swoosh, - type: :group, - description: "`Swoosh.Adapters.Local` adapter specific settings", - children: [ - %{ - group: {:subgroup, Swoosh.Adapters.Local}, - key: :serve_mailbox, - type: :boolean, - description: "Run the preview server together as part of your app" - }, - %{ - group: {:subgroup, Swoosh.Adapters.Local}, - key: :preview_port, - type: :integer, - description: "The preview server port", - suggestions: [4001] - } - ] - }, %{ group: :pleroma, key: :uri_schemes, From 9e3e8e2e30d48c2989bc645f2b7929eb339de09b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 6 Feb 2021 12:04:28 -0600 Subject: [PATCH 10/19] Update Swoosh docs URL, lint --- config/description.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/description.exs b/config/description.exs index 8a4410723..6f6462900 100644 --- a/config/description.exs +++ b/config/description.exs @@ -218,7 +218,7 @@ key: :adapter, type: :module, description: - "One of the mail adapters listed in [Swoosh readme](https://github.com/swoosh/swoosh#adapters)", + "One of the mail adapters listed in [Swoosh documentation](https://hexdocs.pm/swoosh/Swoosh.html#module-adapters)", suggestions: [ Swoosh.Adapters.SMTP, Swoosh.Adapters.Sendgrid, @@ -271,7 +271,7 @@ key: :ssl, label: "SSL", type: :boolean, - description: "Use implicit SSL/TLS: e.g., port 465", + description: "Use implicit SSL/TLS: e.g., port 465" }, %{ group: {:subgroup, Swoosh.Adapters.SMTP}, From 227dd84f1175ed61c768c0ada39b748371c0c441 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 6 Feb 2021 12:06:47 -0600 Subject: [PATCH 11/19] Update SMTP error description and default value --- config/description.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/description.exs b/config/description.exs index 6f6462900..6e1a8e7ea 100644 --- a/config/description.exs +++ b/config/description.exs @@ -292,8 +292,8 @@ group: {:subgroup, Swoosh.Adapters.SMTP}, key: :retries, type: :integer, - description: "`Swoosh.Adapters.SMTP` adapter specific setting", - suggestions: [5] + description: "SMTP temporary (4xx) error retries", + suggestions: [1] }, %{ group: {:subgroup, Swoosh.Adapters.Sendgrid}, From bd828e5c9c2c1a373b13cf80b185d11b1fcd1bc3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 6 Feb 2021 12:28:49 -0600 Subject: [PATCH 12/19] More description improvements --- config/description.exs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/config/description.exs b/config/description.exs index 6e1a8e7ea..54b5fd5d6 100644 --- a/config/description.exs +++ b/config/description.exs @@ -237,7 +237,7 @@ %{ key: :enabled, type: :boolean, - description: "Allow/disallow send emails" + description: "Pleroma Email sending capability" }, %{ group: {:subgroup, Swoosh.Adapters.SMTP}, @@ -256,36 +256,37 @@ group: {:subgroup, Swoosh.Adapters.SMTP}, key: :username, type: :string, - description: "SMTP auth username", + description: "SMTP AUTH username", suggestions: ["user@example.com"] }, %{ group: {:subgroup, Swoosh.Adapters.SMTP}, key: :password, type: :string, - description: "SMTP auth password", + description: "SMTP AUTH password", suggestions: ["password"] }, %{ group: {:subgroup, Swoosh.Adapters.SMTP}, key: :ssl, - label: "SSL", + label: "Use SSL", type: :boolean, - description: "Use implicit SSL/TLS: e.g., port 465" + description: "Use implicit SSL/TLS. e.g. port 465" }, %{ group: {:subgroup, Swoosh.Adapters.SMTP}, key: :tls, - label: "STARTTLS", + label: "STARTTLS Mode", type: {:dropdown, :atom}, - description: "Explicit TLS (STARTTLS) mode", + description: "Explicit TLS (STARTTLS) enforcement mode", suggestions: [:if_available, :always, :never] }, %{ group: {:subgroup, Swoosh.Adapters.SMTP}, key: :auth, + label: "AUTH Mode", type: {:dropdown, :atom}, - description: "SMTP authentication mode", + description: "SMTP AUTH enforcement mode", suggestions: [:if_available, :always, :never] }, %{ From 0fcf16dcb858cdd464fbd614aaba54fb81264199 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 6 Feb 2021 12:34:32 -0600 Subject: [PATCH 13/19] Move Enabled to top as it's the master control of all email. Description not really needed. --- config/description.exs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/config/description.exs b/config/description.exs index 54b5fd5d6..8d3312caa 100644 --- a/config/description.exs +++ b/config/description.exs @@ -214,6 +214,10 @@ type: :group, description: "Mailer-related settings", children: [ + %{ + key: :enabled, + type: :boolean, + }, %{ key: :adapter, type: :module, @@ -234,11 +238,6 @@ Swoosh.Adapters.Gmail ] }, - %{ - key: :enabled, - type: :boolean, - description: "Pleroma Email sending capability" - }, %{ group: {:subgroup, Swoosh.Adapters.SMTP}, key: :relay, From f736501e977f976324cda244b51f0a76ffb4691f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 6 Feb 2021 13:18:48 -0600 Subject: [PATCH 14/19] Alpha-sort adapters Add various labels, descriptions and suggestions for all adapter settings and try to use the same terminology by the service provider. --- config/description.exs | 111 +++++++++++++++++++---------------------- 1 file changed, 52 insertions(+), 59 deletions(-) diff --git a/config/description.exs b/config/description.exs index 8d3312caa..650b504c1 100644 --- a/config/description.exs +++ b/config/description.exs @@ -216,6 +216,7 @@ children: [ %{ key: :enabled, + label: "Mailer Enabled", type: :boolean, }, %{ @@ -224,18 +225,18 @@ description: "One of the mail adapters listed in [Swoosh documentation](https://hexdocs.pm/swoosh/Swoosh.html#module-adapters)", suggestions: [ + Swoosh.Adapters.AmazonSES, + Swoosh.Adapters.Dyn, + Swoosh.Adapters.Gmail, + Swoosh.Adapters.Mailgun, + Swoosh.Adapters.Mailjet, + Swoosh.Adapters.Mandrill, + Swoosh.Adapters.Postmark, Swoosh.Adapters.SMTP, Swoosh.Adapters.Sendgrid, Swoosh.Adapters.Sendmail, - Swoosh.Adapters.Mandrill, - Swoosh.Adapters.Mailgun, - Swoosh.Adapters.Mailjet, - Swoosh.Adapters.Postmark, - Swoosh.Adapters.SparkPost, - Swoosh.Adapters.AmazonSES, - Swoosh.Adapters.Dyn, Swoosh.Adapters.SocketLabs, - Swoosh.Adapters.Gmail + Swoosh.Adapters.SparkPost ] }, %{ @@ -249,7 +250,8 @@ group: {:subgroup, Swoosh.Adapters.SMTP}, key: :port, type: :integer, - description: "SMTP port" + description: "SMTP port", + suggestions: ["1025"] }, %{ group: {:subgroup, Swoosh.Adapters.SMTP}, @@ -270,7 +272,7 @@ key: :ssl, label: "Use SSL", type: :boolean, - description: "Use implicit SSL/TLS. e.g. port 465" + description: "Use Implicit SSL/TLS. e.g. port 465" }, %{ group: {:subgroup, Swoosh.Adapters.SMTP}, @@ -298,139 +300,130 @@ %{ group: {:subgroup, Swoosh.Adapters.Sendgrid}, key: :api_key, - label: "API key", + label: "SendGrid API Key", type: :string, - description: "`Swoosh.Adapters.Sendgrid` adapter specific setting", - suggestions: ["my-api-key"] + suggestions: ["YOUR_API_KEY"] }, %{ group: {:subgroup, Swoosh.Adapters.Sendmail}, key: :cmd_path, type: :string, - description: "`Swoosh.Adapters.Sendmail` adapter specific setting", suggestions: ["/usr/bin/sendmail"] }, %{ group: {:subgroup, Swoosh.Adapters.Sendmail}, key: :cmd_args, type: :string, - description: "`Swoosh.Adapters.Sendmail` adapter specific setting", suggestions: ["-N delay,failure,success"] }, %{ group: {:subgroup, Swoosh.Adapters.Sendmail}, key: :qmail, + label: "Qmail compat mode", type: :boolean, - description: "`Swoosh.Adapters.Sendmail` adapter specific setting" }, %{ group: {:subgroup, Swoosh.Adapters.Mandrill}, key: :api_key, - label: "API key", + label: "Mandrill API Key", type: :string, - description: "`Swoosh.Adapters.Mandrill` adapter specific setting", - suggestions: ["my-api-key"] + suggestions: ["YOUR_API_KEY"] }, %{ group: {:subgroup, Swoosh.Adapters.Mailgun}, key: :api_key, - label: "API key", + label: "Mailgun API Key", type: :string, - description: "`Swoosh.Adapters.Mailgun` adapter specific setting", - suggestions: ["my-api-key"] + suggestions: ["YOUR_API_KEY"] }, %{ group: {:subgroup, Swoosh.Adapters.Mailgun}, key: :domain, type: :string, - description: "`Swoosh.Adapters.Mailgun` adapter specific setting", - suggestions: ["pleroma.com"] + suggestions: ["YOUR_DOMAIN_NAME"] }, %{ group: {:subgroup, Swoosh.Adapters.Mailjet}, key: :api_key, - label: "API key", + label: "MailJet Public API Key", type: :string, - description: "`Swoosh.Adapters.Mailjet` adapter specific setting", - suggestions: ["my-api-key"] + suggestions: ["MJ_APIKEY_PUBLIC"] }, %{ group: {:subgroup, Swoosh.Adapters.Mailjet}, key: :secret, + label: "MailJet Private API Key", type: :string, - description: "`Swoosh.Adapters.Mailjet` adapter specific setting", - suggestions: ["my-secret-key"] + suggestions: ["MJ_APIKEY_PRIVATE"] }, %{ group: {:subgroup, Swoosh.Adapters.Postmark}, key: :api_key, - label: "API key", + label: "Postmark API Key", type: :string, - description: "`Swoosh.Adapters.Postmark` adapter specific setting", - suggestions: ["my-api-key"] + suggestions: ["X-Postmark-Server-Token"] }, %{ group: {:subgroup, Swoosh.Adapters.SparkPost}, key: :api_key, - label: "API key", + label: "SparkPost API key", type: :string, - description: "`Swoosh.Adapters.SparkPost` adapter specific setting", - suggestions: ["my-api-key"] + suggestions: ["YOUR_API_KEY"] }, %{ group: {:subgroup, Swoosh.Adapters.SparkPost}, key: :endpoint, type: :string, - description: "`Swoosh.Adapters.SparkPost` adapter specific setting", suggestions: ["https://api.sparkpost.com/api/v1"] }, - %{ - group: {:subgroup, Swoosh.Adapters.AmazonSES}, - key: :region, - type: :string, - description: "`Swoosh.Adapters.AmazonSES` adapter specific setting", - suggestions: ["us-east-1", "us-east-2"] - }, %{ group: {:subgroup, Swoosh.Adapters.AmazonSES}, key: :access_key, + label: "AWS Access Key", type: :string, - description: "`Swoosh.Adapters.AmazonSES` adapter specific setting", - suggestions: ["aws-access-key"] + suggestions: ["AWS_ACCESS_KEY"] }, %{ group: {:subgroup, Swoosh.Adapters.AmazonSES}, key: :secret, + label: "AWS Secret Key", type: :string, - description: "`Swoosh.Adapters.AmazonSES` adapter specific setting", - suggestions: ["aws-secret-key"] + suggestions: ["AWS_SECRET_KEY"] + }, + %{ + group: {:subgroup, Swoosh.Adapters.AmazonSES}, + key: :region, + label: "AWS Region", + type: :string, + suggestions: ["us-east-1", "us-east-2"] }, %{ group: {:subgroup, Swoosh.Adapters.Dyn}, key: :api_key, - label: "API key", + label: "Dyn API Key", type: :string, - description: "`Swoosh.Adapters.Dyn` adapter specific setting", - suggestions: ["my-api-key"] - }, - %{ - group: {:subgroup, Swoosh.Adapters.SocketLabs}, - key: :server_id, - type: :string, - description: "`Swoosh.Adapters.SocketLabs` adapter specific setting" + suggestions: ["apikey"] }, %{ group: {:subgroup, Swoosh.Adapters.SocketLabs}, key: :api_key, - label: "API key", + label: "SocketLabs API Key", type: :string, - description: "`Swoosh.Adapters.SocketLabs` adapter specific setting" + suggestions: ["INJECTION_API_KEY"] + }, + %{ + group: {:subgroup, Swoosh.Adapters.SocketLabs}, + key: :server_id, + label: "Server ID", + type: :string, + suggestions: ["SERVER_ID"] }, %{ group: {:subgroup, Swoosh.Adapters.Gmail}, key: :access_token, + label: "GMail API Access Token", type: :string, - description: "`Swoosh.Adapters.Gmail` adapter specific setting" + suggestions: ["GMAIL_API_ACCESS_TOKEN"] } ] }, From 4dbb08a19f57e720e299608ebeb4387d37c55e99 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 6 Feb 2021 13:20:58 -0600 Subject: [PATCH 15/19] Improved Mailer descriptions --- CHANGELOG.md | 3 ++- config/description.exs | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1dbdb3f4e..15c75353f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Deprecated `Pleroma.Uploaders.S3, :public_endpoint`. Now `Pleroma.Upload, :base_url` is the standard configuration key for all uploaders. - Improved Apache webserver support: updated sample configuration, MediaProxy cache invalidation verified with the included sample script - Improve OAuth 2.0 provider support. A missing `fqn` field was added to the response, but does not expose the user's email address. -- Provide redirect of external posts from `/notice/:id` to their original URL +- Provide redirect of external posts from `/notice/:id` to their original URL. +- Improved Mailer configuration setting descriptions for AdminFE.
API Changes diff --git a/config/description.exs b/config/description.exs index 650b504c1..8eefa2ba1 100644 --- a/config/description.exs +++ b/config/description.exs @@ -217,7 +217,7 @@ %{ key: :enabled, label: "Mailer Enabled", - type: :boolean, + type: :boolean }, %{ key: :adapter, @@ -320,7 +320,7 @@ group: {:subgroup, Swoosh.Adapters.Sendmail}, key: :qmail, label: "Qmail compat mode", - type: :boolean, + type: :boolean }, %{ group: {:subgroup, Swoosh.Adapters.Mandrill}, From 2bffa8e0202e4db61eb24dae0f7063ac8305cae4 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 8 Feb 2021 11:25:32 -0600 Subject: [PATCH 16/19] Make the suggestion match the default value --- config/description.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/description.exs b/config/description.exs index f84b52a4f..df5108582 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2888,7 +2888,7 @@ type: :integer, description: "Activity pub routes (except question activities). Default: `nil` (no expiration).", - suggestions: [30_000, nil] + suggestions: [nil] }, %{ key: :activity_pub_question, From ce7c275fb35fee87d85ef5165900d2991bdfc660 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 8 Feb 2021 11:45:50 -0600 Subject: [PATCH 17/19] Improve various descriptions and labels --- config/description.exs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/config/description.exs b/config/description.exs index df5108582..0c0963678 100644 --- a/config/description.exs +++ b/config/description.exs @@ -99,7 +99,8 @@ key: :base_url, label: "Base URL", type: :string, - description: "Base URL for the uploads, needed if you use CDN", + description: + "Base URL for the uploads. Required if you use a CDN or host attachments under a different domain.", suggestions: [ "https://cdn-host.com" ] @@ -1545,7 +1546,8 @@ %{ key: :max_body_length, type: :integer, - description: "Maximum file size allowed through the Pleroma MediaProxy cache." + description: + "Maximum file size (in bytes) allowed through the Pleroma MediaProxy cache." }, %{ key: :max_read_duration, @@ -1595,7 +1597,7 @@ key: :min_content_length, type: :integer, description: - "Min content length to perform preview, in bytes. If greater than 0, media smaller in size will be served as is, without thumbnailing." + "Min content length (in bytes) to perform preview. Media smaller in size will be served without thumbnailing." } ] }, @@ -1643,6 +1645,7 @@ }, %{ key: :url_format, + label: "URL Format", type: :string, description: "Optional URL format preprocessing. Only required for Apache's htcacheclean.", @@ -3326,9 +3329,9 @@ }, %{ key: :ip_whitelist, + label: "IP Whitelist", type: [{:list, :string}, {:list, :charlist}, {:list, :tuple}], - description: - "[Pleroma extension] If non-empty, restricts access to app metrics endpoint to specified IP addresses." + description: "Restrict access of app metrics endpoint to the specified IP addresses." }, %{ key: :auth, From ed8ef80b5eb4936087389dd9a6545e9a3b666311 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 8 Feb 2021 22:41:35 +0300 Subject: [PATCH 18/19] RSS: Make sure post URL is the first `` element Otherwise some RSS readers (tested in Miniflux) might pick the context URL as the external link. Related to #2425. --- lib/pleroma/web/templates/feed/feed/_activity.rss.eex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex index 42960de7d..947bbb099 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex @@ -9,7 +9,6 @@ <%= activity_context(@activity) %> - <%= activity_context(@activity) %> <%= if @data["summary"] do %> <%= escape(@data["summary"]) %> @@ -21,6 +20,8 @@ <%= @data["external_url"] %> <% end %> + <%= activity_context(@activity) %> + <%= for tag <- @data["tag"] || [] do %> <% end %> From 55a13fc3607c9d753e6fca596010c0a96ba3fba8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 8 Feb 2021 15:32:47 -0600 Subject: [PATCH 19/19] MRF NoEmptyPolicy: Deny posts from local users if there is no content or only mentions. Helps prevent accidental button mashes from submitting incomplete posts --- CHANGELOG.md | 1 + .../web/activity_pub/mrf/no_empty_policy.ex | 61 +++++++ .../activity_pub/mrf/no_empty_policy_test.exs | 154 ++++++++++++++++++ 3 files changed, 216 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex create mode 100644 test/pleroma/web/activity_pub/mrf/no_empty_policy_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 1dbdb3f4e..d4acbc9a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Ability to set ActivityPub aliases for follower migration. - Configurable background job limits for RichMedia (link previews) and MediaProxyWarmingPolicy - Ability to define custom HTTP headers per each frontend +- MRF (`NoEmptyPolicy`): New MRF Policy which will deny empty statuses or statuses of only mentions from being created by local users
API Changes diff --git a/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex new file mode 100644 index 000000000..32bb1b645 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex @@ -0,0 +1,61 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do + @moduledoc "Filter local activities which have no content" + @behaviour Pleroma.Web.ActivityPub.MRF + + alias Pleroma.Web + + @impl true + def filter(%{"actor" => actor} = object) do + with true <- is_local?(actor), + true <- is_note?(object), + false <- has_attachment?(object), + true <- only_mentions?(object) do + {:reject, "[NoEmptyPolicy]"} + else + _ -> + {:ok, object} + end + end + + def filter(object), do: {:ok, object} + + defp is_local?(actor) do + if actor |> String.starts_with?("#{Web.base_url()}") do + true + else + false + end + end + + defp has_attachment?(%{ + "type" => "Create", + "object" => %{"type" => "Note", "attachment" => attachments} + }) + when length(attachments) > 0, + do: true + + defp has_attachment?(_), do: false + + defp only_mentions?(%{"type" => "Create", "object" => %{"type" => "Note", "source" => source}}) do + non_mentions = + source |> String.split() |> Enum.filter(&(not String.starts_with?(&1, "@"))) |> length + + if non_mentions > 0 do + false + else + true + end + end + + defp only_mentions?(_), do: false + + defp is_note?(%{"type" => "Create", "object" => %{"type" => "Note"}}), do: true + defp is_note?(_), do: false + + @impl true + def describe, do: {:ok, %{}} +end diff --git a/test/pleroma/web/activity_pub/mrf/no_empty_policy_test.exs b/test/pleroma/web/activity_pub/mrf/no_empty_policy_test.exs new file mode 100644 index 000000000..fbcf68414 --- /dev/null +++ b/test/pleroma/web/activity_pub/mrf/no_empty_policy_test.exs @@ -0,0 +1,154 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicyTest do + use Pleroma.DataCase + alias Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy + + setup_all do: clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy]) + + test "Notes with content are exempt" do + message = %{ + "actor" => "http://localhost:4001/users/testuser", + "cc" => ["http://localhost:4001/users/testuser/followers"], + "object" => %{ + "actor" => "http://localhost:4001/users/testuser", + "attachment" => [], + "cc" => ["http://localhost:4001/users/testuser/followers"], + "source" => "this is a test post", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "type" => "Note" + }, + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "type" => "Create" + } + + assert NoEmptyPolicy.filter(message) == {:ok, message} + end + + test "Polls are exempt" do + message = %{ + "actor" => "http://localhost:4001/users/testuser", + "cc" => ["http://localhost:4001/users/testuser/followers"], + "object" => %{ + "actor" => "http://localhost:4001/users/testuser", + "attachment" => [], + "cc" => ["http://localhost:4001/users/testuser/followers"], + "oneOf" => [ + %{ + "name" => "chocolate", + "replies" => %{"totalItems" => 0, "type" => "Collection"}, + "type" => "Note" + }, + %{ + "name" => "vanilla", + "replies" => %{"totalItems" => 0, "type" => "Collection"}, + "type" => "Note" + } + ], + "source" => "@user2", + "to" => [ + "https://www.w3.org/ns/activitystreams#Public", + "http://localhost:4001/users/user2" + ], + "type" => "Question" + }, + "to" => [ + "https://www.w3.org/ns/activitystreams#Public", + "http://localhost:4001/users/user2" + ], + "type" => "Create" + } + + assert NoEmptyPolicy.filter(message) == {:ok, message} + end + + test "Notes with attachments are exempt" do + message = %{ + "actor" => "http://localhost:4001/users/testuser", + "cc" => ["http://localhost:4001/users/testuser/followers"], + "object" => %{ + "actor" => "http://localhost:4001/users/testuser", + "attachment" => [ + %{ + "actor" => "http://localhost:4001/users/testuser", + "mediaType" => "image/png", + "name" => "", + "type" => "Document", + "url" => [ + %{ + "href" => + "http://localhost:4001/media/68ba231cf12e1382ce458f1979969f8ed5cc07ba198a02e653464abaf39bdb90.png", + "mediaType" => "image/png", + "type" => "Link" + } + ] + } + ], + "cc" => ["http://localhost:4001/users/testuser/followers"], + "source" => "@user2", + "to" => [ + "https://www.w3.org/ns/activitystreams#Public", + "http://localhost:4001/users/user2" + ], + "type" => "Note" + }, + "to" => [ + "https://www.w3.org/ns/activitystreams#Public", + "http://localhost:4001/users/user2" + ], + "type" => "Create" + } + + assert NoEmptyPolicy.filter(message) == {:ok, message} + end + + test "Notes with only mentions are denied" do + message = %{ + "actor" => "http://localhost:4001/users/testuser", + "cc" => ["http://localhost:4001/users/testuser/followers"], + "object" => %{ + "actor" => "http://localhost:4001/users/testuser", + "attachment" => [], + "cc" => ["http://localhost:4001/users/testuser/followers"], + "source" => "@user2", + "to" => [ + "https://www.w3.org/ns/activitystreams#Public", + "http://localhost:4001/users/user2" + ], + "type" => "Note" + }, + "to" => [ + "https://www.w3.org/ns/activitystreams#Public", + "http://localhost:4001/users/user2" + ], + "type" => "Create" + } + + assert NoEmptyPolicy.filter(message) == {:reject, "[NoEmptyPolicy]"} + end + + test "Notes with no content are denied" do + message = %{ + "actor" => "http://localhost:4001/users/testuser", + "cc" => ["http://localhost:4001/users/testuser/followers"], + "object" => %{ + "actor" => "http://localhost:4001/users/testuser", + "attachment" => [], + "cc" => ["http://localhost:4001/users/testuser/followers"], + "source" => "", + "to" => [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "type" => "Note" + }, + "to" => [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "type" => "Create" + } + + assert NoEmptyPolicy.filter(message) == {:reject, "[NoEmptyPolicy]"} + end +end