From 736d8ad6be1ccb1514a189ccf2384e9699ea107e Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 19 Jun 2019 15:57:44 +0000 Subject: [PATCH 1/4] implement anti link spam MRF --- CHANGELOG.md | 1 + docs/config.md | 1 + .../activity_pub/mrf/anti_link_spam_policy.ex | 46 +++++++ .../mrf/anti_link_spam_policy_test.exs | 120 ++++++++++++++++++ 4 files changed, 168 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex create mode 100644 test/web/activity_pub/mrf/anti_link_spam_policy_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b7e5c9a1..ced0573f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - MRF: Support for running subchains. - Configuration: `skip_thread_containment` option - Configuration: `rate_limit` option. See `Pleroma.Plugs.RateLimiter` documentation for details. +- MRF: Support for filtering out likely spam messages using naive heuristics. ### Changed - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer diff --git a/docs/config.md b/docs/config.md index ed8e465c6..4e9697afc 100644 --- a/docs/config.md +++ b/docs/config.md @@ -90,6 +90,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `Pleroma.Web.ActivityPub.MRF.SubchainPolicy`: Selectively runs other MRF policies when messages match (see ``:mrf_subchain`` section) * `Pleroma.Web.ActivityPub.MRF.RejectNonPublic`: Drops posts with non-public visibility settings (See ``:mrf_rejectnonpublic`` section) * `Pleroma.Web.ActivityPub.MRF.EnsureRePrepended`: Rewrites posts to ensure that replies to posts with subjects do not have an identical subject and instead begin with re:. + * `Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy`: Rejects posts from likely spambots using naive heuristics. * `public`: Makes the client API in authentificated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. * `quarantined_instances`: List of ActivityPub instances where private(DMs, followers-only) activities will not be send. * `managed_config`: Whenether the config for pleroma-fe is configured in this config or in ``static/config.json`` diff --git a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex new file mode 100644 index 000000000..33ea61f5c --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex @@ -0,0 +1,46 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do + alias Pleroma.User + + require Logger + + # has the user successfully posted before? + defp user_has_posted_before?(%User{} = u) do + u.info.note_count > 0 || u.info.follower_count > 0 + end + + # does the post contain links? + defp contains_links?(%{"content" => content} = _object) do + content + |> Floki.filter_out("a.mention,a.hashtag,a[rel~=\"tag\"],a.zrl") + |> Floki.attribute("a", "href") + |> length() > 0 + end + + def filter(%{"type" => "Create", "actor" => actor, "object" => object} = message) do + with {:ok, %User{} = u} <- User.get_or_fetch_by_ap_id(actor), + {:contains_links, true} <- {:contains_links, contains_links?(object)}, + {:posted_before, true} <- {:posted_before, user_has_posted_before?(u)} do + {:ok, message} + else + {:contains_links, false} -> + {:ok, message} + + {:posted_before, false} -> + {:reject, nil} + + {:error, _} -> + {:reject, nil} + + e -> + Logger.warn("[MRF anti-link-spam] WTF: unhandled error #{inspect(e)}") + {:reject, nil} + end + end + + # in all other cases, pass through + def filter(message), do: {:ok, message} +end diff --git a/test/web/activity_pub/mrf/anti_link_spam_policy_test.exs b/test/web/activity_pub/mrf/anti_link_spam_policy_test.exs new file mode 100644 index 000000000..a456e863c --- /dev/null +++ b/test/web/activity_pub/mrf/anti_link_spam_policy_test.exs @@ -0,0 +1,120 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicyTest do + use Pleroma.DataCase + import Pleroma.Factory + + alias Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy + + @linkless_message %{ + "type" => "Create", + "object" => %{ + "content" => "hi world!" + } + } + + @linkful_message %{ + "type" => "Create", + "object" => %{ + "content" => "hi world!" + } + } + + describe "with new user" do + test "it allows posts without links" do + user = insert(:user) + + assert user.info.note_count == 0 + + message = + @linkless_message + |> Map.put("actor", user.ap_id) + + {:ok, _message} = AntiLinkSpamPolicy.filter(message) + end + + test "it disallows posts with links" do + user = insert(:user) + + assert user.info.note_count == 0 + + message = + @linkful_message + |> Map.put("actor", user.ap_id) + + {:reject, _} = AntiLinkSpamPolicy.filter(message) + end + end + + describe "with old user" do + test "it allows posts without links" do + user = insert(:user, info: %{note_count: 1}) + + assert user.info.note_count == 1 + + message = + @linkless_message + |> Map.put("actor", user.ap_id) + + {:ok, _message} = AntiLinkSpamPolicy.filter(message) + end + + test "it allows posts with links" do + user = insert(:user, info: %{note_count: 1}) + + assert user.info.note_count == 1 + + message = + @linkful_message + |> Map.put("actor", user.ap_id) + + {:ok, _message} = AntiLinkSpamPolicy.filter(message) + end + end + + describe "with followed new user" do + test "it allows posts without links" do + user = insert(:user, info: %{follower_count: 1}) + + assert user.info.follower_count == 1 + + message = + @linkless_message + |> Map.put("actor", user.ap_id) + + {:ok, _message} = AntiLinkSpamPolicy.filter(message) + end + + test "it allows posts with links" do + user = insert(:user, info: %{follower_count: 1}) + + assert user.info.follower_count == 1 + + message = + @linkful_message + |> Map.put("actor", user.ap_id) + + {:ok, _message} = AntiLinkSpamPolicy.filter(message) + end + end + + describe "with unknown actors" do + test "it rejects posts without links" do + message = + @linkless_message + |> Map.put("actor", "http://invalid.actor") + + {:reject, _} = AntiLinkSpamPolicy.filter(message) + end + + test "it rejects posts with links" do + message = + @linkful_message + |> Map.put("actor", "http://invalid.actor") + + {:reject, _} = AntiLinkSpamPolicy.filter(message) + end + end +end From 21dacd4b15f92726f8a26fb4ec7b06b7f98d97f1 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 19 Jun 2019 16:33:49 +0000 Subject: [PATCH 2/4] unbreak polls --- .../activity_pub/mrf/anti_link_spam_policy.ex | 2 ++ .../mrf/anti_link_spam_policy_test.exs | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex index 33ea61f5c..14e5955ee 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex @@ -20,6 +20,8 @@ defp contains_links?(%{"content" => content} = _object) do |> length() > 0 end + defp contains_links?(_), do: false + def filter(%{"type" => "Create", "actor" => actor, "object" => object} = message) do with {:ok, %User{} = u} <- User.get_or_fetch_by_ap_id(actor), {:contains_links, true} <- {:contains_links, contains_links?(object)}, diff --git a/test/web/activity_pub/mrf/anti_link_spam_policy_test.exs b/test/web/activity_pub/mrf/anti_link_spam_policy_test.exs index a456e863c..284c13336 100644 --- a/test/web/activity_pub/mrf/anti_link_spam_policy_test.exs +++ b/test/web/activity_pub/mrf/anti_link_spam_policy_test.exs @@ -22,6 +22,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicyTest do } } + @response_message %{ + "type" => "Create", + "object" => %{ + "name" => "yes", + "type" => "Answer" + } + } + describe "with new user" do test "it allows posts without links" do user = insert(:user) @@ -117,4 +125,16 @@ test "it rejects posts with links" do {:reject, _} = AntiLinkSpamPolicy.filter(message) end end + + describe "with contentless-objects" do + test "it does not reject them or error out" do + user = insert(:user, info: %{note_count: 1}) + + message = + @response_message + |> Map.put("actor", user.ap_id) + + {:ok, _message} = AntiLinkSpamPolicy.filter(message) + end + end end From 630ac6a921bc80a93f5f994731c1085fd1b9d3e8 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 20 Jun 2019 03:01:03 +0000 Subject: [PATCH 3/4] docs: better description for mrf anti link spam --- CHANGELOG.md | 2 +- docs/config.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ced0573f6..0dc8b547d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,7 +62,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - MRF: Support for running subchains. - Configuration: `skip_thread_containment` option - Configuration: `rate_limit` option. See `Pleroma.Plugs.RateLimiter` documentation for details. -- MRF: Support for filtering out likely spam messages using naive heuristics. +- MRF: Support for filtering out likely spam messages by rejecting posts from new users that contain links. ### Changed - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer diff --git a/docs/config.md b/docs/config.md index 4e9697afc..b75193545 100644 --- a/docs/config.md +++ b/docs/config.md @@ -90,7 +90,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `Pleroma.Web.ActivityPub.MRF.SubchainPolicy`: Selectively runs other MRF policies when messages match (see ``:mrf_subchain`` section) * `Pleroma.Web.ActivityPub.MRF.RejectNonPublic`: Drops posts with non-public visibility settings (See ``:mrf_rejectnonpublic`` section) * `Pleroma.Web.ActivityPub.MRF.EnsureRePrepended`: Rewrites posts to ensure that replies to posts with subjects do not have an identical subject and instead begin with re:. - * `Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy`: Rejects posts from likely spambots using naive heuristics. + * `Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy`: Rejects posts from likely spambots by rejecting posts from new users that contain links. * `public`: Makes the client API in authentificated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. * `quarantined_instances`: List of ActivityPub instances where private(DMs, followers-only) activities will not be send. * `managed_config`: Whenether the config for pleroma-fe is configured in this config or in ``static/config.json`` From 127a5a7d6567124b834a1f5399a0032c1c1f849d Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Fri, 21 Jun 2019 22:27:14 +0000 Subject: [PATCH 4/4] change the anti-link-spam MRF implementation to use old_user? instead of the previous name --- lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex index 14e5955ee..2da3eac2f 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do require Logger # has the user successfully posted before? - defp user_has_posted_before?(%User{} = u) do + defp old_user?(%User{} = u) do u.info.note_count > 0 || u.info.follower_count > 0 end @@ -25,13 +25,13 @@ defp contains_links?(_), do: false def filter(%{"type" => "Create", "actor" => actor, "object" => object} = message) do with {:ok, %User{} = u} <- User.get_or_fetch_by_ap_id(actor), {:contains_links, true} <- {:contains_links, contains_links?(object)}, - {:posted_before, true} <- {:posted_before, user_has_posted_before?(u)} do + {:old_user, true} <- {:old_user, old_user?(u)} do {:ok, message} else {:contains_links, false} -> {:ok, message} - {:posted_before, false} -> + {:old_user, false} -> {:reject, nil} {:error, _} ->