Merge branch 'features/emoji_stealer' into 'develop'

MRF.StealEmojiPolicy: New Policy

See merge request pleroma/pleroma!2385

CHANGELOG.md

@@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mix task to create trusted OAuth App.
 - Notifications: Added `follow_request` notification type.
 - Added `:reject_deletes` group to SimplePolicy
+- MRF (`EmojiStealPolicy`): New MRF Policy which allows to automatically download emojis from remote instances
 <details>
   <summary>API Changes</summary>
 - Mastodon API: Extended `/api/v1/instance`.

docs/configuration/cheatsheet.md

@@ -149,6 +149,11 @@ config :pleroma, :mrf_user_allowlist,
   * `:strip_followers` removes followers from the ActivityPub recipient list, ensuring they won't be delivered to home timelines
   * `:reject` rejects the message entirely
 
+#### mrf_steal_emoji
+* `hosts`: List of hosts to steal emojis from
+* `rejected_shortcodes`: Regex-list of shortcodes to reject
+* `size_limit`: File size limit (in bytes), checked before an emoji is saved to the disk
+
 ### :activitypub
 * `unfollow_blocked`: Whether blocks result in people getting unfollowed
 * `outgoing_blocks`: Whether to federate blocks to other instances

lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex

@@ -0,0 +1,97 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do
+  require Logger
+
+  alias Pleroma.Config
+
+  @moduledoc "Detect new emojis by their shortcode and steals them"
+  @behaviour Pleroma.Web.ActivityPub.MRF
+
+  defp remote_host?(host), do: host != Config.get([Pleroma.Web.Endpoint, :url, :host])
+
+  defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], [])
+
+  defp steal_emoji({shortcode, url}) do
+    url = Pleroma.Web.MediaProxy.url(url)
+    {:ok, response} = Pleroma.HTTP.get(url)
+    size_limit = Config.get([:mrf_steal_emoji, :size_limit], 50_000)
+
+    if byte_size(response.body) <= size_limit do
+      emoji_dir_path =
+        Config.get(
+          [:mrf_steal_emoji, :path],
+          Path.join(Config.get([:instance, :static_dir]), "emoji/stolen")
+        )
+
+      extension =
+        url
+        |> URI.parse()
+        |> Map.get(:path)
+        |> Path.basename()
+        |> Path.extname()
+
+      file_path = Path.join([emoji_dir_path, shortcode <> (extension || ".png")])
+
+      try do
+        :ok = File.write(file_path, response.body)
+
+        shortcode
+      rescue
+        e ->
+          Logger.warn("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}")
+          nil
+      end
+    else
+      Logger.debug(
+        "MRF.StealEmojiPolicy: :#{shortcode}: at #{url} (#{byte_size(response.body)} B) over size limit (#{
+          size_limit
+        } B)"
+      )
+
+      nil
+    end
+  rescue
+    e ->
+      Logger.warn("MRF.StealEmojiPolicy: Failed to fetch #{url}: #{inspect(e)}")
+      nil
+  end
+
+  @impl true
+  def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = message) do
+    host = URI.parse(actor).host
+
+    if remote_host?(host) and accept_host?(host) do
+      installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
+
+      new_emojis =
+        foreign_emojis
+        |> Enum.filter(fn {shortcode, _url} -> shortcode not in installed_emoji end)
+        |> Enum.filter(fn {shortcode, _url} ->
+          reject_emoji? =
+            Config.get([:mrf_steal_emoji, :rejected_shortcodes], [])
+            |> Enum.find(false, fn regex -> String.match?(shortcode, regex) end)
+
+          !reject_emoji?
+        end)
+        |> Enum.map(&steal_emoji(&1))
+        |> Enum.filter(& &1)
+
+      if !Enum.empty?(new_emojis) do
+        Logger.info("Stole new emojis: #{inspect(new_emojis)}")
+        Pleroma.Emoji.reload()
+      end
+    end
+
+    {:ok, message}
+  end
+
+  def filter(message), do: {:ok, message}
+
+  @impl true
+  def describe do
+    {:ok, %{}}
+  end
+end

test/support/http_request_mock.ex

@@ -1291,6 +1291,10 @@ def get("https://skippers-bin.com/notes/7x9tmrp97i", _, _, _) do
      }}
   end
 
+  def get("https://example.org/emoji/firedfox.png", _, _, _) do
+    {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/image.jpg")}}
+  end
+
   def get("https://skippers-bin.com/users/7v1w1r8ce6", _, _, _) do
     {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/sjw.json")}}
   end

test/web/activity_pub/mrf/steal_emoji_policy_test.exs

@@ -0,0 +1,64 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do
+  use Pleroma.DataCase
+
+  alias Pleroma.Config
+  alias Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy
+
+  setup_all do
+    Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
+    :ok
+  end
+
+  setup do
+    clear_config(:mrf_steal_emoji)
+
+    emoji_path = Path.join(Config.get([:instance, :static_dir]), "emoji/stolen")
+    File.rm_rf!(emoji_path)
+    File.mkdir!(emoji_path)
+
+    Pleroma.Emoji.reload()
+  end
+
+  test "does nothing by default" do
+    installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
+    refute "firedfox" in installed_emoji
+
+    message = %{
+      "type" => "Create",
+      "object" => %{
+        "emoji" => [{"firedfox", "https://example.org/emoji/firedfox.png"}],
+        "actor" => "https://example.org/users/admin"
+      }
+    }
+
+    assert {:ok, message} == StealEmojiPolicy.filter(message)
+
+    installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
+    refute "firedfox" in installed_emoji
+  end
+
+  test "Steals emoji on unknown shortcode from allowed remote host" do
+    installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
+    refute "firedfox" in installed_emoji
+
+    message = %{
+      "type" => "Create",
+      "object" => %{
+        "emoji" => [{"firedfox", "https://example.org/emoji/firedfox.png"}],
+        "actor" => "https://example.org/users/admin"
+      }
+    }
+
+    Config.put([:mrf_steal_emoji, :hosts], ["example.org"])
+    Config.put([:mrf_steal_emoji, :size_limit], 284_468)
+
+    assert {:ok, message} == StealEmojiPolicy.filter(message)
+
+    installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
+    assert "firedfox" in installed_emoji
+  end
+end