From f959bf7aa6b878ee5b669c4caabd5cdc4cc2dc9e Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 17 May 2019 18:21:11 +0200
Subject: [PATCH 01/64] MongooseIM: Add basic integration endpoints.

---
 .../web/mongooseim/mongoose_im_controller.ex  | 41 +++++++++++++
 lib/pleroma/web/router.ex                     |  5 ++
 .../mongoose_im_controller_test.exs           | 59 +++++++++++++++++++
 3 files changed, 105 insertions(+)
 create mode 100644 lib/pleroma/web/mongooseim/mongoose_im_controller.ex
 create mode 100644 test/web/mongooseim/mongoose_im_controller_test.exs

diff --git a/lib/pleroma/web/mongooseim/mongoose_im_controller.ex b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex
new file mode 100644
index 000000000..f8c634653
--- /dev/null
+++ b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex
@@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.MongooseIM.MongooseIMController do
+  use Pleroma.Web, :controller
+  alias Comeonin.Pbkdf2
+  alias Pleroma.User
+  alias Pleroma.Repo
+
+  def user_exists(conn, %{"user" => username}) do
+    with %User{} <- Repo.get_by(User, nickname: username, local: true) do
+      conn
+      |> json(true)
+    else
+      _ ->
+        conn
+        |> put_status(:not_found)
+        |> json(false)
+    end
+  end
+
+  def check_password(conn, %{"user" => username, "pass" => password}) do
+    with %User{password_hash: password_hash} <-
+           Repo.get_by(User, nickname: username, local: true),
+         true <- Pbkdf2.checkpw(password, password_hash) do
+      conn
+      |> json(true)
+    else
+      false ->
+        conn
+        |> put_status(403)
+        |> json(false)
+
+      _ ->
+        conn
+        |> put_status(:not_found)
+        |> json(false)
+    end
+  end
+end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 6a4e4a1d4..552778c92 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -704,6 +704,11 @@ defmodule Pleroma.Web.Router do
     end
   end
 
+  scope "/", Pleroma.Web.MongooseIM do
+    get("/user_exists", MongooseIMController, :user_exists)
+    get("/check_password", MongooseIMController, :check_password)
+  end
+
   scope "/", Fallback do
     get("/registration/:token", RedirectController, :registration_page)
     get("/:maybe_nickname_or_id", RedirectController, :redirector_with_meta)
diff --git a/test/web/mongooseim/mongoose_im_controller_test.exs b/test/web/mongooseim/mongoose_im_controller_test.exs
new file mode 100644
index 000000000..eb83999bb
--- /dev/null
+++ b/test/web/mongooseim/mongoose_im_controller_test.exs
@@ -0,0 +1,59 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.MongooseIMController do
+  use Pleroma.Web.ConnCase
+  import Pleroma.Factory
+
+  test "/user_exists", %{conn: conn} do
+    _user = insert(:user, nickname: "lain")
+    _remote_user = insert(:user, nickname: "alice", local: false)
+
+    res =
+      conn
+      |> get(mongoose_im_path(conn, :user_exists), user: "lain")
+      |> json_response(200)
+
+    assert res == true
+
+    res =
+      conn
+      |> get(mongoose_im_path(conn, :user_exists), user: "alice")
+      |> json_response(404)
+
+    assert res == false
+
+    res =
+      conn
+      |> get(mongoose_im_path(conn, :user_exists), user: "bob")
+      |> json_response(404)
+
+    assert res == false
+  end
+
+  test "/check_password", %{conn: conn} do
+    user = insert(:user, password_hash: Comeonin.Pbkdf2.hashpwsalt("cool"))
+
+    res =
+      conn
+      |> get(mongoose_im_path(conn, :check_password), user: user.nickname, pass: "cool")
+      |> json_response(200)
+
+    assert res == true
+
+    res =
+      conn
+      |> get(mongoose_im_path(conn, :check_password), user: user.nickname, pass: "uncool")
+      |> json_response(403)
+
+    assert res == false
+
+    res =
+      conn
+      |> get(mongoose_im_path(conn, :check_password), user: "nobody", pass: "cool")
+      |> json_response(404)
+
+    assert res == false
+  end
+end

From 075eecec907b0a623a90eed44a0378a6812d8037 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 17 May 2019 18:32:30 +0200
Subject: [PATCH 02/64] Linting.

---
 lib/pleroma/web/mongooseim/mongoose_im_controller.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/mongooseim/mongoose_im_controller.ex b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex
index f8c634653..489d5d3a5 100644
--- a/lib/pleroma/web/mongooseim/mongoose_im_controller.ex
+++ b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex
@@ -5,8 +5,8 @@
 defmodule Pleroma.Web.MongooseIM.MongooseIMController do
   use Pleroma.Web, :controller
   alias Comeonin.Pbkdf2
-  alias Pleroma.User
   alias Pleroma.Repo
+  alias Pleroma.User
 
   def user_exists(conn, %{"user" => username}) do
     with %User{} <- Repo.get_by(User, nickname: username, local: true) do

From 2375e9a95ba9042958ff7e8f75830df4ab53fed2 Mon Sep 17 00:00:00 2001
From: Aaron Tinio <aptinio@gmail.com>
Date: Mon, 20 May 2019 06:02:50 +0800
Subject: [PATCH 03/64] Add report filtering to MRF.SimplePolicy

---
 config/config.exs                             |  1 +
 .../web/activity_pub/mrf/simple_policy.ex     | 13 ++++++++-
 .../activity_pub/mrf/simple_policy_test.exs   | 28 +++++++++++++++++++
 3 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/config/config.exs b/config/config.exs
index 9a10b0ff7..bab47a8a2 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -298,6 +298,7 @@
   media_removal: [],
   media_nsfw: [],
   federated_timeline_removal: [],
+  report_removal: [],
   reject: [],
   accept: []
 
diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
index 9627c3400..7190652d2 100644
--- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
@@ -94,6 +94,16 @@ defp check_ftl_removal(%{host: actor_host} = _actor_info, object) do
     {:ok, object}
   end
 
+  defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = object) do
+    if actor_host in Pleroma.Config.get([:mrf_simple, :report_removal]) do
+      {:reject, nil}
+    else
+      {:ok, object}
+    end
+  end
+
+  defp check_report_removal(_actor_info, object), do: {:ok, object}
+
   @impl true
   def filter(object) do
     actor_info = URI.parse(object["actor"])
@@ -102,7 +112,8 @@ def filter(object) do
          {:ok, object} <- check_reject(actor_info, object),
          {:ok, object} <- check_media_removal(actor_info, object),
          {:ok, object} <- check_media_nsfw(actor_info, object),
-         {:ok, object} <- check_ftl_removal(actor_info, object) do
+         {:ok, object} <- check_ftl_removal(actor_info, object),
+         {:ok, object} <- check_report_removal(actor_info, object) do
       {:ok, object}
     else
       _e -> {:reject, nil}
diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs
index 1e0511975..74af7dcde 100644
--- a/test/web/activity_pub/mrf/simple_policy_test.exs
+++ b/test/web/activity_pub/mrf/simple_policy_test.exs
@@ -15,6 +15,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do
       media_removal: [],
       media_nsfw: [],
       federated_timeline_removal: [],
+      report_removal: [],
       reject: [],
       accept: []
     )
@@ -85,6 +86,33 @@ defp build_media_message do
     }
   end
 
+  describe "when :report_removal" do
+    test "is empty" do
+      Config.put([:mrf_simple, :report_removal], [])
+      report_message = build_report_message()
+      local_message = build_local_message()
+
+      assert SimplePolicy.filter(report_message) == {:ok, report_message}
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+    end
+
+    test "has a matching host" do
+      Config.put([:mrf_simple, :report_removal], ["remote.instance"])
+      report_message = build_report_message()
+      local_message = build_local_message()
+
+      assert SimplePolicy.filter(report_message) == {:reject, nil}
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+    end
+  end
+
+  defp build_report_message do
+    %{
+      "actor" => "https://remote.instance/users/bob",
+      "type" => "Flag"
+    }
+  end
+
   describe "when :federated_timeline_removal" do
     test "is empty" do
       Config.put([:mrf_simple, :federated_timeline_removal], [])

From 54e9cb5c2db580bc12441f3651fa87a7b976137d Mon Sep 17 00:00:00 2001
From: Sadposter <hannah+pleroma@coffee-and-dreams.uk>
Date: Mon, 20 May 2019 12:39:23 +0100
Subject: [PATCH 04/64] Add API endpoints for a custom user mascot

---
 docs/api/pleroma_api.md                       | 39 +++++++++++++++++++
 lib/pleroma/user/info.ex                      | 21 ++++++++++
 .../mastodon_api/mastodon_api_controller.ex   | 36 ++++++++++++++++-
 lib/pleroma/web/router.ex                     |  3 ++
 4 files changed, 98 insertions(+), 1 deletion(-)

diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md
index dd0b6ca73..4d99a2d2b 100644
--- a/docs/api/pleroma_api.md
+++ b/docs/api/pleroma_api.md
@@ -252,6 +252,45 @@ See [Admin-API](Admin-API.md)
 ]
 ```
 
+## `/api/v1/pleroma/mascot`
+### Gets user mascot image
+* Method `GET`
+* Authentication: required
+
+* Response: JSON. Returns a mastodon media attachment entity.
+* Example response:
+```json
+{
+    "id": "abcdefg",
+    "url": "https://pleroma.example.org/media/abcdefg.png",
+    "type": "image",
+    "pleroma": {
+        "mime_type": "image/png"
+    }
+}
+```
+
+### Updates user mascot image
+* Method `PUT`
+* Authentication: required
+* Params:
+    * `image`: Multipart image
+* Response: JSON. Returns a mastodon media attachment entity
+  when successful, otherwise returns HTTP 415 `{"error": "error_msg"}`
+* Example response:
+```json
+{
+    "id": "abcdefg",
+    "url": "https://pleroma.example.org/media/abcdefg.png",
+    "type": "image",
+    "pleroma": {
+        "mime_type": "image/png"
+    }
+}
+```
+* Note: Behaves exactly the same as `POST /api/v1/upload`.
+  Can only accept images - any attempt to upload non-image files will be met with `HTTP 415 Unsupported Media Type`.
+
 ## `/api/pleroma/notification_settings`
 ### Updates user notification settings
 * Method `PUT`
diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex
index 5f0cefc00..ffcd06e3e 100644
--- a/lib/pleroma/user/info.ex
+++ b/lib/pleroma/user/info.ex
@@ -43,6 +43,19 @@ defmodule Pleroma.User.Info do
     field(:hide_favorites, :boolean, default: true)
     field(:pinned_activities, {:array, :string}, default: [])
     field(:flavour, :string, default: nil)
+
+    field(:mascot, :map,
+      default: %{
+        id: "pleromatan",
+        url: "/images/pleroma-fox-tan-smol.png",
+        type: "image",
+        preview_url: "/images/pleroma-fox-tan-smol.png",
+        pleroma: %{
+          mime_type: "image/png"
+        }
+      }
+    )
+
     field(:emoji, {:array, :map}, default: [])
 
     field(:notification_settings, :map,
@@ -248,6 +261,14 @@ def mastodon_flavour_update(info, flavour) do
     |> validate_required([:flavour])
   end
 
+  def mascot_update(info, url) do
+    params = %{mascot: url}
+
+    info
+    |> cast(params, [:mascot])
+    |> validate_required([:mascot])
+  end
+
   def set_source_data(info, source_data) do
     params = %{source_data: source_data}
 
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 1051861ff..67f363859 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -707,6 +707,40 @@ def upload(%{assigns: %{user: user}} = conn, %{"file" => file} = data) do
     end
   end
 
+  def set_mascot(%{assigns: %{user: user}} = conn, %{"file" => file}) do
+    with {:ok, object} <- ActivityPub.upload(file, actor: User.ap_id(user)),
+         %{} = attachment_data <- Map.put(object.data, "id", object.id),
+         %{type: type} = rendered <-
+           StatusView.render("attachment.json", %{attachment: attachment_data}) do
+      # Reject if not an image
+      if type == "image" do
+        # Sure!
+        # Save to the user's info
+        info_changeset = User.Info.mascot_update(user.info, rendered)
+
+        user_changeset =
+          user
+          |> Ecto.Changeset.change()
+          |> Ecto.Changeset.put_embed(:info, info_changeset)
+
+        {:ok, _user} = User.update_and_set_cache(user_changeset)
+
+        conn
+        |> json(rendered)
+      else
+        conn
+        |> send_resp(415, Jason.encode!(%{"error" => "mascots can only be images"}))
+      end
+    end
+  end
+
+  def get_mascot(%{assigns: %{user: user}} = conn, _params) do
+    %{info: %{mascot: mascot}} = user
+
+    conn
+    |> json(mascot)
+  end
+
   def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id),
          %Object{data: %{"likes" => likes}} <- Object.normalize(object) do
@@ -1329,7 +1363,7 @@ def index(%{assigns: %{user: user}} = conn, _params) do
             display_sensitive_media: false,
             reduce_motion: false,
             max_toot_chars: limit,
-            mascot: "/images/pleroma-fox-tan-smol.png"
+            mascot: Map.get(user.info.mascot, "url", "/images/pleroma-fox-tan-smol.png")
           },
           rights: %{
             delete_others_notice: present?(user.info.is_moderator),
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 6a4e4a1d4..4c29b24eb 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -352,6 +352,9 @@ defmodule Pleroma.Web.Router do
 
       post("/pleroma/flavour/:flavour", MastodonAPIController, :set_flavour)
 
+      get("/pleroma/mascot", MastodonAPIController, :get_mascot)
+      put("/pleroma/mascot", MastodonAPIController, :set_mascot)
+
       post("/reports", MastodonAPIController, :reports)
     end
 

From e81f0fc6d45249dd70656c58af926c21c70c482f Mon Sep 17 00:00:00 2001
From: Sadposter <hannah+pleroma@coffee-and-dreams.uk>
Date: Mon, 20 May 2019 12:58:06 +0100
Subject: [PATCH 05/64] Add mascot get/set tests

---
 .../mastodon_api/mastodon_api_controller.ex   |   1 +
 test/fixtures/sound.mp3                       | Bin 0 -> 521 bytes
 .../mastodon_api_controller_test.exs          |  65 ++++++++++++++++++
 3 files changed, 66 insertions(+)
 create mode 100644 test/fixtures/sound.mp3

diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 67f363859..d7f095a1f 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -729,6 +729,7 @@ def set_mascot(%{assigns: %{user: user}} = conn, %{"file" => file}) do
         |> json(rendered)
       else
         conn
+        |> put_resp_content_type("application/json")
         |> send_resp(415, Jason.encode!(%{"error" => "mascots can only be images"}))
       end
     end
diff --git a/test/fixtures/sound.mp3 b/test/fixtures/sound.mp3
new file mode 100644
index 0000000000000000000000000000000000000000..9f0f661a3bc34b6366304e9fd1d4fe8513411083
GIT binary patch
literal 521
zcmezWd%_V0bP$o5mkt!;2VzDB1}091|Fj1{y8?V1eO-<93=IrecEX$_s-VK;;K;!E
q0OXGES_-dW5+jBF|6AY)1M>j}#w9=>D=;vaG%zr*zym6jY5)LcXO3V1

literal 0
HcmV?d00001

diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs
index cbff141c8..87e1c105d 100644
--- a/test/web/mastodon_api/mastodon_api_controller_test.exs
+++ b/test/web/mastodon_api/mastodon_api_controller_test.exs
@@ -1455,6 +1455,71 @@ test "media upload", %{conn: conn} do
     assert object.data["actor"] == User.ap_id(user)
   end
 
+  test "mascot upload", %{conn: conn} do
+    user = insert(:user)
+
+    non_image_file = %Plug.Upload{
+      content_type: "audio/mpeg",
+      path: Path.absname("test/fixtures/sound.mp3"),
+      filename: "sound.mp3"
+    }
+
+    conn =
+      conn
+      |> assign(:user, user)
+      |> put("/api/v1/pleroma/mascot", %{"file" => non_image_file})
+
+    assert json_response(conn, 415)
+
+    file = %Plug.Upload{
+      content_type: "image/jpg",
+      path: Path.absname("test/fixtures/image.jpg"),
+      filename: "an_image.jpg"
+    }
+
+    conn =
+      build_conn()
+      |> assign(:user, user)
+      |> put("/api/v1/pleroma/mascot", %{"file" => file})
+
+    assert %{"id" => _, "type" => image} = json_response(conn, 200)
+  end
+
+  test "mascot retrieving", %{conn: conn} do
+    user = insert(:user)
+    # When user hasn't set a mascot, we should just get pleroma tan back
+    conn =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/pleroma/mascot")
+
+    assert %{"url" => url} = json_response(conn, 200)
+    assert url =~ "pleroma-fox-tan-smol"
+
+    # When a user sets their mascot, we should get that back
+    file = %Plug.Upload{
+      content_type: "image/jpg",
+      path: Path.absname("test/fixtures/image.jpg"),
+      filename: "an_image.jpg"
+    }
+
+    conn = 
+      build_conn()
+      |> assign(:user, user)
+      |> put("/api/v1/pleroma/mascot", %{"file" => file})
+    assert json_response(conn, 200)
+
+    user = User.get_cached_by_id(user.id)
+
+    conn =
+      build_conn()
+      |> assign(:user, user)
+      |> get("/api/v1/pleroma/mascot")
+
+    assert %{"url" => url, "type" => "image"} = json_response(conn, 200)
+    assert url =~ "an_image"
+  end
+
   test "hashtag timeline", %{conn: conn} do
     following = insert(:user)
 

From dc916ba15f0fd77afa015084849f082065ed6f74 Mon Sep 17 00:00:00 2001
From: Sadposter <hannah+pleroma@coffee-and-dreams.uk>
Date: Mon, 20 May 2019 12:58:17 +0100
Subject: [PATCH 06/64] Format mascot tests

---
 test/web/mastodon_api/mastodon_api_controller_test.exs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs
index 87e1c105d..1d9f5a816 100644
--- a/test/web/mastodon_api/mastodon_api_controller_test.exs
+++ b/test/web/mastodon_api/mastodon_api_controller_test.exs
@@ -1503,10 +1503,11 @@ test "mascot retrieving", %{conn: conn} do
       filename: "an_image.jpg"
     }
 
-    conn = 
+    conn =
       build_conn()
       |> assign(:user, user)
       |> put("/api/v1/pleroma/mascot", %{"file" => file})
+
     assert json_response(conn, 200)
 
     user = User.get_cached_by_id(user.id)

From 3d0d9e7a5680bcb1b79e5f4aab0e8c514fc9e5ff Mon Sep 17 00:00:00 2001
From: Sadposter <hannah+pleroma@coffee-and-dreams.uk>
Date: Mon, 20 May 2019 13:10:04 +0100
Subject: [PATCH 07/64] Use string map for default mascot

---
 lib/pleroma/user/info.ex | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex
index ffcd06e3e..e76d04d7f 100644
--- a/lib/pleroma/user/info.ex
+++ b/lib/pleroma/user/info.ex
@@ -46,12 +46,12 @@ defmodule Pleroma.User.Info do
 
     field(:mascot, :map,
       default: %{
-        id: "pleromatan",
-        url: "/images/pleroma-fox-tan-smol.png",
-        type: "image",
-        preview_url: "/images/pleroma-fox-tan-smol.png",
-        pleroma: %{
-          mime_type: "image/png"
+        "id" => "pleromatan",
+        "url" => "/images/pleroma-fox-tan-smol.png",
+        "type" => "image",
+        "preview_url" => "/images/pleroma-fox-tan-smol.png",
+        "pleroma" => %{
+          "mime_type" => "image/png"
         }
       }
     )

From d835810610e5e7660a56d305bc8509e38c642942 Mon Sep 17 00:00:00 2001
From: Sadposter <hannah+pleroma@coffee-and-dreams.uk>
Date: Mon, 20 May 2019 14:19:42 +0100
Subject: [PATCH 08/64] Add changelog entry for mascot config

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 87101db30..07e6bce31 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Configuration: `report_uri` option
 - Pleroma API: User subscriptions
 - Pleroma API: Healthcheck endpoint
+- Pleroma API: `/api/v1/pleroma/mascot` per-user frontend mascot configuration endpoints
 - Admin API: Endpoints for listing/revoking invite tokens
 - Admin API: Endpoints for making users follow/unfollow each other
 - Admin API: added filters (role, tags, email, name) for users endpoint

From daeae8e2e7c506b72c66dea6ac790408f948ec16 Mon Sep 17 00:00:00 2001
From: Sadposter <hannah+pleroma@coffee-and-dreams.uk>
Date: Mon, 20 May 2019 16:12:55 +0100
Subject: [PATCH 09/64] Move default mascot configuration to `config/`

---
 config/config.exs                             | 13 ++++++++++++
 docs/config.md                                | 10 ++++++++++
 lib/pleroma/user.ex                           | 20 +++++++++++++++++++
 lib/pleroma/user/info.ex                      | 14 +------------
 .../mastodon_api/mastodon_api_controller.ex   |  4 ++--
 5 files changed, 46 insertions(+), 15 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index bab47a8a2..72908266d 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -276,6 +276,19 @@
     showInstanceSpecificPanel: true
   }
 
+config :pleroma, :assets,
+  mascots: [
+    pleroma_fox_tan: %{
+      url: "/images/pleroma-fox-tan-smol.png",
+      mime_type: "image/png"
+    },
+    pleroma_fox_tan_shy: %{
+      url: "/images/pleroma-fox-tan-shy.png",
+      mime_type: "image/png"
+    }
+  ],
+  default_mascot: :pleroma_fox_tan
+
 config :pleroma, :activitypub,
   accept_blocks: true,
   unfollow_blocked: true,
diff --git a/docs/config.md b/docs/config.md
index 450d73fda..197326bbd 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -203,6 +203,16 @@ This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:i
 * `hide_post_stats`: Hide notices statistics(repeats, favorites, …)
 * `hide_user_stats`: Hide profile statistics(posts, posts per day, followers, followings, …)
 
+## :assets
+
+This section configures assets to be used with various frontends. Currently the only option
+relates to mascots on the mastodon frontend
+
+* `mascots`: KeywordList of mascots, each element __MUST__ contain both a `url` and a
+  `mime_type` key.
+* `default_mascot`: An element from `mascots` - This will be used as the default mascot
+  on MastoFE (default: `:pleroma_fox_tan`)
+
 ## :mrf_simple
 * `media_removal`: List of instances to remove medias from
 * `media_nsfw`: List of instances to put medias as NSFW(sensitive) from
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 28da310ee..05fe58f7c 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1402,4 +1402,24 @@ def toggle_confirmation(%User{} = user) do
     |> put_embed(:info, info_changeset)
     |> update_and_set_cache()
   end
+
+  def get_mascot(%{info: %{mascot: %{} = mascot}}) when not is_nil(mascot) do
+    mascot
+  end
+
+  def get_mascot(%{info: %{mascot: mascot}}) when is_nil(mascot) do
+    # use instance-default
+    config = Pleroma.Config.get([:assets, :mascots])
+    default_mascot = Pleroma.Config.get([:assets, :default_mascot])
+    mascot = Keyword.get(config, default_mascot)
+
+    %{
+      "id" => "default-mascot",
+      "url" => mascot[:url],
+      "preview_url" => mascot[:url],
+      "pleroma" => %{
+        "mime_type" => mascot[:mime_type]
+      }
+    }
+  end
 end
diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex
index e76d04d7f..6397e2737 100644
--- a/lib/pleroma/user/info.ex
+++ b/lib/pleroma/user/info.ex
@@ -43,19 +43,7 @@ defmodule Pleroma.User.Info do
     field(:hide_favorites, :boolean, default: true)
     field(:pinned_activities, {:array, :string}, default: [])
     field(:flavour, :string, default: nil)
-
-    field(:mascot, :map,
-      default: %{
-        "id" => "pleromatan",
-        "url" => "/images/pleroma-fox-tan-smol.png",
-        "type" => "image",
-        "preview_url" => "/images/pleroma-fox-tan-smol.png",
-        "pleroma" => %{
-          "mime_type" => "image/png"
-        }
-      }
-    )
-
+    field(:mascot, :map, default: nil)
     field(:emoji, {:array, :map}, default: [])
 
     field(:notification_settings, :map,
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index d7f095a1f..1ec0f30a1 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -736,7 +736,7 @@ def set_mascot(%{assigns: %{user: user}} = conn, %{"file" => file}) do
   end
 
   def get_mascot(%{assigns: %{user: user}} = conn, _params) do
-    %{info: %{mascot: mascot}} = user
+    mascot = User.get_mascot(user)
 
     conn
     |> json(mascot)
@@ -1364,7 +1364,7 @@ def index(%{assigns: %{user: user}} = conn, _params) do
             display_sensitive_media: false,
             reduce_motion: false,
             max_toot_chars: limit,
-            mascot: Map.get(user.info.mascot, "url", "/images/pleroma-fox-tan-smol.png")
+            mascot: User.get_mascot(user)["url"]
           },
           rights: %{
             delete_others_notice: present?(user.info.is_moderator),

From eb02edcad9cb0d65fc216408960aec63713e5d2b Mon Sep 17 00:00:00 2001
From: Aaron Tinio <aptinio@gmail.com>
Date: Tue, 21 May 2019 00:35:46 +0800
Subject: [PATCH 10/64] Add virtual :thread_muted? field

that may be set when fetching activities
---
 lib/pleroma/activity.ex                       | 12 +++++++++
 lib/pleroma/web/activity_pub/activity_pub.ex  |  8 ++++++
 .../web/mastodon_api/views/status_view.ex     |  8 +++++-
 .../web/twitter_api/views/activity_view.ex    |  8 +++++-
 test/activity_test.exs                        | 26 +++++++++++++++++++
 test/user_test.exs                            |  2 +-
 6 files changed, 61 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 4e54b15ba..99589590c 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -10,6 +10,7 @@ defmodule Pleroma.Activity do
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Repo
+  alias Pleroma.ThreadMute
   alias Pleroma.User
 
   import Ecto.Changeset
@@ -37,6 +38,7 @@ defmodule Pleroma.Activity do
     field(:local, :boolean, default: true)
     field(:actor, :string)
     field(:recipients, {:array, :string}, default: [])
+    field(:thread_muted?, :boolean, virtual: true)
     # This is a fake relation, do not use outside of with_preloaded_bookmark/get_bookmark
     has_one(:bookmark, Bookmark)
     has_many(:notifications, Notification, on_delete: :delete_all)
@@ -90,6 +92,16 @@ def with_preloaded_bookmark(query, %User{} = user) do
 
   def with_preloaded_bookmark(query, _), do: query
 
+  def with_set_thread_muted_field(query, %User{} = user) do
+    from([a] in query,
+      left_join: tm in ThreadMute,
+      on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data),
+      select: %Activity{a | thread_muted?: not is_nil(tm.id)}
+    )
+  end
+
+  def with_set_thread_muted_field(query, _), do: query
+
   def get_by_ap_id(ap_id) do
     Repo.one(
       from(
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 5c3156978..3d9679ec0 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -834,6 +834,13 @@ defp maybe_preload_bookmarks(query, opts) do
     |> Activity.with_preloaded_bookmark(opts["user"])
   end
 
+  defp maybe_set_thread_muted_field(query, %{"skip_preload" => true}), do: query
+
+  defp maybe_set_thread_muted_field(query, opts) do
+    query
+    |> Activity.with_set_thread_muted_field(opts["user"])
+  end
+
   defp maybe_order(query, %{order: :desc}) do
     query
     |> order_by(desc: :id)
@@ -852,6 +859,7 @@ def fetch_activities_query(recipients, opts \\ %{}) do
     base_query
     |> maybe_preload_objects(opts)
     |> maybe_preload_bookmarks(opts)
+    |> maybe_set_thread_muted_field(opts)
     |> maybe_order(opts)
     |> restrict_recipients(recipients, opts["user"])
     |> restrict_tag(opts)
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index c93d915e5..e55f9b96e 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -157,6 +157,12 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity
 
     bookmarked = Activity.get_bookmark(activity, opts[:for]) != nil
 
+    thread_muted? =
+      case activity.thread_muted? do
+        thread_muted? when is_boolean(thread_muted?) -> thread_muted?
+        nil -> CommonAPI.thread_muted?(user, activity)
+      end
+
     attachment_data = object.data["attachment"] || []
     attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment)
 
@@ -228,7 +234,7 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity
       reblogged: reblogged?(activity, opts[:for]),
       favourited: present?(favorited),
       bookmarked: present?(bookmarked),
-      muted: CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user),
+      muted: thread_muted? || User.mutes?(opts[:for], user),
       pinned: pinned?(activity, user),
       sensitive: sensitive,
       spoiler_text: summary_html,
diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex
index 44bcafe0e..e84af84dc 100644
--- a/lib/pleroma/web/twitter_api/views/activity_view.ex
+++ b/lib/pleroma/web/twitter_api/views/activity_view.ex
@@ -284,6 +284,12 @@ def render(
         Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
       )
 
+    thread_muted? =
+      case activity.thread_muted? do
+        thread_muted? when is_boolean(thread_muted?) -> thread_muted?
+        nil -> CommonAPI.thread_muted?(user, activity)
+      end
+
     %{
       "id" => activity.id,
       "uri" => object.data["id"],
@@ -314,7 +320,7 @@ def render(
       "summary" => summary,
       "summary_html" => summary |> Formatter.emojify(object.data["emoji"]),
       "card" => card,
-      "muted" => CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user)
+      "muted" => thread_muted? || User.mutes?(opts[:for], user)
     }
   end
 
diff --git a/test/activity_test.exs b/test/activity_test.exs
index 7e91d534b..15c95502a 100644
--- a/test/activity_test.exs
+++ b/test/activity_test.exs
@@ -6,6 +6,7 @@ defmodule Pleroma.ActivityTest do
   use Pleroma.DataCase
   alias Pleroma.Activity
   alias Pleroma.Bookmark
+  alias Pleroma.ThreadMute
   import Pleroma.Factory
 
   test "returns an activity by it's AP id" do
@@ -47,6 +48,31 @@ test "preloading a bookmark" do
     assert queried_activity.bookmark == bookmark3
   end
 
+  test "setting thread_muted?" do
+    activity = insert(:note_activity)
+    user = insert(:user)
+    annoyed_user = insert(:user)
+    {:ok, _} = ThreadMute.add_mute(annoyed_user.id, activity.data["context"])
+
+    activity_with_unset_thread_muted_field =
+      Ecto.Query.from(Activity)
+      |> Repo.one()
+
+    activity_for_user =
+      Ecto.Query.from(Activity)
+      |> Activity.with_set_thread_muted_field(user)
+      |> Repo.one()
+
+    activity_for_annoyed_user =
+      Ecto.Query.from(Activity)
+      |> Activity.with_set_thread_muted_field(annoyed_user)
+      |> Repo.one()
+
+    assert activity_with_unset_thread_muted_field.thread_muted? == nil
+    assert activity_for_user.thread_muted? == false
+    assert activity_for_annoyed_user.thread_muted? == true
+  end
+
   describe "getting a bookmark" do
     test "when association is loaded" do
       user = insert(:user)
diff --git a/test/user_test.exs b/test/user_test.exs
index 10e463ff8..cb6afbe07 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -902,7 +902,7 @@ test "hide a user's statuses from timelines and notifications" do
 
       assert [activity] == ActivityPub.fetch_public_activities(%{}) |> Repo.preload(:bookmark)
 
-      assert [activity] ==
+      assert [%{activity | thread_muted?: CommonAPI.thread_muted?(user2, activity)}] ==
                ActivityPub.fetch_activities([user2.ap_id | user2.following], %{"user" => user2})
 
       {:ok, _user} = User.deactivate(user)

From 5aa107d9126c484f0ae06eddec6995b535623b32 Mon Sep 17 00:00:00 2001
From: Aaron Tinio <aptinio@gmail.com>
Date: Tue, 21 May 2019 00:59:12 +0800
Subject: [PATCH 11/64] Document MRF.Simple :report_removal

---
 docs/config/mrf.md | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/docs/config/mrf.md b/docs/config/mrf.md
index 2cc16cef0..45be18fc5 100644
--- a/docs/config/mrf.md
+++ b/docs/config/mrf.md
@@ -5,11 +5,12 @@ Possible uses include:
 
 * marking incoming messages with media from a given account or instance as sensitive
 * rejecting messages from a specific instance
+* rejecting reports (flags) from a specific instance
 * removing/unlisting messages from the public timelines
 * removing media from messages
 * sending only public messages to a specific instance
 
-The MRF provides user-configurable policies.  The default policy is `NoOpPolicy`, which disables the MRF functionality.  Pleroma also includes an easy to use policy called `SimplePolicy` which maps messages matching certain pre-defined criterion to actions built into the policy module.  
+The MRF provides user-configurable policies.  The default policy is `NoOpPolicy`, which disables the MRF functionality.  Pleroma also includes an easy to use policy called `SimplePolicy` which maps messages matching certain pre-defined criterion to actions built into the policy module.
 It is possible to use multiple, active MRF policies at the same time.
 
 ## Quarantine Instances
@@ -41,12 +42,13 @@ Once `SimplePolicy` is enabled, you can configure various groups in the `:mrf_si
 * `media_nsfw`: Servers in this group will have the #nsfw tag and sensitive setting injected into incoming messages which contain media.
 * `reject`: Servers in this group will have their messages rejected.
 * `federated_timeline_removal`: Servers in this group will have their messages unlisted from the public timelines by flipping the `to` and `cc` fields.
+* `report_removal`: Servers in this group will have their reports (flags) rejected.
 
 Servers should be configured as lists.
 
 ### Example
 
-This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com` and remove messages from `spam.university` from the federated timeline:
+This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com`, remove messages from `spam.university` from the federated timeline and block reports (flags) from `whiny.whiner`:
 
 ```
 config :pleroma, :instance,
@@ -56,7 +58,8 @@ config :pleroma, :mrf_simple,
   media_removal: ["illegalporn.biz"],
   media_nsfw: ["porn.biz", "porn.business"],
   reject: ["spam.com"],
-  federated_timeline_removal: ["spam.university"]
+  federated_timeline_removal: ["spam.university"],
+  report_removal: ["whiny.whiner"]
 
 ```
 

From 75c7bb9289066bfaebe7d96aaa474d34ec4d5a5f Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Mon, 20 May 2019 17:18:59 -0500
Subject: [PATCH 12/64] Additional reserved usernames

---
 config/config.exs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/config/config.exs b/config/config.exs
index 61e2648a9..387c1a5a7 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -369,6 +369,7 @@
     "activities",
     "api",
     "auth",
+    "check_password",
     "dev",
     "friend-requests",
     "inbox",
@@ -389,6 +390,7 @@
     "status",
     "tag",
     "user-search",
+    "user_exists",
     "users",
     "web"
   ]

From f96e9b28bb5ee241a3f0ca6a622b925ca560c141 Mon Sep 17 00:00:00 2001
From: Aaron Tinio <aptinio@gmail.com>
Date: Tue, 21 May 2019 07:30:18 +0800
Subject: [PATCH 13/64] Fix prometheus-ecto error when not configured

---
 lib/pleroma/application.ex | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index eeb415084..dab45a0b2 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -131,19 +131,22 @@ def start(_type, _args) do
   defp setup_instrumenters do
     require Prometheus.Registry
 
-    :ok =
-      :telemetry.attach(
-        "prometheus-ecto",
-        [:pleroma, :repo, :query],
-        &Pleroma.Repo.Instrumenter.handle_event/4,
-        %{}
-      )
+    if Application.get_env(:prometheus, Pleroma.Repo.Instrumenter) do
+      :ok =
+        :telemetry.attach(
+          "prometheus-ecto",
+          [:pleroma, :repo, :query],
+          &Pleroma.Repo.Instrumenter.handle_event/4,
+          %{}
+        )
+
+      Pleroma.Repo.Instrumenter.setup()
+    end
 
     Prometheus.Registry.register_collector(:prometheus_process_collector)
     Pleroma.Web.Endpoint.MetricsExporter.setup()
     Pleroma.Web.Endpoint.PipelineInstrumenter.setup()
     Pleroma.Web.Endpoint.Instrumenter.setup()
-    Pleroma.Repo.Instrumenter.setup()
   end
 
   def enabled_hackney_pools do

From c972d0bb14ee5a65f053b8c9629d93fc9b94ca78 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Tue, 21 May 2019 04:58:26 +0000
Subject: [PATCH 14/64] http: bump connection timeout to 10 seconds

---
 CHANGELOG.md                   | 1 +
 lib/pleroma/http/connection.ex | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 07e6bce31..256df91b7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -73,6 +73,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Don't ship finmoji by default, they can be installed as an emoji pack
 - Hide deactivated users and their statuses
 - Posts which are marked sensitive or tagged nsfw no longer have link previews.
+- HTTP connection timeout is now set to 10 seconds.
 
 ### Fixed
 - Added an FTS index on objects. Running `vacuum analyze` and setting a larger `work_mem` is recommended.
diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex
index c0173465a..558005c19 100644
--- a/lib/pleroma/http/connection.ex
+++ b/lib/pleroma/http/connection.ex
@@ -8,7 +8,7 @@ defmodule Pleroma.HTTP.Connection do
   """
 
   @hackney_options [
-    connect_timeout: 2_000,
+    connect_timeout: 10_000,
     recv_timeout: 20_000,
     follow_redirect: true,
     pool: :federation

From d378b342ba0d7f54be3b47f031c602a578191dfd Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 21 May 2019 18:57:36 +0200
Subject: [PATCH 15/64] MongooseIM: Add documentation.

---
 CHANGELOG.md                    |  1 +
 docs/config/howto_mongooseim.md | 10 ++++++++++
 2 files changed, 11 insertions(+)
 create mode 100644 docs/config/howto_mongooseim.md

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 12c439135..b50ca895c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [unreleased]
 ### Added
+- [MongooseIM](https://github.com/esl/MongooseIM) http authentication support.
 - LDAP authentication
 - External OAuth provider authentication
 - A [job queue](https://git.pleroma.social/pleroma/pleroma_job_queue) for federation, emails, web push, etc.
diff --git a/docs/config/howto_mongooseim.md b/docs/config/howto_mongooseim.md
new file mode 100644
index 000000000..a33e590a1
--- /dev/null
+++ b/docs/config/howto_mongooseim.md
@@ -0,0 +1,10 @@
+# Configuring MongooseIM (XMPP Server) to use Pleroma for authentication
+
+If you want to give your Pleroma users an XMPP (chat) account, you can configure [MongooseIM](https://github.com/esl/MongooseIM) to use your Pleroma server for user authentication, automatically giving every local user an XMPP account.
+
+In general, you just have to follow the configuration described at [https://mongooseim.readthedocs.io/en/latest/authentication-backends/HTTP-authentication-module/](https://mongooseim.readthedocs.io/en/latest/authentication-backends/HTTP-authentication-module/) and do these changes to your mongooseim.cfg.
+
+1. Set the auth_method to `{auth_method, http}`.
+2. Add the http auth pool like this: `{http, global, auth, [{workers, 50}], [{server, "https://yourpleromainstance.com"}]}`
+
+Restart your MongooseIM server, your users should now be able to connect with their Pleroma credentials.

From c2b0b82e6a6d40f945feacc001ad984a17e23336 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Tue, 21 May 2019 00:41:40 +0000
Subject: [PATCH 16/64] object: add Object.prune()

---
 lib/pleroma/object.ex | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
index 740d687a3..cc6fc9c5d 100644
--- a/lib/pleroma/object.ex
+++ b/lib/pleroma/object.ex
@@ -130,6 +130,13 @@ def delete(%Object{data: %{"id" => id}} = object) do
     end
   end
 
+  def prune(%Object{data: %{"id" => id}} = object) do
+    with {:ok, object} <- Repo.delete(object),
+         {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
+      {:ok, object}
+    end
+  end
+
   def set_cache(%Object{data: %{"id" => ap_id}} = object) do
     Cachex.put(:object_cache, "object:#{ap_id}", object)
     {:ok, object}

From 73df9d690d5c1a9c11f0f04b8d877c0677022591 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Tue, 21 May 2019 00:41:58 +0000
Subject: [PATCH 17/64] object: fetcher: add support for reinjecting pruned
 objects

---
 lib/pleroma/object/fetcher.ex | 22 ++++++++++++++++++++--
 test/object/fetcher_test.exs  | 19 +++++++++++++++++++
 2 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index 8d4bcc95e..bb9388d4f 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -8,6 +8,19 @@ defmodule Pleroma.Object.Fetcher do
 
   @httpoison Application.get_env(:pleroma, :httpoison)
 
+  defp reinject_object(data) do
+    Logger.debug("Reinjecting object #{data["id"]}")
+
+    with data <- Transmogrifier.fix_object(data),
+         {:ok, object} <- Object.create(data) do
+      {:ok, object}
+    else
+      e ->
+        Logger.error("Error while processing object: #{inspect(e)}")
+        {:error, e}
+    end
+  end
+
   # TODO:
   # This will create a Create activity, which we need internally at the moment.
   def fetch_object_from_id(id) do
@@ -26,12 +39,17 @@ def fetch_object_from_id(id) do
              "object" => data
            },
            :ok <- Containment.contain_origin(id, params),
-           {:ok, activity} <- Transmogrifier.handle_incoming(params) do
-        {:ok, Object.normalize(activity, false)}
+           {:ok, activity} <- Transmogrifier.handle_incoming(params),
+           {:object, _data, %Object{} = object} <-
+             {:object, data, Object.normalize(activity, false)} do
+        {:ok, object}
       else
         {:error, {:reject, nil}} ->
           {:reject, nil}
 
+        {:object, data, nil} ->
+          reinject_object(data)
+
         object = %Object{} ->
           {:ok, object}
 
diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs
index 72f616782..d604fd5f5 100644
--- a/test/object/fetcher_test.exs
+++ b/test/object/fetcher_test.exs
@@ -87,4 +87,23 @@ test "all objects with fake directions are rejected by the object fetcher" do
         )
     end
   end
+
+  describe "pruning" do
+    test "it can refetch pruned objects" do
+      object_id = "http://mastodon.example.org/@admin/99541947525187367"
+
+      {:ok, object} = Fetcher.fetch_object_from_id(object_id)
+
+      assert object
+
+      {:ok, _object} = Object.prune(object)
+
+      refute Object.get_by_ap_id(object_id)
+
+      {:ok, %Object{} = object_two} = Fetcher.fetch_object_from_id(object_id)
+
+      assert object.data["id"] == object_two.data["id"]
+      assert object.id != object_two.id
+    end
+  end
 end

From 16b260fb19cca02463766c2e36a41bfcc823af9b Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Tue, 21 May 2019 01:21:28 +0000
Subject: [PATCH 18/64] add mix task to prune the object database using a
 configured retention period

---
 config/config.exs                 |  3 ++-
 docs/config.md                    |  1 +
 lib/mix/tasks/pleroma/database.ex | 40 +++++++++++++++++++++++++++++++
 3 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/config/config.exs b/config/config.exs
index 72908266d..466a6e9b7 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -239,7 +239,8 @@
   welcome_message: nil,
   max_report_comment_size: 1000,
   safe_dm_mentions: false,
-  healthcheck: false
+  healthcheck: false,
+  remote_post_retention_days: 90
 
 config :pleroma, :app_account_creation, enabled: true, max_requests: 25, interval: 1800
 
diff --git a/docs/config.md b/docs/config.md
index 197326bbd..a050068f4 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -104,6 +104,7 @@ config :pleroma, Pleroma.Emails.Mailer,
 * `max_report_comment_size`: The maximum size of the report comment (Default: `1000`)
 * `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). (Default: `false`)
 * `healthcheck`: if set to true, system data will be shown on ``/api/pleroma/healthcheck``.
+* `remote_post_retention_days`: the default amount of days to retain remote posts when pruning the database
 
 ## :app_account_creation
 REST API for creating an account settings
diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index f650b447d..fdb216037 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -23,6 +23,10 @@ defmodule Mix.Tasks.Pleroma.Database do
     Options:
     - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references
 
+  ## Prune old objects from the database
+
+      mix pleroma.database prune_objects
+
   ## Create a conversation for all existing DMs. Can be safely re-run.
 
       mix pleroma.database bump_all_conversations
@@ -72,4 +76,40 @@ def run(["update_users_following_followers_counts"]) do
     Enum.each(users, &User.remove_duplicated_following/1)
     Enum.each(users, &User.update_follower_count/1)
   end
+
+  def run(["prune_objects" | args]) do
+    {options, [], []} =
+      OptionParser.parse(
+        args,
+        strict: [
+          vacuum: :boolean
+        ]
+      )
+
+    Common.start_pleroma()
+
+    deadline = Pleroma.Config.get([:instance, :remote_post_retention_days])
+
+    Logger.info("Pruning objects older than #{deadline} days")
+
+    time_deadline =
+      NaiveDateTime.utc_now()
+      |> NaiveDateTime.add(-(deadline * 86_400))
+
+    Repo.query!(
+      "DELETE FROM objects WHERE inserted_at < $1 AND split_part(data->>'actor', '/', 3) != $2",
+      [time_deadline, Pleroma.Web.Endpoint.host()],
+      timeout: :infinity
+    )
+
+    if Keyword.get(options, :vacuum) do
+      Logger.info("Runnning VACUUM FULL")
+
+      Repo.query!(
+        "vacuum full;",
+        [],
+        timeout: :infinity
+      )
+    end
+  end
 end

From f446f94290a6cdae531859c2efa55f55dbaeb7e8 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Tue, 21 May 2019 01:22:27 +0000
Subject: [PATCH 19/64] add changelog entry for object pruning

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 256df91b7..a21c4bff2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -39,6 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Metadata: RelMe provider
 - OAuth: added support for refresh tokens
 - Emoji packs and emoji pack manager
+- Object pruning (`mix pleroma.database prune_objects`)
 
 ### Changed
 - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer

From a13d449b241b6e5fa14fb741694e63cd21569b2c Mon Sep 17 00:00:00 2001
From: Aaron Tinio <aptinio@gmail.com>
Date: Tue, 21 May 2019 09:39:32 +0800
Subject: [PATCH 20/64] Add tests for fallback routes

---
 test/web/fallback_test.exs | 46 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 test/web/fallback_test.exs

diff --git a/test/web/fallback_test.exs b/test/web/fallback_test.exs
new file mode 100644
index 000000000..514923a20
--- /dev/null
+++ b/test/web/fallback_test.exs
@@ -0,0 +1,46 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.FallbackTest do
+  use Pleroma.Web.ConnCase
+  import Pleroma.Factory
+
+  test "GET /registration/:token", %{conn: conn} do
+    assert conn
+           |> get("/registration/foo")
+           |> html_response(200) =~ "<!--server-generated-meta-->"
+  end
+
+  test "GET /:maybe_nickname_or_id", %{conn: conn} do
+    user = insert(:user)
+
+    assert conn
+           |> get("/foo")
+           |> html_response(200) =~ "<!--server-generated-meta-->"
+
+    refute conn
+           |> get("/" <> user.nickname)
+           |> html_response(200) =~ "<!--server-generated-meta-->"
+  end
+
+  test "GET /*path", %{conn: conn} do
+    assert conn
+           |> get("/foo")
+           |> html_response(200) =~ "<!--server-generated-meta-->"
+
+    assert conn
+           |> get("/foo/bar")
+           |> html_response(200) =~ "<!--server-generated-meta-->"
+  end
+
+  test "OPTIONS /*path", %{conn: conn} do
+    assert conn
+           |> options("/foo")
+           |> response(204) == ""
+
+    assert conn
+           |> options("/foo/bar")
+           |> response(204) == ""
+  end
+end

From 3ab9255eda21a5f8a25047375af9608e0c0c7592 Mon Sep 17 00:00:00 2001
From: Aaron Tinio <aptinio@gmail.com>
Date: Tue, 21 May 2019 09:40:29 +0800
Subject: [PATCH 21/64] Respond with a 404 Not implemented JSON error message

when requested API is not implemented
---
 CHANGELOG.md               | 1 +
 lib/pleroma/web/router.ex  | 7 +++++++
 test/web/fallback_test.exs | 6 ++++++
 3 files changed, 14 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 256df91b7..2ed380102 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -74,6 +74,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Hide deactivated users and their statuses
 - Posts which are marked sensitive or tagged nsfw no longer have link previews.
 - HTTP connection timeout is now set to 10 seconds.
+- Respond with a 404 Not implemented JSON error message when requested API is not implemented
 
 ### Fixed
 - Added an FTS index on objects. Running `vacuum analyze` and setting a larger `work_mem` is recommended.
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 4c29b24eb..49e28cc2d 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -710,6 +710,7 @@ defmodule Pleroma.Web.Router do
   scope "/", Fallback do
     get("/registration/:token", RedirectController, :registration_page)
     get("/:maybe_nickname_or_id", RedirectController, :redirector_with_meta)
+    get("/api*path", RedirectController, :api_not_implemented)
     get("/*path", RedirectController, :redirector)
 
     options("/*path", RedirectController, :empty)
@@ -721,6 +722,12 @@ defmodule Fallback.RedirectController do
   alias Pleroma.User
   alias Pleroma.Web.Metadata
 
+  def api_not_implemented(conn, _params) do
+    conn
+    |> put_status(404)
+    |> json(%{error: "Not implemented"})
+  end
+
   def redirector(conn, _params, code \\ 200) do
     conn
     |> put_resp_content_type("text/html")
diff --git a/test/web/fallback_test.exs b/test/web/fallback_test.exs
index 514923a20..cc78b3ae1 100644
--- a/test/web/fallback_test.exs
+++ b/test/web/fallback_test.exs
@@ -24,6 +24,12 @@ test "GET /:maybe_nickname_or_id", %{conn: conn} do
            |> html_response(200) =~ "<!--server-generated-meta-->"
   end
 
+  test "GET /api*path", %{conn: conn} do
+    assert conn
+           |> get("/api/foo")
+           |> json_response(404) == %{"error" => "Not implemented"}
+  end
+
   test "GET /*path", %{conn: conn} do
     assert conn
            |> get("/foo")

From f76268135c014c20a482d30a7c9596ec2e7d6a69 Mon Sep 17 00:00:00 2001
From: Aaron Tinio <aptinio@gmail.com>
Date: Wed, 22 May 2019 07:11:09 +0800
Subject: [PATCH 22/64] Fix failing test

---
 test/web/admin_api/admin_api_controller_test.exs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs
index ca12c7215..c15c67e31 100644
--- a/test/web/admin_api/admin_api_controller_test.exs
+++ b/test/web/admin_api/admin_api_controller_test.exs
@@ -397,14 +397,14 @@ test "it returns 500 if `registrations_open` is enabled", %{conn: conn, user: us
     end
   end
 
-  test "/api/pleroma/admin/invite_token" do
+  test "/api/pleroma/admin/users/invite_token" do
     admin = insert(:user, info: %{is_admin: true})
 
     conn =
       build_conn()
       |> assign(:user, admin)
       |> put_req_header("accept", "application/json")
-      |> get("/api/pleroma/admin/invite_token")
+      |> get("/api/pleroma/admin/users/invite_token")
 
     assert conn.status == 200
   end

From a023ca004cbd90e330cab35e4dfda16346d08668 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Wed, 22 May 2019 03:12:48 +0000
Subject: [PATCH 23/64] prune objects task: use Repo.delete_all()

---
 lib/mix/tasks/pleroma/database.ex | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index fdb216037..f9bafb277 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -5,6 +5,7 @@
 defmodule Mix.Tasks.Pleroma.Database do
   alias Mix.Tasks.Pleroma.Common
   alias Pleroma.Conversation
+  alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
   require Logger
@@ -78,6 +79,8 @@ def run(["update_users_following_followers_counts"]) do
   end
 
   def run(["prune_objects" | args]) do
+    import Ecto.Query
+
     {options, [], []} =
       OptionParser.parse(
         args,
@@ -96,11 +99,15 @@ def run(["prune_objects" | args]) do
       NaiveDateTime.utc_now()
       |> NaiveDateTime.add(-(deadline * 86_400))
 
-    Repo.query!(
-      "DELETE FROM objects WHERE inserted_at < $1 AND split_part(data->>'actor', '/', 3) != $2",
-      [time_deadline, Pleroma.Web.Endpoint.host()],
-      timeout: :infinity
+    public = "https://www.w3.org/ns/activitystreams#Public"
+
+    from(o in Object,
+      where: fragment("?->'to' \\? ? OR ?->'cc' \\? ?", o.data, ^public, o.data, ^public),
+      where: o.inserted_at < ^time_deadline,
+      where:
+        fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host())
     )
+    |> Repo.delete_all()
 
     if Keyword.get(options, :vacuum) do
       Logger.info("Runnning VACUUM FULL")

From 045803346d70c1f9c6ea770485904fd7cc52969a Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Wed, 22 May 2019 03:58:15 +0000
Subject: [PATCH 24/64] move key generation functions into Pleroma.Keys module

---
 lib/pleroma/keys.ex                           | 44 +++++++++++++++++++
 lib/pleroma/signature.ex                      |  7 ++-
 lib/pleroma/user.ex                           | 21 +++++++++
 .../activity_pub/activity_pub_controller.ex   | 14 +++---
 .../web/activity_pub/views/user_view.ex       | 11 +++--
 lib/pleroma/web/federator/federator.ex        |  6 +--
 lib/pleroma/web/salmon/salmon.ex              | 44 ++-----------------
 lib/pleroma/web/web_finger/web_finger.ex      | 26 +----------
 test/keys_test.exs                            | 20 +++++++++
 test/user_test.exs                            | 15 +++++++
 test/web/activity_pub/activity_pub_test.exs   |  2 +-
 .../web/activity_pub/views/user_view_test.exs | 13 +++---
 test/web/salmon/salmon_test.exs               | 19 ++------
 test/web/web_finger/web_finger_test.exs       | 15 -------
 14 files changed, 133 insertions(+), 124 deletions(-)
 create mode 100644 lib/pleroma/keys.ex
 create mode 100644 test/keys_test.exs

diff --git a/lib/pleroma/keys.ex b/lib/pleroma/keys.ex
new file mode 100644
index 000000000..b7bc7a4da
--- /dev/null
+++ b/lib/pleroma/keys.ex
@@ -0,0 +1,44 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Keys do
+  # Native generation of RSA keys is only available since OTP 20+ and in default build conditions
+  # We try at compile time to generate natively an RSA key otherwise we fallback on the old way.
+  try do
+    _ = :public_key.generate_key({:rsa, 2048, 65_537})
+
+    def generate_rsa_pem do
+      key = :public_key.generate_key({:rsa, 2048, 65_537})
+      entry = :public_key.pem_entry_encode(:RSAPrivateKey, key)
+      pem = :public_key.pem_encode([entry]) |> String.trim_trailing()
+      {:ok, pem}
+    end
+  rescue
+    _ ->
+      def generate_rsa_pem do
+        port = Port.open({:spawn, "openssl genrsa"}, [:binary])
+
+        {:ok, pem} =
+          receive do
+            {^port, {:data, pem}} -> {:ok, pem}
+          end
+
+        Port.close(port)
+
+        if Regex.match?(~r/RSA PRIVATE KEY/, pem) do
+          {:ok, pem}
+        else
+          :error
+        end
+      end
+  end
+
+  def keys_from_pem(pem) do
+    [private_key_code] = :public_key.pem_decode(pem)
+    private_key = :public_key.pem_entry_decode(private_key_code)
+    {:RSAPrivateKey, _, modulus, exponent, _, _, _, _, _, _, _} = private_key
+    public_key = {:RSAPublicKey, modulus, exponent}
+    {:ok, private_key, public_key}
+  end
+end
diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex
index b7ecf00a0..1a4d54c62 100644
--- a/lib/pleroma/signature.ex
+++ b/lib/pleroma/signature.ex
@@ -5,11 +5,10 @@
 defmodule Pleroma.Signature do
   @behaviour HTTPSignatures.Adapter
 
+  alias Pleroma.Keys
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Utils
-  alias Pleroma.Web.Salmon
-  alias Pleroma.Web.WebFinger
 
   def fetch_public_key(conn) do
     with actor_id <- Utils.get_ap_id(conn.params["actor"]),
@@ -33,8 +32,8 @@ def refetch_public_key(conn) do
   end
 
   def sign(%User{} = user, headers) do
-    with {:ok, %{info: %{keys: keys}}} <- WebFinger.ensure_keys_present(user),
-         {:ok, private_key, _} <- Salmon.keys_from_pem(keys) do
+    with {:ok, %{info: %{keys: keys}}} <- User.ensure_keys_present(user),
+         {:ok, private_key, _} <- Keys.keys_from_pem(keys) do
       HTTPSignatures.sign(private_key, user.ap_id <> "#main-key", headers)
     end
   end
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 05fe58f7c..653dec95f 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -10,6 +10,7 @@ defmodule Pleroma.User do
 
   alias Comeonin.Pbkdf2
   alias Pleroma.Activity
+  alias Pleroma.Keys
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Registration
@@ -1422,4 +1423,24 @@ def get_mascot(%{info: %{mascot: mascot}}) when is_nil(mascot) do
       }
     }
   end
+
+  def ensure_keys_present(user) do
+    info = user.info
+
+    if info.keys do
+      {:ok, user}
+    else
+      {:ok, pem} = Keys.generate_rsa_pem()
+
+      info_cng =
+        info
+        |> User.Info.set_keys(pem)
+
+      cng =
+        Ecto.Changeset.change(user)
+        |> Ecto.Changeset.put_embed(:info, info_cng)
+
+      update_and_set_cache(cng)
+    end
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index c967ab7a9..ad2ca1e54 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -39,7 +39,7 @@ def relay_active?(conn, _) do
 
   def user(conn, %{"nickname" => nickname}) do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
+         {:ok, user} <- User.ensure_keys_present(user) do
       conn
       |> put_resp_header("content-type", "application/activity+json")
       |> json(UserView.render("user.json", %{user: user}))
@@ -106,7 +106,7 @@ def activity(conn, %{"uuid" => uuid}) do
 
   def following(conn, %{"nickname" => nickname, "page" => page}) do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
+         {:ok, user} <- User.ensure_keys_present(user) do
       {page, _} = Integer.parse(page)
 
       conn
@@ -117,7 +117,7 @@ def following(conn, %{"nickname" => nickname, "page" => page}) do
 
   def following(conn, %{"nickname" => nickname}) do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
+         {:ok, user} <- User.ensure_keys_present(user) do
       conn
       |> put_resp_header("content-type", "application/activity+json")
       |> json(UserView.render("following.json", %{user: user}))
@@ -126,7 +126,7 @@ def following(conn, %{"nickname" => nickname}) do
 
   def followers(conn, %{"nickname" => nickname, "page" => page}) do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
+         {:ok, user} <- User.ensure_keys_present(user) do
       {page, _} = Integer.parse(page)
 
       conn
@@ -137,7 +137,7 @@ def followers(conn, %{"nickname" => nickname, "page" => page}) do
 
   def followers(conn, %{"nickname" => nickname}) do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
+         {:ok, user} <- User.ensure_keys_present(user) do
       conn
       |> put_resp_header("content-type", "application/activity+json")
       |> json(UserView.render("followers.json", %{user: user}))
@@ -146,7 +146,7 @@ def followers(conn, %{"nickname" => nickname}) do
 
   def outbox(conn, %{"nickname" => nickname} = params) do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
+         {:ok, user} <- User.ensure_keys_present(user) do
       conn
       |> put_resp_header("content-type", "application/activity+json")
       |> json(UserView.render("outbox.json", %{user: user, max_id: params["max_id"]}))
@@ -195,7 +195,7 @@ def inbox(conn, params) do
 
   def relay(conn, _params) do
     with %User{} = user <- Relay.get_actor(),
-         {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
+         {:ok, user} <- User.ensure_keys_present(user) do
       conn
       |> put_resp_header("content-type", "application/activity+json")
       |> json(UserView.render("user.json", %{user: user}))
diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex
index 1254fdf6c..327e0e05b 100644
--- a/lib/pleroma/web/activity_pub/views/user_view.ex
+++ b/lib/pleroma/web/activity_pub/views/user_view.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.ActivityPub.UserView do
   use Pleroma.Web, :view
 
+  alias Pleroma.Keys
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -12,8 +13,6 @@ defmodule Pleroma.Web.ActivityPub.UserView do
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.Endpoint
   alias Pleroma.Web.Router.Helpers
-  alias Pleroma.Web.Salmon
-  alias Pleroma.Web.WebFinger
 
   import Ecto.Query
 
@@ -34,8 +33,8 @@ def render("endpoints.json", _), do: %{}
 
   # the instance itself is not a Person, but instead an Application
   def render("user.json", %{user: %{nickname: nil} = user}) do
-    {:ok, user} = WebFinger.ensure_keys_present(user)
-    {:ok, _, public_key} = Salmon.keys_from_pem(user.info.keys)
+    {:ok, user} = User.ensure_keys_present(user)
+    {:ok, _, public_key} = Keys.keys_from_pem(user.info.keys)
     public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key)
     public_key = :public_key.pem_encode([public_key])
 
@@ -62,8 +61,8 @@ def render("user.json", %{user: %{nickname: nil} = user}) do
   end
 
   def render("user.json", %{user: user}) do
-    {:ok, user} = WebFinger.ensure_keys_present(user)
-    {:ok, _, public_key} = Salmon.keys_from_pem(user.info.keys)
+    {:ok, user} = User.ensure_keys_present(user)
+    {:ok, _, public_key} = Keys.keys_from_pem(user.info.keys)
     public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key)
     public_key = :public_key.pem_encode([public_key])
 
diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex
index 169fdf4dc..6b0b75284 100644
--- a/lib/pleroma/web/federator/federator.ex
+++ b/lib/pleroma/web/federator/federator.ex
@@ -11,7 +11,6 @@ defmodule Pleroma.Web.Federator do
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.Federator.Publisher
   alias Pleroma.Web.Federator.RetryQueue
-  alias Pleroma.Web.WebFinger
   alias Pleroma.Web.Websub
 
   require Logger
@@ -77,9 +76,8 @@ def perform(:request_subscription, websub) do
   def perform(:publish, activity) do
     Logger.debug(fn -> "Running publish for #{activity.data["id"]}" end)
 
-    with actor when not is_nil(actor) <- User.get_cached_by_ap_id(activity.data["actor"]) do
-      {:ok, actor} = WebFinger.ensure_keys_present(actor)
-
+    with %User{} = actor <- User.get_cached_by_ap_id(activity.data["actor"]),
+         {:ok, actor} <- User.ensure_keys_present(actor) do
       Publisher.publish(actor, activity)
     end
   end
diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index 42709ab47..fa30f73cd 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -10,6 +10,7 @@ defmodule Pleroma.Web.Salmon do
   use Bitwise
 
   alias Pleroma.Activity
+  alias Pleroma.Keys
   alias Pleroma.Instances
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Visibility
@@ -89,45 +90,6 @@ def encode_key({:RSAPublicKey, modulus, exponent}) do
     "RSA.#{modulus_enc}.#{exponent_enc}"
   end
 
-  # Native generation of RSA keys is only available since OTP 20+ and in default build conditions
-  # We try at compile time to generate natively an RSA key otherwise we fallback on the old way.
-  try do
-    _ = :public_key.generate_key({:rsa, 2048, 65_537})
-
-    def generate_rsa_pem do
-      key = :public_key.generate_key({:rsa, 2048, 65_537})
-      entry = :public_key.pem_entry_encode(:RSAPrivateKey, key)
-      pem = :public_key.pem_encode([entry]) |> String.trim_trailing()
-      {:ok, pem}
-    end
-  rescue
-    _ ->
-      def generate_rsa_pem do
-        port = Port.open({:spawn, "openssl genrsa"}, [:binary])
-
-        {:ok, pem} =
-          receive do
-            {^port, {:data, pem}} -> {:ok, pem}
-          end
-
-        Port.close(port)
-
-        if Regex.match?(~r/RSA PRIVATE KEY/, pem) do
-          {:ok, pem}
-        else
-          :error
-        end
-      end
-  end
-
-  def keys_from_pem(pem) do
-    [private_key_code] = :public_key.pem_decode(pem)
-    private_key = :public_key.pem_entry_decode(private_key_code)
-    {:RSAPrivateKey, _, modulus, exponent, _, _, _, _, _, _, _} = private_key
-    public_key = {:RSAPublicKey, modulus, exponent}
-    {:ok, private_key, public_key}
-  end
-
   def encode(private_key, doc) do
     type = "application/atom+xml"
     encoding = "base64url"
@@ -227,7 +189,7 @@ def publish(%{info: %{keys: keys}} = user, %{data: %{"type" => type}} = activity
         |> :xmerl.export_simple(:xmerl_xml)
         |> to_string
 
-      {:ok, private, _} = keys_from_pem(keys)
+      {:ok, private, _} = Keys.keys_from_pem(keys)
       {:ok, feed} = encode(private, feed)
 
       remote_users = remote_users(activity)
@@ -253,7 +215,7 @@ def publish(%{info: %{keys: keys}} = user, %{data: %{"type" => type}} = activity
   def publish(%{id: id}, _), do: Logger.debug(fn -> "Keys missing for user #{id}" end)
 
   def gather_webfinger_links(%User{} = user) do
-    {:ok, _private, public} = keys_from_pem(user.info.keys)
+    {:ok, _private, public} = Keys.keys_from_pem(user.info.keys)
     magic_key = encode_key(public)
 
     [
diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex
index 1239b962a..c5b7d4acb 100644
--- a/lib/pleroma/web/web_finger/web_finger.ex
+++ b/lib/pleroma/web/web_finger/web_finger.ex
@@ -8,7 +8,6 @@ defmodule Pleroma.Web.WebFinger do
   alias Pleroma.User
   alias Pleroma.Web
   alias Pleroma.Web.Federator.Publisher
-  alias Pleroma.Web.Salmon
   alias Pleroma.Web.XML
   alias Pleroma.XmlBuilder
   require Jason
@@ -61,7 +60,7 @@ defp gather_links(%User{} = user) do
   end
 
   def represent_user(user, "JSON") do
-    {:ok, user} = ensure_keys_present(user)
+    {:ok, user} = User.ensure_keys_present(user)
 
     %{
       "subject" => "acct:#{user.nickname}@#{Pleroma.Web.Endpoint.host()}",
@@ -71,7 +70,7 @@ def represent_user(user, "JSON") do
   end
 
   def represent_user(user, "XML") do
-    {:ok, user} = ensure_keys_present(user)
+    {:ok, user} = User.ensure_keys_present(user)
 
     links =
       gather_links(user)
@@ -88,27 +87,6 @@ def represent_user(user, "XML") do
     |> XmlBuilder.to_doc()
   end
 
-  # This seems a better fit in Salmon
-  def ensure_keys_present(user) do
-    info = user.info
-
-    if info.keys do
-      {:ok, user}
-    else
-      {:ok, pem} = Salmon.generate_rsa_pem()
-
-      info_cng =
-        info
-        |> User.Info.set_keys(pem)
-
-      cng =
-        Ecto.Changeset.change(user)
-        |> Ecto.Changeset.put_embed(:info, info_cng)
-
-      User.update_and_set_cache(cng)
-    end
-  end
-
   defp get_magic_key(magic_key) do
     "data:application/magic-public-key," <> magic_key = magic_key
     {:ok, magic_key}
diff --git a/test/keys_test.exs b/test/keys_test.exs
new file mode 100644
index 000000000..776fdea6f
--- /dev/null
+++ b/test/keys_test.exs
@@ -0,0 +1,20 @@
+defmodule Pleroma.KeysTest do
+  use Pleroma.DataCase
+
+  alias Pleroma.Keys
+
+  test "generates an RSA private key pem" do
+    {:ok, key} = Keys.generate_rsa_pem()
+
+    assert is_binary(key)
+    assert Regex.match?(~r/RSA/, key)
+  end
+
+  test "returns a public and private key from a pem" do
+    pem = File.read!("test/fixtures/private_key.pem")
+    {:ok, private, public} = Keys.keys_from_pem(pem)
+
+    assert elem(private, 0) == :RSAPrivateKey
+    assert elem(public, 0) == :RSAPublicKey
+  end
+end
diff --git a/test/user_test.exs b/test/user_test.exs
index cb6afbe07..019f2b56d 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -1251,4 +1251,19 @@ test "if user is unconfirmed" do
       refute user.info.confirmation_token
     end
   end
+
+  describe "ensure_keys_present" do
+    test "it creates keys for a user and stores them in info" do
+      user = insert(:user)
+      refute is_binary(user.info.keys)
+      {:ok, user} = User.ensure_keys_present(user)
+      assert is_binary(user.info.keys)
+    end
+
+    test "it doesn't create keys if there already are some" do
+      user = insert(:user, %{info: %{keys: "xxx"}})
+      {:ok, user} = User.ensure_keys_present(user)
+      assert user.info.keys == "xxx"
+    end
+  end
 end
diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs
index c18e0ab5f..f743f380b 100644
--- a/test/web/activity_pub/activity_pub_test.exs
+++ b/test/web/activity_pub/activity_pub_test.exs
@@ -1005,7 +1005,7 @@ test "it filters broken threads" do
   describe "update" do
     test "it creates an update activity with the new user data" do
       user = insert(:user)
-      {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+      {:ok, user} = User.ensure_keys_present(user)
       user_data = Pleroma.Web.ActivityPub.UserView.render("user.json", %{user: user})
 
       {:ok, update} =
diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs
index 9fb9455d2..e6483db8b 100644
--- a/test/web/activity_pub/views/user_view_test.exs
+++ b/test/web/activity_pub/views/user_view_test.exs
@@ -2,11 +2,12 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do
   use Pleroma.DataCase
   import Pleroma.Factory
 
+  alias Pleroma.User
   alias Pleroma.Web.ActivityPub.UserView
 
   test "Renders a user, including the public key" do
     user = insert(:user)
-    {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+    {:ok, user} = User.ensure_keys_present(user)
 
     result = UserView.render("user.json", %{user: user})
 
@@ -18,7 +19,7 @@ test "Renders a user, including the public key" do
 
   test "Does not add an avatar image if the user hasn't set one" do
     user = insert(:user)
-    {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+    {:ok, user} = User.ensure_keys_present(user)
 
     result = UserView.render("user.json", %{user: user})
     refute result["icon"]
@@ -32,7 +33,7 @@ test "Does not add an avatar image if the user hasn't set one" do
         }
       )
 
-    {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+    {:ok, user} = User.ensure_keys_present(user)
 
     result = UserView.render("user.json", %{user: user})
     assert result["icon"]["url"] == "https://someurl"
@@ -42,7 +43,7 @@ test "Does not add an avatar image if the user hasn't set one" do
   describe "endpoints" do
     test "local users have a usable endpoints structure" do
       user = insert(:user)
-      {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+      {:ok, user} = User.ensure_keys_present(user)
 
       result = UserView.render("user.json", %{user: user})
 
@@ -58,7 +59,7 @@ test "local users have a usable endpoints structure" do
 
     test "remote users have an empty endpoints structure" do
       user = insert(:user, local: false)
-      {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+      {:ok, user} = User.ensure_keys_present(user)
 
       result = UserView.render("user.json", %{user: user})
 
@@ -68,7 +69,7 @@ test "remote users have an empty endpoints structure" do
 
     test "instance users do not expose oAuth endpoints" do
       user = insert(:user, nickname: nil, local: true)
-      {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+      {:ok, user} = User.ensure_keys_present(user)
 
       result = UserView.render("user.json", %{user: user})
 
diff --git a/test/web/salmon/salmon_test.exs b/test/web/salmon/salmon_test.exs
index 232082779..e86e76fe9 100644
--- a/test/web/salmon/salmon_test.exs
+++ b/test/web/salmon/salmon_test.exs
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.Salmon.SalmonTest do
   use Pleroma.DataCase
   alias Pleroma.Activity
+  alias Pleroma.Keys
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.Federator.Publisher
@@ -34,12 +35,6 @@ test "errors on wrong magic key" do
     assert Salmon.decode_and_validate(@wrong_magickey, salmon) == :error
   end
 
-  test "generates an RSA private key pem" do
-    {:ok, key} = Salmon.generate_rsa_pem()
-    assert is_binary(key)
-    assert Regex.match?(~r/RSA/, key)
-  end
-
   test "it encodes a magic key from a public key" do
     key = Salmon.decode_key(@magickey)
     magic_key = Salmon.encode_key(key)
@@ -51,18 +46,10 @@ test "it decodes a friendica public key" do
     _key = Salmon.decode_key(@magickey_friendica)
   end
 
-  test "returns a public and private key from a pem" do
-    pem = File.read!("test/fixtures/private_key.pem")
-    {:ok, private, public} = Salmon.keys_from_pem(pem)
-
-    assert elem(private, 0) == :RSAPrivateKey
-    assert elem(public, 0) == :RSAPublicKey
-  end
-
   test "encodes an xml payload with a private key" do
     doc = File.read!("test/fixtures/incoming_note_activity.xml")
     pem = File.read!("test/fixtures/private_key.pem")
-    {:ok, private, public} = Salmon.keys_from_pem(pem)
+    {:ok, private, public} = Keys.keys_from_pem(pem)
 
     # Let's try a roundtrip.
     {:ok, salmon} = Salmon.encode(private, doc)
@@ -105,7 +92,7 @@ test "it gets a magic key" do
 
     {:ok, activity} = Repo.insert(%Activity{data: activity_data, recipients: activity_data["to"]})
     user = User.get_cached_by_ap_id(activity.data["actor"])
-    {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+    {:ok, user} = User.ensure_keys_present(user)
 
     Salmon.publish(user, activity)
 
diff --git a/test/web/web_finger/web_finger_test.exs b/test/web/web_finger/web_finger_test.exs
index 6b20d8d56..335c95b18 100644
--- a/test/web/web_finger/web_finger_test.exs
+++ b/test/web/web_finger/web_finger_test.exs
@@ -105,19 +105,4 @@ test "it gets the xrd endpoint for statusnet" do
       assert template == "http://status.alpicola.com/main/xrd?uri={uri}"
     end
   end
-
-  describe "ensure_keys_present" do
-    test "it creates keys for a user and stores them in info" do
-      user = insert(:user)
-      refute is_binary(user.info.keys)
-      {:ok, user} = WebFinger.ensure_keys_present(user)
-      assert is_binary(user.info.keys)
-    end
-
-    test "it doesn't create keys if there already are some" do
-      user = insert(:user, %{info: %{keys: "xxx"}})
-      {:ok, user} = WebFinger.ensure_keys_present(user)
-      assert user.info.keys == "xxx"
-    end
-  end
 end

From 913484817076bf5ca4bdbe7c3c1ff34f7debd3e5 Mon Sep 17 00:00:00 2001
From: Sergey Suprunenko <suprunenko.s@gmail.com>
Date: Wed, 22 May 2019 04:04:20 +0000
Subject: [PATCH 25/64] Do not truncate DM when it contains newlines and
 safe_dm_mentions is set to true

---
 lib/pleroma/formatter.ex | 2 +-
 test/formatter_test.exs  | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex
index 3d7c36d21..3e3b9fe97 100644
--- a/lib/pleroma/formatter.ex
+++ b/lib/pleroma/formatter.ex
@@ -8,7 +8,7 @@ defmodule Pleroma.Formatter do
   alias Pleroma.User
   alias Pleroma.Web.MediaProxy
 
-  @safe_mention_regex ~r/^(\s*(?<mentions>@.+?\s+)+)(?<rest>.*)/
+  @safe_mention_regex ~r/^(\s*(?<mentions>@.+?\s+)+)(?<rest>.*)/s
   @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui
   @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/
 
diff --git a/test/formatter_test.exs b/test/formatter_test.exs
index 5e7011160..47b91b121 100644
--- a/test/formatter_test.exs
+++ b/test/formatter_test.exs
@@ -206,6 +206,15 @@ test "given the 'safe_mention' option, it will still work without any mention" d
       assert mentions == []
       assert expected_text == text
     end
+
+    test "given the 'safe_mention' option, it will keep text after newlines" do
+      user = insert(:user)
+      text = " @#{user.nickname}\n hey dude\n\nhow are you doing?"
+
+      {expected_text, _, _} = Formatter.linkify(text, safe_mention: true)
+
+      assert expected_text =~ "how are you doing?"
+    end
   end
 
   describe ".parse_tags" do

From 0e2c215a006c7ca5756e80a357ff6395a4325946 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Wed, 22 May 2019 07:22:19 +0200
Subject: [PATCH 26/64] MastoAPI AccountView: fill source.note with plaintext
 version of note

Closes: https://git.pleroma.social/pleroma/pleroma/issues/926
---
 lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +-
 test/web/mastodon_api/account_view_test.exs        | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 134c07b7e..b82d3319b 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -112,7 +112,7 @@ defp do_render("account.json", %{user: user} = opts) do
       fields: fields,
       bot: bot,
       source: %{
-        note: "",
+        note: HTML.strip_tags((user.bio || "") |> String.replace("<br>", "\n")),
         sensitive: false,
         pleroma: %{}
       },
diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs
index a24f2a050..aaf2261bb 100644
--- a/test/web/mastodon_api/account_view_test.exs
+++ b/test/web/mastodon_api/account_view_test.exs
@@ -55,7 +55,7 @@ test "Represent a user account" do
       fields: [],
       bot: false,
       source: %{
-        note: "",
+        note: "valid html",
         sensitive: false,
         pleroma: %{}
       },
@@ -120,7 +120,7 @@ test "Represent a Service(bot) account" do
       fields: [],
       bot: true,
       source: %{
-        note: "",
+        note: user.bio,
         sensitive: false,
         pleroma: %{}
       },
@@ -209,7 +209,7 @@ test "represent an embedded relationship" do
       fields: [],
       bot: true,
       source: %{
-        note: "",
+        note: user.bio,
         sensitive: false,
         pleroma: %{}
       },

From 1344c85e2fe636fd6b9d033eb7add1c3a9701c7f Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Wed, 22 May 2019 05:58:51 +0000
Subject: [PATCH 27/64] salmon: fix credo

---
 lib/pleroma/web/salmon/salmon.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index fa30f73cd..f25d92fad 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -10,8 +10,8 @@ defmodule Pleroma.Web.Salmon do
   use Bitwise
 
   alias Pleroma.Activity
-  alias Pleroma.Keys
   alias Pleroma.Instances
+  alias Pleroma.Keys
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.Federator.Publisher

From f9e0d09ec0082a096dcd4980bc5ffebe8e3139ae Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 22 May 2019 10:17:32 +0200
Subject: [PATCH 28/64] Changelog: Add SSH mode.

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3ff70e6e5..3d1e7640d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [unreleased]
 ### Added
+- Optional SSH access mode.
 - [MongooseIM](https://github.com/esl/MongooseIM) http authentication support.
 - LDAP authentication
 - External OAuth provider authentication

From b6cf62ddeab04db6bd2695c5537c81e0fb1aecaf Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 22 May 2019 10:28:50 +0200
Subject: [PATCH 29/64] Mix: Don't start esshd application if we don't need it.

---
 mix.exs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/mix.exs b/mix.exs
index 95c052c34..bc5b6204f 100644
--- a/mix.exs
+++ b/mix.exs
@@ -40,9 +40,16 @@ def project do
   #
   # Type `mix help compile.app` for more information.
   def application do
+    extra_applications = [:logger, :runtime_tools, :comeonin, :quack]
+    extra_applications = if Application.get_env(:esshd, :enabled, false) do
+      [:esshd | extra_applications]
+    else
+      extra_applications
+    end
+
     [
       mod: {Pleroma.Application, []},
-      extra_applications: [:logger, :runtime_tools, :comeonin, :esshd, :quack],
+      extra_applications: extra_applications,
       included_applications: [:ex_syslogger]
     ]
   end

From db9a82d168cfc452611a44d92df2b81a5e6d1e69 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 22 May 2019 10:40:15 +0200
Subject: [PATCH 30/64] Linting.

---
 mix.exs | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/mix.exs b/mix.exs
index bc5b6204f..25ec46a46 100644
--- a/mix.exs
+++ b/mix.exs
@@ -41,11 +41,13 @@ def project do
   # Type `mix help compile.app` for more information.
   def application do
     extra_applications = [:logger, :runtime_tools, :comeonin, :quack]
-    extra_applications = if Application.get_env(:esshd, :enabled, false) do
-      [:esshd | extra_applications]
-    else
-      extra_applications
-    end
+
+    extra_applications =
+      if Application.get_env(:esshd, :enabled, false) do
+        [:esshd | extra_applications]
+      else
+        extra_applications
+      end
 
     [
       mod: {Pleroma.Application, []},

From b22145cbc40b57cf83f6389f063a76a03625ff16 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 22 May 2019 10:44:26 +0200
Subject: [PATCH 31/64] Documentation: Specify PEM format for SSH keys.

Otherwise openssh-client 7.9 will generate a different format that can't be used
by esshd.
---
 docs/config.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/config.md b/docs/config.md
index 197326bbd..63ca61d1e 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -477,7 +477,7 @@ config :esshd,
   password_authenticator: "Pleroma.BBS.Authenticator"
 ```
 
-Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT`
+Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -m PEM -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT`
 
 ## :auth
 

From 3b12e1ba7c99382c678ce17629352135f44dcb9f Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 22 May 2019 11:01:10 +0200
Subject: [PATCH 32/64] Changelog: Add tip for debian users.

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3d1e7640d..b88edd072 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [unreleased]
 ### Added
-- Optional SSH access mode.
+- Optional SSH access mode. (Needs `erlang-ssh` package on Debian).
 - [MongooseIM](https://github.com/esl/MongooseIM) http authentication support.
 - LDAP authentication
 - External OAuth provider authentication

From f4cfcead8868481c19ebd93b0fcd2b942dc0e477 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 22 May 2019 11:44:17 +0200
Subject: [PATCH 33/64] Mix: Bring ecto-sql back to mainline.

---
 mix.exs  | 5 +----
 mix.lock | 2 +-
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/mix.exs b/mix.exs
index 95c052c34..9149b241f 100644
--- a/mix.exs
+++ b/mix.exs
@@ -66,10 +66,7 @@ defp deps do
       {:plug_cowboy, "~> 2.0"},
       {:phoenix_pubsub, "~> 1.1"},
       {:phoenix_ecto, "~> 4.0"},
-      {:ecto_sql,
-       git: "https://github.com/elixir-ecto/ecto_sql",
-       ref: "14cb065a74c488d737d973f7a91bc036c6245f78",
-       override: true},
+      {:ecto_sql, "~> 3.1"},
       {:postgrex, ">= 0.13.5"},
       {:gettext, "~> 0.15"},
       {:comeonin, "~> 4.1.1"},
diff --git a/mix.lock b/mix.lock
index bacc09787..857bfca79 100644
--- a/mix.lock
+++ b/mix.lock
@@ -21,7 +21,7 @@
   "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"},
   "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"},
   "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
-  "ecto_sql": {:git, "https://github.com/elixir-ecto/ecto_sql", "14cb065a74c488d737d973f7a91bc036c6245f78", [ref: "14cb065a74c488d737d973f7a91bc036c6245f78"]},
+  "ecto_sql": {:hex, :ecto_sql, "3.1.3", "2c536139190492d9de33c5fefac7323c5eaaa82e1b9bf93482a14649042f7cd9", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"},
   "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"},
   "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"},
   "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"},

From f323031927ecaf155e661b17cc9b96333fb9e4ad Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 22 May 2019 12:57:20 +0200
Subject: [PATCH 34/64] Mix: Only start sshd when needed, second try.

---
 mix.exs | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/mix.exs b/mix.exs
index 25ec46a46..b2c075c85 100644
--- a/mix.exs
+++ b/mix.exs
@@ -40,18 +40,9 @@ def project do
   #
   # Type `mix help compile.app` for more information.
   def application do
-    extra_applications = [:logger, :runtime_tools, :comeonin, :quack]
-
-    extra_applications =
-      if Application.get_env(:esshd, :enabled, false) do
-        [:esshd | extra_applications]
-      else
-        extra_applications
-      end
-
     [
       mod: {Pleroma.Application, []},
-      extra_applications: extra_applications,
+      extra_applications: [:logger, :runtime_tools, :comeonin, :quack],
       included_applications: [:ex_syslogger]
     ]
   end
@@ -129,7 +120,7 @@ defp deps do
       {:recon, github: "ferd/recon", tag: "2.4.0"},
       {:quack, "~> 0.1.1"},
       {:benchee, "~> 1.0"},
-      {:esshd, "~> 0.1.0"},
+      {:esshd, "~> 0.1.0", runtime: Application.get_env(:esshd, :enabled, false)},
       {:ex_rated, "~> 1.2"},
       {:plug_static_index_html, "~> 1.0.0"},
       {:excoveralls, "~> 0.11.1", only: :test}

From 78ac8ee56139ed98625c54ce627eb37047a361f0 Mon Sep 17 00:00:00 2001
From: lambda <lain@soykaf.club>
Date: Wed, 22 May 2019 11:07:51 +0000
Subject: [PATCH 35/64] Update CHANGELOG.md

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b88edd072..bb2306fc4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [unreleased]
 ### Added
-- Optional SSH access mode. (Needs `erlang-ssh` package on Debian).
+- Optional SSH access mode. (Needs `erlang-ssh` package on some distributions).
 - [MongooseIM](https://github.com/esl/MongooseIM) http authentication support.
 - LDAP authentication
 - External OAuth provider authentication

From 620908a2db86942a00bc0ba9c71c037061e26967 Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Wed, 22 May 2019 15:44:50 +0000
Subject: [PATCH 36/64] [#699] add worker to clean expired oauth tokens

---
 CHANGELOG.md                                |  1 +
 config/config.exs                           |  4 +-
 docs/config.md                              |  2 +
 lib/pleroma/application.ex                  |  1 +
 lib/pleroma/web/oauth/token.ex              | 36 ++++++--------
 lib/pleroma/web/oauth/token/clean_worker.ex | 41 +++++++++++++++
 lib/pleroma/web/oauth/token/query.ex        | 55 +++++++++++++++++++++
 test/web/oauth/token_test.exs               | 13 +++++
 8 files changed, 132 insertions(+), 21 deletions(-)
 create mode 100644 lib/pleroma/web/oauth/token/clean_worker.ex
 create mode 100644 lib/pleroma/web/oauth/token/query.ex

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b5c42d1fd..02d64a850 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -41,6 +41,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - OAuth: added support for refresh tokens
 - Emoji packs and emoji pack manager
 - Object pruning (`mix pleroma.database prune_objects`)
+- OAuth: added job to clean expired access tokens
 
 ### Changed
 - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer
diff --git a/config/config.exs b/config/config.exs
index a05f8b1d2..33b7e713d 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -481,7 +481,9 @@
 
 config :pleroma, :oauth2,
   token_expires_in: 600,
-  issue_new_refresh_token: true
+  issue_new_refresh_token: true,
+  clean_expired_tokens: false,
+  clean_expired_tokens_interval: 86_400_000
 
 config :pleroma, :database, rum_enabled: false
 
diff --git a/docs/config.md b/docs/config.md
index a050068f4..264b65499 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -550,6 +550,8 @@ Configure OAuth 2 provider capabilities:
 
 * `token_expires_in` - The lifetime in seconds of the access token.
 * `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token.
+* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`.
+* `clean_expired_tokens_interval` - Interval to run the job to clean expired tokens. Defaults to `86_400_000` (24 hours).
 
 ## :emoji
 * `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]`
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index dab45a0b2..76df3945e 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -110,6 +110,7 @@ def start(_type, _args) do
         hackney_pool_children() ++
         [
           worker(Pleroma.Web.Federator.RetryQueue, []),
+          worker(Pleroma.Web.OAuth.Token.CleanWorker, []),
           worker(Pleroma.Stats, []),
           worker(Task, [&Pleroma.Web.Push.init/0], restart: :temporary, id: :web_push_init),
           worker(Task, [&Pleroma.Web.Federator.init/0], restart: :temporary, id: :federator_init)
diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex
index 66c95c2e9..f412f7eb2 100644
--- a/lib/pleroma/web/oauth/token.ex
+++ b/lib/pleroma/web/oauth/token.ex
@@ -5,7 +5,6 @@
 defmodule Pleroma.Web.OAuth.Token do
   use Ecto.Schema
 
-  import Ecto.Query
   import Ecto.Changeset
 
   alias Pleroma.Repo
@@ -13,6 +12,7 @@ defmodule Pleroma.Web.OAuth.Token do
   alias Pleroma.Web.OAuth.App
   alias Pleroma.Web.OAuth.Authorization
   alias Pleroma.Web.OAuth.Token
+  alias Pleroma.Web.OAuth.Token.Query
 
   @expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600)
   @type t :: %__MODULE__{}
@@ -31,17 +31,17 @@ defmodule Pleroma.Web.OAuth.Token do
   @doc "Gets token for app by access token"
   @spec get_by_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found}
   def get_by_token(%App{id: app_id} = _app, token) do
-    from(t in __MODULE__, where: t.app_id == ^app_id and t.token == ^token)
+    Query.get_by_app(app_id)
+    |> Query.get_by_token(token)
     |> Repo.find_resource()
   end
 
   @doc "Gets token for app by refresh token"
   @spec get_by_refresh_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found}
   def get_by_refresh_token(%App{id: app_id} = _app, token) do
-    from(t in __MODULE__,
-      where: t.app_id == ^app_id and t.refresh_token == ^token,
-      preload: [:user]
-    )
+    Query.get_by_app(app_id)
+    |> Query.get_by_refresh_token(token)
+    |> Query.preload([:user])
     |> Repo.find_resource()
   end
 
@@ -97,29 +97,25 @@ def create_token(%App{} = app, %User{} = user, attrs \\ %{}) do
   end
 
   def delete_user_tokens(%User{id: user_id}) do
-    from(
-      t in Token,
-      where: t.user_id == ^user_id
-    )
+    Query.get_by_user(user_id)
     |> Repo.delete_all()
   end
 
   def delete_user_token(%User{id: user_id}, token_id) do
-    from(
-      t in Token,
-      where: t.user_id == ^user_id,
-      where: t.id == ^token_id
-    )
+    Query.get_by_user(user_id)
+    |> Query.get_by_id(token_id)
+    |> Repo.delete_all()
+  end
+
+  def delete_expired_tokens do
+    Query.get_expired_tokens()
     |> Repo.delete_all()
   end
 
   def get_user_tokens(%User{id: user_id}) do
-    from(
-      t in Token,
-      where: t.user_id == ^user_id
-    )
+    Query.get_by_user(user_id)
+    |> Query.preload([:app])
     |> Repo.all()
-    |> Repo.preload(:app)
   end
 
   def is_expired?(%__MODULE__{valid_until: valid_until}) do
diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex
new file mode 100644
index 000000000..dca852449
--- /dev/null
+++ b/lib/pleroma/web/oauth/token/clean_worker.ex
@@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.OAuth.Token.CleanWorker do
+  @moduledoc """
+  The module represents functions to clean an expired oauth tokens.
+  """
+
+  # 10 seconds
+  @start_interval 10_000
+  @interval Pleroma.Config.get(
+              # 24 hours
+              [:oauth2, :clean_expired_tokens_interval],
+              86_400_000
+            )
+  @queue :background
+
+  alias Pleroma.Web.OAuth.Token
+
+  def start_link, do: GenServer.start_link(__MODULE__, nil)
+
+  def init(_) do
+    if Pleroma.Config.get([:oauth2, :clean_expired_tokens], false) do
+      Process.send_after(self(), :perform, @start_interval)
+      {:ok, nil}
+    else
+      :ignore
+    end
+  end
+
+  @doc false
+  def handle_info(:perform, state) do
+    Process.send_after(self(), :perform, @interval)
+    PleromaJobQueue.enqueue(@queue, __MODULE__, [:clean])
+    {:noreply, state}
+  end
+
+  # Job Worker Callbacks
+  def perform(:clean), do: Token.delete_expired_tokens()
+end
diff --git a/lib/pleroma/web/oauth/token/query.ex b/lib/pleroma/web/oauth/token/query.ex
new file mode 100644
index 000000000..d92e1f071
--- /dev/null
+++ b/lib/pleroma/web/oauth/token/query.ex
@@ -0,0 +1,55 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.OAuth.Token.Query do
+  @moduledoc """
+  Contains queries for OAuth Token.
+  """
+
+  import Ecto.Query, only: [from: 2]
+
+  @type query :: Ecto.Queryable.t() | Token.t()
+
+  alias Pleroma.Web.OAuth.Token
+
+  @spec get_by_refresh_token(query, String.t()) :: query
+  def get_by_refresh_token(query \\ Token, refresh_token) do
+    from(q in query, where: q.refresh_token == ^refresh_token)
+  end
+
+  @spec get_by_token(query, String.t()) :: query
+  def get_by_token(query \\ Token, token) do
+    from(q in query, where: q.token == ^token)
+  end
+
+  @spec get_by_app(query, String.t()) :: query
+  def get_by_app(query \\ Token, app_id) do
+    from(q in query, where: q.app_id == ^app_id)
+  end
+
+  @spec get_by_id(query, String.t()) :: query
+  def get_by_id(query \\ Token, id) do
+    from(q in query, where: q.id == ^id)
+  end
+
+  @spec get_expired_tokens(query, DateTime.t() | nil) :: query
+  def get_expired_tokens(query \\ Token, date \\ nil) do
+    expired_date = date || Timex.now()
+    from(q in query, where: fragment("?", q.valid_until) < ^expired_date)
+  end
+
+  @spec get_by_user(query, String.t()) :: query
+  def get_by_user(query \\ Token, user_id) do
+    from(q in query, where: q.user_id == ^user_id)
+  end
+
+  @spec preload(query, any) :: query
+  def preload(query \\ Token, assoc_preload \\ [])
+
+  def preload(query, assoc_preload) when is_list(assoc_preload) do
+    from(q in query, preload: ^assoc_preload)
+  end
+
+  def preload(query, _assoc_preload), do: query
+end
diff --git a/test/web/oauth/token_test.exs b/test/web/oauth/token_test.exs
index ad2a49f09..3c07309b7 100644
--- a/test/web/oauth/token_test.exs
+++ b/test/web/oauth/token_test.exs
@@ -69,4 +69,17 @@ test "deletes all tokens of a user" do
 
     assert tokens == 2
   end
+
+  test "deletes expired tokens" do
+    insert(:oauth_token, valid_until: Timex.shift(Timex.now(), days: -3))
+    insert(:oauth_token, valid_until: Timex.shift(Timex.now(), days: -3))
+    t3 = insert(:oauth_token)
+    t4 = insert(:oauth_token, valid_until: Timex.shift(Timex.now(), minutes: 10))
+    {tokens, _} = Token.delete_expired_tokens()
+    assert tokens == 2
+    available_tokens = Pleroma.Repo.all(Token)
+
+    token_ids = available_tokens |> Enum.map(& &1.id)
+    assert token_ids == [t3.id, t4.id]
+  end
 end

From 54e10a3e55fe46b71ef7f330605baf8bcccd5a44 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 22 May 2019 20:10:52 +0300
Subject: [PATCH 37/64] Disable timeouts for object pruning query

---
 lib/mix/tasks/pleroma/database.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index f9bafb277..4d480ac3f 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -107,7 +107,7 @@ def run(["prune_objects" | args]) do
       where:
         fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host())
     )
-    |> Repo.delete_all()
+    |> Repo.delete_all(timeout: :infinity)
 
     if Keyword.get(options, :vacuum) do
       Logger.info("Runnning VACUUM FULL")

From 75b6c4b00433560fb5ee502f13e8261b4b8a246a Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Wed, 22 May 2019 04:15:59 +0000
Subject: [PATCH 38/64] mrf: defang policy modules for filtering user profile
 objects

---
 lib/pleroma/web/activity_pub/mrf/simple_policy.ex  | 6 ++++--
 lib/pleroma/web/activity_pub/mrf/user_allowlist.ex | 6 ++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
index 7190652d2..ffaa4b7db 100644
--- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
@@ -105,8 +105,8 @@ defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"}
   defp check_report_removal(_actor_info, object), do: {:ok, object}
 
   @impl true
-  def filter(object) do
-    actor_info = URI.parse(object["actor"])
+  def filter(%{"actor" => actor} = object) do
+    actor_info = URI.parse(actor)
 
     with {:ok, object} <- check_accept(actor_info, object),
          {:ok, object} <- check_reject(actor_info, object),
@@ -119,4 +119,6 @@ def filter(object) do
       _e -> {:reject, nil}
     end
   end
+
+  def filter(object), do: {:ok, object}
 end
diff --git a/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex b/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex
index f5078d818..47663414a 100644
--- a/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex
+++ b/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex
@@ -19,10 +19,12 @@ defp filter_by_list(%{"actor" => actor} = object, allow_list) do
   end
 
   @impl true
-  def filter(object) do
-    actor_info = URI.parse(object["actor"])
+  def filter(%{"actor" => actor} = object) do
+    actor_info = URI.parse(actor)
     allow_list = Config.get([:mrf_user_allowlist, String.to_atom(actor_info.host)], [])
 
     filter_by_list(object, allow_list)
   end
+
+  def filter(object), do: {:ok, object}
 end

From 60f882b09f5f837546d59f8eef56b905e065ec60 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Wed, 22 May 2019 04:33:10 +0000
Subject: [PATCH 39/64] activitypub: run user objects through MRF filters

---
 lib/pleroma/web/activity_pub/activity_pub.ex | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 3d9679ec0..aa0229db7 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -909,7 +909,7 @@ def upload(file, opts \\ []) do
     end
   end
 
-  def user_data_from_user_object(data) do
+  defp object_to_user_data(data) do
     avatar =
       data["icon"]["url"] &&
         %{
@@ -956,9 +956,19 @@ def user_data_from_user_object(data) do
     {:ok, user_data}
   end
 
+  def user_data_from_user_object(data) do
+    with {:ok, data} <- MRF.filter(data),
+         {:ok, data} <- object_to_user_data(data) do
+      {:ok, data}
+    else
+      e -> {:error, e}
+    end
+  end
+
   def fetch_and_prepare_user_from_ap_id(ap_id) do
-    with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do
-      user_data_from_user_object(data)
+    with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id),
+         {:ok, data} <- user_data_from_user_object(data) do
+      {:ok, data}
     else
       e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
     end

From baf72d6c580e5c05ef5fea8a57c57150a5d38589 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Wed, 22 May 2019 04:55:16 +0000
Subject: [PATCH 40/64] mrf: simple policy: add the ability to strip avatars
 and banners from user profiles

---
 config/config.exs                             |  4 ++-
 .../web/activity_pub/mrf/simple_policy.ex     | 32 +++++++++++++++++++
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/config/config.exs b/config/config.exs
index 33b7e713d..e90821d66 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -314,7 +314,9 @@
   federated_timeline_removal: [],
   report_removal: [],
   reject: [],
-  accept: []
+  accept: [],
+  avatar_removal: [],
+  banner_removal: []
 
 config :pleroma, :mrf_keyword,
   reject: [],
diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
index ffaa4b7db..890d70a7a 100644
--- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
@@ -104,6 +104,26 @@ defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"}
 
   defp check_report_removal(_actor_info, object), do: {:ok, object}
 
+  defp check_avatar_removal(%{host: actor_host} = _actor_info, %{"icon" => _icon} = object) do
+    if actor_host in Pleroma.Config.get([:mrf_simple, :avatar_removal]) do
+      {:ok, Map.delete(object, "icon")}
+    else
+      {:ok, object}
+    end
+  end
+
+  defp check_avatar_removal(_actor_info, object), do: {:ok, object}
+
+  defp check_banner_removal(%{host: actor_host} = _actor_info, %{"image" => _image} = object) do
+    if actor_host in Pleroma.Config.get([:mrf_simple, :banner_removal]) do
+      {:ok, Map.delete(object, "image")}
+    else
+      {:ok, object}
+    end
+  end
+
+  defp check_banner_removal(_actor_info, object), do: {:ok, object}
+
   @impl true
   def filter(%{"actor" => actor} = object) do
     actor_info = URI.parse(actor)
@@ -120,5 +140,17 @@ def filter(%{"actor" => actor} = object) do
     end
   end
 
+  def filter(%{"id" => actor, "type" => obj_type} = object)
+      when obj_type in ["Application", "Group", "Organization", "Person", "Service"] do
+    actor_info = URI.parse(actor)
+
+    with {:ok, object} <- check_avatar_removal(actor_info, object),
+         {:ok, object} <- check_banner_removal(actor_info, object) do
+      {:ok, object}
+    else
+      _e -> {:reject, nil}
+    end
+  end
+
   def filter(object), do: {:ok, object}
 end

From 8086c7aed6cdc3b2ac1c09c6c40344e47be08ed9 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Wed, 22 May 2019 05:08:37 +0000
Subject: [PATCH 41/64] tests: add tests for banner and avatar removal

---
 .../activity_pub/mrf/simple_policy_test.exs   | 73 ++++++++++++++++++-
 1 file changed, 72 insertions(+), 1 deletion(-)

diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs
index 74af7dcde..3d1f26e60 100644
--- a/test/web/activity_pub/mrf/simple_policy_test.exs
+++ b/test/web/activity_pub/mrf/simple_policy_test.exs
@@ -17,7 +17,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do
       federated_timeline_removal: [],
       report_removal: [],
       reject: [],
-      accept: []
+      accept: [],
+      avatar_removal: [],
+      banner_removal: []
     )
 
     on_exit(fn ->
@@ -206,6 +208,60 @@ test "has a matching host" do
     end
   end
 
+  describe "when :avatar_removal" do
+    test "is empty" do
+      Config.put([:mrf_simple, :avatar_removal], [])
+
+      remote_user = build_remote_user()
+
+      assert SimplePolicy.filter(remote_user) == {:ok, remote_user}
+    end
+
+    test "is not empty but it doesn't have a matching host" do
+      Config.put([:mrf_simple, :avatar_removal], ["non.matching.remote"])
+
+      remote_user = build_remote_user()
+
+      assert SimplePolicy.filter(remote_user) == {:ok, remote_user}
+    end
+
+    test "has a matching host" do
+      Config.put([:mrf_simple, :avatar_removal], ["remote.instance"])
+
+      remote_user = build_remote_user()
+      {:ok, filtered} = SimplePolicy.filter(remote_user)
+
+      refute filtered["icon"]
+    end
+  end
+
+  describe "when :banner_removal" do
+    test "is empty" do
+      Config.put([:mrf_simple, :banner_removal], [])
+
+      remote_user = build_remote_user()
+
+      assert SimplePolicy.filter(remote_user) == {:ok, remote_user}
+    end
+
+    test "is not empty but it doesn't have a matching host" do
+      Config.put([:mrf_simple, :banner_removal], ["non.matching.remote"])
+
+      remote_user = build_remote_user()
+
+      assert SimplePolicy.filter(remote_user) == {:ok, remote_user}
+    end
+
+    test "has a matching host" do
+      Config.put([:mrf_simple, :banner_removal], ["remote.instance"])
+
+      remote_user = build_remote_user()
+      {:ok, filtered} = SimplePolicy.filter(remote_user)
+
+      refute filtered["image"]
+    end
+  end
+
   defp build_local_message do
     %{
       "actor" => "#{Pleroma.Web.base_url()}/users/alice",
@@ -217,4 +273,19 @@ defp build_local_message do
   defp build_remote_message do
     %{"actor" => "https://remote.instance/users/bob"}
   end
+
+  defp build_remote_user do
+    %{
+      "id" => "https://remote.instance/users/bob",
+      "icon" => %{
+        "url" => "http://example.com/image.jpg",
+        "type" => "Image"
+      },
+      "image" => %{
+        "url" => "http://example.com/image.jpg",
+        "type" => "Image"
+      },
+      "type" => "Person"
+    }
+  end
 end

From 7d9b33b3cebeed451210f754a8c34cc14a9e969b Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Wed, 22 May 2019 05:55:09 +0000
Subject: [PATCH 42/64] update documentation for the new MRF features [no-ci]

---
 CHANGELOG.md   | 2 ++
 docs/config.md | 3 +++
 2 files changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 02d64a850..2a2b11ddf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -42,6 +42,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Emoji packs and emoji pack manager
 - Object pruning (`mix pleroma.database prune_objects`)
 - OAuth: added job to clean expired access tokens
+- MRF: Support for rejecting reports from specific instances (`mrf_simple`)
+- MRF: Support for stripping avatars and banner images from specific instances (`mrf_simple`)
 
 ### Changed
 - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer
diff --git a/docs/config.md b/docs/config.md
index 264b65499..1d1d24c32 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -220,6 +220,9 @@ relates to mascots on the mastodon frontend
 * `federated_timeline_removal`: List of instances to remove from Federated (aka The Whole Known Network) Timeline
 * `reject`: List of instances to reject any activities from
 * `accept`: List of instances to accept any activities from
+* `report_removal`: List of instances to reject reports from
+* `avatar_removal`: List of instances to strip avatars from
+* `banner_removal`: List of instances to strip banners from
 
 ## :mrf_rejectnonpublic
 * `allow_followersonly`: whether to allow followers-only posts

From 356c047759735bcd984ebd44059a21a3cf22af0e Mon Sep 17 00:00:00 2001
From: Alfie Pates <alfie@alfiepates.me>
Date: Thu, 23 May 2019 22:33:27 +0100
Subject: [PATCH 43/64] explicitly set reverse proxy upstream to IPv4

  since Pleroma.Web.Endpoint binds on IPv4 only and `localhost.` resolves to
  [::0] on some systems

  fixes #930.
---
 installation/caddyfile-pleroma.example | 4 +++-
 installation/pleroma-apache.conf       | 6 ++++--
 installation/pleroma.nginx             | 4 +++-
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/installation/caddyfile-pleroma.example b/installation/caddyfile-pleroma.example
index fcf76718e..7985d9c67 100644
--- a/installation/caddyfile-pleroma.example
+++ b/installation/caddyfile-pleroma.example
@@ -10,7 +10,9 @@ example.tld  {
 
   gzip
 
-  proxy / localhost:4000 {
+  # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
+  # and `localhost.` resolves to [::0] on some systems: see issue #930
+  proxy / 127.0.0.1:4000 {
     websocket
     transparent
   }
diff --git a/installation/pleroma-apache.conf b/installation/pleroma-apache.conf
index 2beb7c4cc..b5640ac3d 100644
--- a/installation/pleroma-apache.conf
+++ b/installation/pleroma-apache.conf
@@ -58,8 +58,10 @@ CustomLog ${APACHE_LOG_DIR}/access.log combined
     RewriteRule /(.*) ws://localhost:4000/$1 [P,L]
 
     ProxyRequests off
-    ProxyPass / http://localhost:4000/
-    ProxyPassReverse / http://localhost:4000/
+    # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
+    # and `localhost.` resolves to [::0] on some systems: see issue #930
+    ProxyPass / http://127.0.0.1:4000/
+    ProxyPassReverse / http://127.0.0.1:4000/
 
     RequestHeader set Host ${servername}
     ProxyPreserveHost On
diff --git a/installation/pleroma.nginx b/installation/pleroma.nginx
index cc75d78b2..7425da33f 100644
--- a/installation/pleroma.nginx
+++ b/installation/pleroma.nginx
@@ -69,7 +69,9 @@ server {
         proxy_set_header Connection "upgrade";
         proxy_set_header Host $http_host;
 
-        proxy_pass http://localhost:4000;
+	# this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
+	# and `localhost.` resolves to [::0] on some systems: see issue #930
+        proxy_pass http://127.0.0.1:4000;
 
         client_max_body_size 16m;
     }

From f916e4cdd9a502b83c615146c598be135f47e57a Mon Sep 17 00:00:00 2001
From: feld <feld@feld.me>
Date: Fri, 24 May 2019 20:33:55 +0000
Subject: [PATCH 44/64] Move the Cache Control header test to its own file

We can consolidate our cache control header tests here
---
 lib/pleroma/web/endpoint.ex       | 19 +++++++++++++++++--
 test/plugs/cache_control_test.exs | 20 ++++++++++++++++++++
 2 files changed, 37 insertions(+), 2 deletions(-)
 create mode 100644 test/plugs/cache_control_test.exs

diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex
index 9ef30e885..8cd7a2270 100644
--- a/lib/pleroma/web/endpoint.ex
+++ b/lib/pleroma/web/endpoint.ex
@@ -16,17 +16,32 @@ defmodule Pleroma.Web.Endpoint do
 
   plug(Pleroma.Plugs.UploadedMedia)
 
+  @static_cache_control "public, no-cache"
+
   # InstanceStatic needs to be before Plug.Static to be able to override shipped-static files
   # If you're adding new paths to `only:` you'll need to configure them in InstanceStatic as well
-  plug(Pleroma.Plugs.InstanceStatic, at: "/")
+  # Cache-control headers are duplicated in case we turn off etags in the future
+  plug(Pleroma.Plugs.InstanceStatic,
+    at: "/",
+    gzip: true,
+    cache_control_for_etags: @static_cache_control,
+    headers: %{
+      "cache-control" => @static_cache_control
+    }
+  )
 
   plug(
     Plug.Static,
     at: "/",
     from: :pleroma,
     only:
-      ~w(index.html robots.txt static finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc)
+      ~w(index.html robots.txt static finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc),
     # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength
+    gzip: true,
+    cache_control_for_etags: @static_cache_control,
+    headers: %{
+      "cache-control" => @static_cache_control
+    }
   )
 
   plug(Plug.Static.IndexHtml, at: "/pleroma/admin/")
diff --git a/test/plugs/cache_control_test.exs b/test/plugs/cache_control_test.exs
new file mode 100644
index 000000000..45151b289
--- /dev/null
+++ b/test/plugs/cache_control_test.exs
@@ -0,0 +1,20 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.CacheControlTest do
+  use Pleroma.Web.ConnCase
+  alias Plug.Conn
+
+  test "Verify Cache-Control header on static assets", %{conn: conn} do
+    conn = get(conn, "/index.html")
+
+    assert Conn.get_resp_header(conn, "cache-control") == ["public, no-cache"]
+  end
+
+  test "Verify Cache-Control header on the API", %{conn: conn} do
+    conn = get(conn, "/api/v1/instance")
+
+    assert Conn.get_resp_header(conn, "cache-control") == ["max-age=0, private, must-revalidate"]
+  end
+end

From bbea5691da67916151a883f09e24da7c2e27d9ba Mon Sep 17 00:00:00 2001
From: Sergey Suprunenko <suprunenko.s@gmail.com>
Date: Fri, 24 May 2019 20:34:23 +0000
Subject: [PATCH 45/64] Mention all people in the beginning of DM

---
 lib/pleroma/formatter.ex |  2 +-
 test/formatter_test.exs  | 10 ++++++----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex
index 3e3b9fe97..607843a5b 100644
--- a/lib/pleroma/formatter.ex
+++ b/lib/pleroma/formatter.ex
@@ -8,7 +8,7 @@ defmodule Pleroma.Formatter do
   alias Pleroma.User
   alias Pleroma.Web.MediaProxy
 
-  @safe_mention_regex ~r/^(\s*(?<mentions>@.+?\s+)+)(?<rest>.*)/s
+  @safe_mention_regex ~r/^(\s*(?<mentions>(@.+?\s+){1,})+)(?<rest>.*)/s
   @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui
   @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/
 
diff --git a/test/formatter_test.exs b/test/formatter_test.exs
index 47b91b121..bfa673049 100644
--- a/test/formatter_test.exs
+++ b/test/formatter_test.exs
@@ -184,17 +184,19 @@ test "does not give a replacement for single-character local nicknames who don't
 
     test "given the 'safe_mention' option, it will only mention people in the beginning" do
       user = insert(:user)
-      _other_user = insert(:user)
+      other_user = insert(:user)
       third_user = insert(:user)
-      text = " @#{user.nickname} hey dude i hate @#{third_user.nickname}"
+      text = " @#{user.nickname} @#{other_user.nickname} hey dudes i hate @#{third_user.nickname}"
       {expected_text, mentions, [] = _tags} = Formatter.linkify(text, safe_mention: true)
 
-      assert mentions == [{"@#{user.nickname}", user}]
+      assert mentions == [{"@#{user.nickname}", user}, {"@#{other_user.nickname}", other_user}]
 
       assert expected_text ==
                "<span class='h-card'><a data-user='#{user.id}' class='u-url mention' href='#{
                  user.ap_id
-               }'>@<span>#{user.nickname}</span></a></span> hey dude i hate <span class='h-card'><a data-user='#{
+               }'>@<span>#{user.nickname}</span></a></span> <span class='h-card'><a data-user='#{
+                 other_user.id
+               }' class='u-url mention' href='#{other_user.ap_id}'>@<span>#{other_user.nickname}</span></a></span> hey dudes i hate <span class='h-card'><a data-user='#{
                  third_user.id
                }' class='u-url mention' href='#{third_user.ap_id}'>@<span>#{third_user.nickname}</span></a></span>"
     end

From 9415932af5829f2aa19e362076e0653dd1ce9c5a Mon Sep 17 00:00:00 2001
From: Aaron Tinio <aptinio@gmail.com>
Date: Sat, 25 May 2019 08:15:12 +0800
Subject: [PATCH 46/64] Keep nodeinfo available when not federating

---
 .../web/nodeinfo/nodeinfo_controller.ex       |  2 -
 test/web/node_info_test.exs                   | 80 ++++---------------
 2 files changed, 16 insertions(+), 66 deletions(-)

diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex
index 3bf2a0fbc..45f90c579 100644
--- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex
+++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex
@@ -12,8 +12,6 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do
   alias Pleroma.Web.ActivityPub.MRF
   alias Pleroma.Web.Federator.Publisher
 
-  plug(Pleroma.Web.FederatingPlug)
-
   def schemas(conn, _params) do
     response = %{
       links: [
diff --git a/test/web/node_info_test.exs b/test/web/node_info_test.exs
index 2fc42b7cc..be1173513 100644
--- a/test/web/node_info_test.exs
+++ b/test/web/node_info_test.exs
@@ -7,6 +7,22 @@ defmodule Pleroma.Web.NodeInfoTest do
 
   import Pleroma.Factory
 
+  test "GET /.well-known/nodeinfo", %{conn: conn} do
+    links =
+      conn
+      |> get("/.well-known/nodeinfo")
+      |> json_response(200)
+      |> Map.fetch!("links")
+
+    Enum.each(links, fn link ->
+      href = Map.fetch!(link, "href")
+
+      conn
+      |> get(href)
+      |> json_response(200)
+    end)
+  end
+
   test "nodeinfo shows staff accounts", %{conn: conn} do
     moderator = insert(:user, %{local: true, info: %{is_moderator: true}})
     admin = insert(:user, %{local: true, info: %{is_admin: true}})
@@ -32,70 +48,6 @@ test "nodeinfo shows restricted nicknames", %{conn: conn} do
              result["metadata"]["restrictedNicknames"]
   end
 
-  test "returns 404 when federation is disabled", %{conn: conn} do
-    instance =
-      Application.get_env(:pleroma, :instance)
-      |> Keyword.put(:federating, false)
-
-    Application.put_env(:pleroma, :instance, instance)
-
-    conn
-    |> get("/.well-known/nodeinfo")
-    |> json_response(404)
-
-    conn
-    |> get("/nodeinfo/2.1.json")
-    |> json_response(404)
-
-    instance =
-      Application.get_env(:pleroma, :instance)
-      |> Keyword.put(:federating, true)
-
-    Application.put_env(:pleroma, :instance, instance)
-  end
-
-  test "returns 200 when federation is enabled", %{conn: conn} do
-    conn
-    |> get("/.well-known/nodeinfo")
-    |> json_response(200)
-
-    conn
-    |> get("/nodeinfo/2.1.json")
-    |> json_response(200)
-  end
-
-  test "returns 404 when federation is disabled (nodeinfo 2.0)", %{conn: conn} do
-    instance =
-      Application.get_env(:pleroma, :instance)
-      |> Keyword.put(:federating, false)
-
-    Application.put_env(:pleroma, :instance, instance)
-
-    conn
-    |> get("/.well-known/nodeinfo")
-    |> json_response(404)
-
-    conn
-    |> get("/nodeinfo/2.0.json")
-    |> json_response(404)
-
-    instance =
-      Application.get_env(:pleroma, :instance)
-      |> Keyword.put(:federating, true)
-
-    Application.put_env(:pleroma, :instance, instance)
-  end
-
-  test "returns 200 when federation is enabled (nodeinfo 2.0)", %{conn: conn} do
-    conn
-    |> get("/.well-known/nodeinfo")
-    |> json_response(200)
-
-    conn
-    |> get("/nodeinfo/2.0.json")
-    |> json_response(200)
-  end
-
   test "returns software.repository field in nodeinfo 2.1", %{conn: conn} do
     conn
     |> get("/.well-known/nodeinfo")

From 9bec891eb4d5d06e6bd84dd2c95259d2c1a4f563 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sat, 25 May 2019 04:24:21 +0000
Subject: [PATCH 47/64] kill @httpoison

---
 config/config.exs                                     |  1 -
 lib/pleroma/object/fetcher.ex                         |  5 ++---
 lib/pleroma/reverse_proxy.ex                          |  5 +++--
 lib/pleroma/uploaders/mdii.ex                         |  5 ++---
 lib/pleroma/web/activity_pub/publisher.ex             |  5 ++---
 .../web/mastodon_api/mastodon_api_controller.ex       |  4 ++--
 lib/pleroma/web/ostatus/ostatus.ex                    |  7 +++----
 lib/pleroma/web/salmon/salmon.ex                      |  5 ++---
 lib/pleroma/web/web_finger/web_finger.ex              |  9 ++++-----
 lib/pleroma/web/websub/websub.ex                      | 11 +++++------
 10 files changed, 25 insertions(+), 32 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index e90821d66..01827fb5f 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -186,7 +186,6 @@
 
 config :pleroma, :websub, Pleroma.Web.Websub
 config :pleroma, :ostatus, Pleroma.Web.OStatus
-config :pleroma, :httpoison, Pleroma.HTTP
 config :tesla, adapter: Tesla.Adapter.Hackney
 
 # Configures http settings, upstream proxy etc.
diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index bb9388d4f..ca980c629 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -1,4 +1,5 @@
 defmodule Pleroma.Object.Fetcher do
+  alias Pleroma.HTTP
   alias Pleroma.Object
   alias Pleroma.Object.Containment
   alias Pleroma.Web.ActivityPub.Transmogrifier
@@ -6,8 +7,6 @@ defmodule Pleroma.Object.Fetcher do
 
   require Logger
 
-  @httpoison Application.get_env(:pleroma, :httpoison)
-
   defp reinject_object(data) do
     Logger.debug("Reinjecting object #{data["id"]}")
 
@@ -78,7 +77,7 @@ def fetch_and_contain_remote_object_from_id(id) do
 
     with true <- String.starts_with?(id, "http"),
          {:ok, %{body: body, status: code}} when code in 200..299 <-
-           @httpoison.get(
+           HTTP.get(
              id,
              [{:Accept, "application/activity+json"}]
            ),
diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex
index a3f177fec..6e5feb4c3 100644
--- a/lib/pleroma/reverse_proxy.ex
+++ b/lib/pleroma/reverse_proxy.ex
@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.ReverseProxy do
+  alias Pleroma.HTTP
+
   @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++
                       ~w(if-unmodified-since if-none-match if-range range)
   @resp_cache_headers ~w(etag date last-modified cache-control)
@@ -60,7 +62,6 @@ defmodule Pleroma.ReverseProxy do
 
   """
   @hackney Application.get_env(:pleroma, :hackney, :hackney)
-  @httpoison Application.get_env(:pleroma, :httpoison, HTTPoison)
 
   @default_hackney_options []
 
@@ -97,7 +98,7 @@ def call(conn = %{method: method}, url, opts) when method in @methods do
     hackney_opts =
       @default_hackney_options
       |> Keyword.merge(Keyword.get(opts, :http, []))
-      |> @httpoison.process_request_options()
+      |> HTTP.process_request_options()
 
     req_headers = build_req_headers(conn.req_headers, opts)
 
diff --git a/lib/pleroma/uploaders/mdii.ex b/lib/pleroma/uploaders/mdii.ex
index 190ed9f3a..237544337 100644
--- a/lib/pleroma/uploaders/mdii.ex
+++ b/lib/pleroma/uploaders/mdii.ex
@@ -4,11 +4,10 @@
 
 defmodule Pleroma.Uploaders.MDII do
   alias Pleroma.Config
+  alias Pleroma.HTTP
 
   @behaviour Pleroma.Uploaders.Uploader
 
-  @httpoison Application.get_env(:pleroma, :httpoison)
-
   # MDII-hosted images are never passed through the MediaPlug; only local media.
   # Delegate to Pleroma.Uploaders.Local
   def get_file(file) do
@@ -25,7 +24,7 @@ def put_file(upload) do
     query = "#{cgi}?#{extension}"
 
     with {:ok, %{status: 200, body: body}} <-
-           @httpoison.post(query, file_data, [], adapter: [pool: :default]) do
+           HTTP.post(query, file_data, [], adapter: [pool: :default]) do
       remote_file_name = String.split(body) |> List.first()
       public_url = "#{files}/#{remote_file_name}.#{extension}"
       {:ok, {:url, public_url}}
diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex
index 11dba87de..8f1399ce6 100644
--- a/lib/pleroma/web/activity_pub/publisher.ex
+++ b/lib/pleroma/web/activity_pub/publisher.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.ActivityPub.Publisher do
   alias Pleroma.Activity
   alias Pleroma.Config
+  alias Pleroma.HTTP
   alias Pleroma.Instances
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Relay
@@ -16,8 +17,6 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
 
   require Logger
 
-  @httpoison Application.get_env(:pleroma, :httpoison)
-
   @moduledoc """
   ActivityPub outgoing federation module.
   """
@@ -63,7 +62,7 @@ def publish_one(%{inbox: inbox, json: json, actor: %User{} = actor, id: id} = pa
 
     with {:ok, %{status: code}} when code in 200..299 <-
            result =
-             @httpoison.post(
+             HTTP.post(
                inbox,
                json,
                [
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 1ec0f30a1..0fe09c285 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -11,6 +11,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   alias Pleroma.Conversation.Participation
   alias Pleroma.Filter
   alias Pleroma.Formatter
+  alias Pleroma.HTTP
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Object.Fetcher
@@ -55,7 +56,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     when action in [:account_register]
   )
 
-  @httpoison Application.get_env(:pleroma, :httpoison)
   @local_mastodon_name "Mastodon-Local"
 
   action_fallback(:errors)
@@ -1691,7 +1691,7 @@ def suggestions(%{assigns: %{user: user}} = conn, _) do
         |> String.replace("{{user}}", user)
 
       with {:ok, %{status: 200, body: body}} <-
-             @httpoison.get(
+             HTTP.get(
                url,
                [],
                adapter: [
diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex
index 61515b31e..6ed089d84 100644
--- a/lib/pleroma/web/ostatus/ostatus.ex
+++ b/lib/pleroma/web/ostatus/ostatus.ex
@@ -3,13 +3,12 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.OStatus do
-  @httpoison Application.get_env(:pleroma, :httpoison)
-
   import Ecto.Query
   import Pleroma.Web.XML
   require Logger
 
   alias Pleroma.Activity
+  alias Pleroma.HTTP
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
@@ -363,7 +362,7 @@ def get_atom_url(body) do
   def fetch_activity_from_atom_url(url) do
     with true <- String.starts_with?(url, "http"),
          {:ok, %{body: body, status: code}} when code in 200..299 <-
-           @httpoison.get(
+           HTTP.get(
              url,
              [{:Accept, "application/atom+xml"}]
            ) do
@@ -380,7 +379,7 @@ def fetch_activity_from_html_url(url) do
     Logger.debug("Trying to fetch #{url}")
 
     with true <- String.starts_with?(url, "http"),
-         {:ok, %{body: body}} <- @httpoison.get(url, []),
+         {:ok, %{body: body}} <- HTTP.get(url, []),
          {:ok, atom_url} <- get_atom_url(body) do
       fetch_activity_from_atom_url(atom_url)
     else
diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index f25d92fad..9e91a5a40 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -5,11 +5,10 @@
 defmodule Pleroma.Web.Salmon do
   @behaviour Pleroma.Web.Federator.Publisher
 
-  @httpoison Application.get_env(:pleroma, :httpoison)
-
   use Bitwise
 
   alias Pleroma.Activity
+  alias Pleroma.HTTP
   alias Pleroma.Instances
   alias Pleroma.Keys
   alias Pleroma.User
@@ -138,7 +137,7 @@ def publish_one(%{recipient: %{info: %{salmon: salmon}}} = params),
 
   def publish_one(%{recipient: url, feed: feed} = params) when is_binary(url) do
     with {:ok, %{status: code}} when code in 200..299 <-
-           @httpoison.post(
+           HTTP.post(
              url,
              feed,
              [{"Content-Type", "application/magic-envelope+xml"}]
diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex
index c5b7d4acb..3fca72de8 100644
--- a/lib/pleroma/web/web_finger/web_finger.ex
+++ b/lib/pleroma/web/web_finger/web_finger.ex
@@ -3,8 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.WebFinger do
-  @httpoison Application.get_env(:pleroma, :httpoison)
-
+  alias Pleroma.HTTP
   alias Pleroma.User
   alias Pleroma.Web
   alias Pleroma.Web.Federator.Publisher
@@ -176,11 +175,11 @@ def get_template_from_xml(body) do
 
   def find_lrdd_template(domain) do
     with {:ok, %{status: status, body: body}} when status in 200..299 <-
-           @httpoison.get("http://#{domain}/.well-known/host-meta", []) do
+           HTTP.get("http://#{domain}/.well-known/host-meta", []) do
       get_template_from_xml(body)
     else
       _ ->
-        with {:ok, %{body: body}} <- @httpoison.get("https://#{domain}/.well-known/host-meta", []) do
+        with {:ok, %{body: body}} <- HTTP.get("https://#{domain}/.well-known/host-meta", []) do
           get_template_from_xml(body)
         else
           e -> {:error, "Can't find LRDD template: #{inspect(e)}"}
@@ -209,7 +208,7 @@ def finger(account) do
       end
 
     with response <-
-           @httpoison.get(
+           HTTP.get(
              address,
              Accept: "application/xrd+xml,application/jrd+json"
            ),
diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex
index 7ad0414ab..b61f388b8 100644
--- a/lib/pleroma/web/websub/websub.ex
+++ b/lib/pleroma/web/websub/websub.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.Websub do
   alias Ecto.Changeset
   alias Pleroma.Activity
+  alias Pleroma.HTTP
   alias Pleroma.Instances
   alias Pleroma.Repo
   alias Pleroma.User
@@ -24,9 +25,7 @@ defmodule Pleroma.Web.Websub do
 
   @behaviour Pleroma.Web.Federator.Publisher
 
-  @httpoison Application.get_env(:pleroma, :httpoison)
-
-  def verify(subscription, getter \\ &@httpoison.get/3) do
+  def verify(subscription, getter \\ &HTTP.get/3) do
     challenge = Base.encode16(:crypto.strong_rand_bytes(8))
     lease_seconds = NaiveDateTime.diff(subscription.valid_until, subscription.updated_at)
     lease_seconds = lease_seconds |> to_string
@@ -207,7 +206,7 @@ def subscribe(subscriber, subscribed, requester \\ &request_subscription/1) do
     requester.(subscription)
   end
 
-  def gather_feed_data(topic, getter \\ &@httpoison.get/1) do
+  def gather_feed_data(topic, getter \\ &HTTP.get/1) do
     with {:ok, response} <- getter.(topic),
          status when status in 200..299 <- response.status,
          body <- response.body,
@@ -236,7 +235,7 @@ def gather_feed_data(topic, getter \\ &@httpoison.get/1) do
     end
   end
 
-  def request_subscription(websub, poster \\ &@httpoison.post/3, timeout \\ 10_000) do
+  def request_subscription(websub, poster \\ &HTTP.post/3, timeout \\ 10_000) do
     data = [
       "hub.mode": "subscribe",
       "hub.topic": websub.topic,
@@ -294,7 +293,7 @@ def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret} =
     Logger.info(fn -> "Pushing #{topic} to #{callback}" end)
 
     with {:ok, %{status: code}} when code in 200..299 <-
-           @httpoison.post(
+           HTTP.post(
              callback,
              xml,
              [

From 80d55d428fedfe4ebc44569c6134908d269db698 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sat, 25 May 2019 04:34:16 +0000
Subject: [PATCH 48/64] tests: websub: check only that signature validation
 succeeds or fails

---
 config/test.exs                            | 4 ++--
 test/web/websub/websub_controller_test.exs | 6 +-----
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/config/test.exs b/config/test.exs
index 6100989c4..3bc407840 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -39,8 +39,8 @@
 # Reduce hash rounds for testing
 config :pbkdf2_elixir, rounds: 1
 
-config :pleroma, :websub, Pleroma.Web.WebsubMock
-config :pleroma, :ostatus, Pleroma.Web.OStatusMock
+#config :pleroma, :websub, Pleroma.Web.WebsubMock
+#config :pleroma, :ostatus, Pleroma.Web.OStatusMock
 config :tesla, adapter: Tesla.Mock
 config :pleroma, :rich_media, enabled: false
 
diff --git a/test/web/websub/websub_controller_test.exs b/test/web/websub/websub_controller_test.exs
index 1e69ed01a..bf2ee31ee 100644
--- a/test/web/websub/websub_controller_test.exs
+++ b/test/web/websub/websub_controller_test.exs
@@ -52,7 +52,7 @@ test "websub subscription confirmation", %{conn: conn} do
   end
 
   describe "websub_incoming" do
-    test "handles incoming feed updates", %{conn: conn} do
+    test "accepts incoming feed updates", %{conn: conn} do
       websub = insert(:websub_client_subscription)
       doc = "some stuff"
       signature = Websub.sign(websub.secret, doc)
@@ -64,8 +64,6 @@ test "handles incoming feed updates", %{conn: conn} do
         |> post("/push/subscriptions/#{websub.id}", doc)
 
       assert response(conn, 200) == "OK"
-
-      assert length(Repo.all(Activity)) == 1
     end
 
     test "rejects incoming feed updates with the wrong signature", %{conn: conn} do
@@ -80,8 +78,6 @@ test "rejects incoming feed updates with the wrong signature", %{conn: conn} do
         |> post("/push/subscriptions/#{websub.id}", doc)
 
       assert response(conn, 500) == "Error"
-
-      assert Enum.empty?(Repo.all(Activity))
     end
   end
 end

From 56fd7dbdd7e3f1a0bb05430ed4805d8715e54935 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sat, 25 May 2019 04:39:32 +0000
Subject: [PATCH 49/64] remove @websub and @ostatus module-level constants

---
 config/config.exs                          |  2 --
 config/test.exs                            |  2 --
 lib/pleroma/web/federator/federator.ex     |  8 +++-----
 test/support/ostatus_mock.ex               | 11 -----------
 test/support/websub_mock.ex                |  9 ---------
 test/web/websub/websub_controller_test.exs |  1 -
 6 files changed, 3 insertions(+), 30 deletions(-)
 delete mode 100644 test/support/ostatus_mock.ex
 delete mode 100644 test/support/websub_mock.ex

diff --git a/config/config.exs b/config/config.exs
index 01827fb5f..68168b279 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -184,8 +184,6 @@
   "application/ld+json" => ["activity+json"]
 }
 
-config :pleroma, :websub, Pleroma.Web.Websub
-config :pleroma, :ostatus, Pleroma.Web.OStatus
 config :tesla, adapter: Tesla.Adapter.Hackney
 
 # Configures http settings, upstream proxy etc.
diff --git a/config/test.exs b/config/test.exs
index 3bc407840..41cddb9bd 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -39,8 +39,6 @@
 # Reduce hash rounds for testing
 config :pbkdf2_elixir, rounds: 1
 
-#config :pleroma, :websub, Pleroma.Web.WebsubMock
-#config :pleroma, :ostatus, Pleroma.Web.OStatusMock
 config :tesla, adapter: Tesla.Mock
 config :pleroma, :rich_media, enabled: false
 
diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex
index 6b0b75284..f4c9fe284 100644
--- a/lib/pleroma/web/federator/federator.ex
+++ b/lib/pleroma/web/federator/federator.ex
@@ -11,13 +11,11 @@ defmodule Pleroma.Web.Federator do
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.Federator.Publisher
   alias Pleroma.Web.Federator.RetryQueue
+  alias Pleroma.Web.OStatus
   alias Pleroma.Web.Websub
 
   require Logger
 
-  @websub Application.get_env(:pleroma, :websub)
-  @ostatus Application.get_env(:pleroma, :ostatus)
-
   def init do
     # 1 minute
     Process.sleep(1000 * 60)
@@ -87,12 +85,12 @@ def perform(:verify_websub, websub) do
       "Running WebSub verification for #{websub.id} (#{websub.topic}, #{websub.callback})"
     end)
 
-    @websub.verify(websub)
+    Websub.verify(websub)
   end
 
   def perform(:incoming_doc, doc) do
     Logger.info("Got document, trying to parse")
-    @ostatus.handle_incoming(doc)
+    OStatus.handle_incoming(doc)
   end
 
   def perform(:incoming_ap_doc, params) do
diff --git a/test/support/ostatus_mock.ex b/test/support/ostatus_mock.ex
deleted file mode 100644
index 9c0f2f323..000000000
--- a/test/support/ostatus_mock.ex
+++ /dev/null
@@ -1,11 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.OStatusMock do
-  import Pleroma.Factory
-
-  def handle_incoming(_doc) do
-    insert(:note_activity)
-  end
-end
diff --git a/test/support/websub_mock.ex b/test/support/websub_mock.ex
deleted file mode 100644
index e3d5a5b16..000000000
--- a/test/support/websub_mock.ex
+++ /dev/null
@@ -1,9 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.WebsubMock do
-  def verify(sub) do
-    {:ok, sub}
-  end
-end
diff --git a/test/web/websub/websub_controller_test.exs b/test/web/websub/websub_controller_test.exs
index bf2ee31ee..f79745d58 100644
--- a/test/web/websub/websub_controller_test.exs
+++ b/test/web/websub/websub_controller_test.exs
@@ -5,7 +5,6 @@
 defmodule Pleroma.Web.Websub.WebsubControllerTest do
   use Pleroma.Web.ConnCase
   import Pleroma.Factory
-  alias Pleroma.Activity
   alias Pleroma.Repo
   alias Pleroma.Web.Websub
   alias Pleroma.Web.Websub.WebsubClientSubscription

From 79503ce90f6d85f00ee9e2dbc6358df2237d5036 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sun, 26 May 2019 01:57:22 +0000
Subject: [PATCH 50/64] mrf: simple policy: fix matching imported activitypub
 and ostatus statuses

---
 CHANGELOG.md                                   |  1 +
 .../web/activity_pub/mrf/simple_policy.ex      |  3 +--
 .../activity_pub/mrf/simple_policy_test.exs    | 18 ++++++++++++++++++
 3 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7869e299b..f689160e9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -113,6 +113,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: Exposing default scope of the user to anyone
 - Mastodon API: Make `irreversible` field default to `false` [`POST /api/v1/filters`]
 - User-Agent is now sent correctly for all HTTP requests.
+- MRF: Simple policy now properly delists imported or relayed statuses
 
 ## Removed
 - Configuration: `config :pleroma, :fe` in favor of the more flexible `config :pleroma, :frontend_configurations`
diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
index 890d70a7a..433d23c5f 100644
--- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
@@ -74,8 +74,7 @@ defp check_ftl_removal(%{host: actor_host} = _actor_info, object) do
                actor_host
              ),
            user <- User.get_cached_by_ap_id(object["actor"]),
-           true <- "https://www.w3.org/ns/activitystreams#Public" in object["to"],
-           true <- user.follower_address in object["cc"] do
+           true <- "https://www.w3.org/ns/activitystreams#Public" in object["to"] do
         to =
           List.delete(object["to"], "https://www.w3.org/ns/activitystreams#Public") ++
             [user.follower_address]
diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs
index 3d1f26e60..0fd68e103 100644
--- a/test/web/activity_pub/mrf/simple_policy_test.exs
+++ b/test/web/activity_pub/mrf/simple_policy_test.exs
@@ -145,6 +145,24 @@ test "has a matching host" do
 
       assert SimplePolicy.filter(local_message) == {:ok, local_message}
     end
+
+    test "has a matching host but only as:Public in to" do
+      {_actor, ftl_message} = build_ftl_actor_and_message()
+
+      ftl_message_actor_host =
+        ftl_message
+        |> Map.fetch!("actor")
+        |> URI.parse()
+        |> Map.fetch!(:host)
+
+      ftl_message = Map.put(ftl_message, "cc", [])
+
+      Config.put([:mrf_simple, :federated_timeline_removal], [ftl_message_actor_host])
+
+      assert {:ok, ftl_message} = SimplePolicy.filter(ftl_message)
+      refute "https://www.w3.org/ns/activitystreams#Public" in ftl_message["to"]
+      assert "https://www.w3.org/ns/activitystreams#Public" in ftl_message["cc"]
+    end
   end
 
   defp build_ftl_actor_and_message do

From 9f3bcf0efe7fa06e2b8970386c099c1ea2974d0a Mon Sep 17 00:00:00 2001
From: jeff <jeff@i2p.rocks>
Date: Tue, 28 May 2019 06:49:53 +0000
Subject: [PATCH 51/64] Respect proxy settings federation

---
 lib/pleroma/http/connection.ex | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex
index 558005c19..c216cdcb1 100644
--- a/lib/pleroma/http/connection.ex
+++ b/lib/pleroma/http/connection.ex
@@ -32,9 +32,11 @@ def new(opts \\ []) do
   defp hackney_options(opts) do
     options = Keyword.get(opts, :adapter, [])
     adapter_options = Pleroma.Config.get([:http, :adapter], [])
+    proxy_url = Pleroma.Config.get([:http, :proxy_url], nil)
 
     @hackney_options
     |> Keyword.merge(adapter_options)
     |> Keyword.merge(options)
+    |> Keyword.merge(proxy: proxy_url)
   end
 end

From abc15b6dcca39f62f043fcaf0292443a3dcb3d76 Mon Sep 17 00:00:00 2001
From: feld <feld@feld.me>
Date: Tue, 28 May 2019 21:20:24 +0000
Subject: [PATCH 52/64] Improve Varnish config. We set sane headers from the
 backend now.

---
 installation/pleroma.vcl | 70 ++++++++++++++++++++--------------------
 1 file changed, 35 insertions(+), 35 deletions(-)

diff --git a/installation/pleroma.vcl b/installation/pleroma.vcl
index 92153d8ef..154747aa6 100644
--- a/installation/pleroma.vcl
+++ b/installation/pleroma.vcl
@@ -1,4 +1,4 @@
-vcl 4.0;
+vcl 4.1;
 import std;
 
 backend default {
@@ -35,24 +35,6 @@ sub vcl_recv {
       }
       return(purge);
     }
-
-    # Pleroma MediaProxy - strip headers that will affect caching
-    if (req.url ~ "^/proxy/") {
-      unset req.http.Cookie;
-      unset req.http.Authorization;
-      unset req.http.Accept;
-      return (hash);
-    }
-
-    # Strip headers that will affect caching from all other static content
-    # This also permits caching of individual toots and AP Activities
-    if ((req.url ~ "^/(media|static)/") ||
-    (req.url ~ "(?i)\.(html|js|css|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|mp4|ogg|webm|svg|swf|ttf|pdf|woff|woff2)$"))
-    {
-      unset req.http.Cookie;
-      unset req.http.Authorization;
-      return (hash);
-    }
 }
 
 sub vcl_backend_response {
@@ -61,6 +43,12 @@ sub vcl_backend_response {
       set beresp.do_gzip = true;
     }
 
+    # Retry broken backend responses.
+    if (beresp.status == 503) {
+      set bereq.http.X-Varnish-Backend-503 = "1";
+      return (retry);
+    }
+
     # CHUNKED SUPPORT
     if (bereq.http.x-range ~ "bytes=" && beresp.status == 206) {
       set beresp.ttl = 10m;
@@ -73,8 +61,6 @@ sub vcl_backend_response {
       return (deliver);
     }
 
-    # Default object caching of 86400s;
-    set beresp.ttl = 86400s;
     # Allow serving cached content for 6h in case backend goes down
     set beresp.grace = 6h;
 
@@ -90,20 +76,6 @@ sub vcl_backend_response {
       set beresp.ttl = 30s;
       return (deliver);
     }
-
-    # Pleroma MediaProxy internally sets headers properly
-    if (bereq.url ~ "^/proxy/") {
-      return (deliver);
-    }
-
-    # Strip cache-restricting headers from Pleroma on static content that we want to cache
-    if (bereq.url ~ "(?i)\.(js|css|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|mp4|ogg|webm|svg|swf|ttf|pdf|woff|woff2)$")
-    {
-      unset beresp.http.set-cookie;
-      unset beresp.http.Cache-Control;
-      unset beresp.http.x-request-id;
-      set beresp.http.Cache-Control = "public, max-age=86400";
-    }
 }
 
 # The synthetic response for 301 redirects
@@ -132,10 +104,32 @@ sub vcl_hash {
 }
 
 sub vcl_backend_fetch {
+    # Be more lenient for slow servers on the fediverse
+    if bereq.url ~ "^/proxy/" {
+      set bereq.first_byte_timeout = 300s;
+    }
+
     # CHUNKED SUPPORT
     if (bereq.http.x-range) {
       set bereq.http.Range = bereq.http.x-range;
     }
+
+    if (bereq.retries == 0) {
+        # Clean up the X-Varnish-Backend-503 flag that is used internally
+        # to mark broken backend responses that should be retried.
+        unset bereq.http.X-Varnish-Backend-503;
+    } else {
+        if (bereq.http.X-Varnish-Backend-503) {
+            if (bereq.method != "POST" &&
+              std.healthy(bereq.backend) &&
+              bereq.retries <= 4) {
+              # Flush broken backend response flag & try again.
+              unset bereq.http.X-Varnish-Backend-503;
+            } else {
+              return (abandon);
+            }
+        }
+    }
 }
 
 sub vcl_deliver {
@@ -145,3 +139,9 @@ sub vcl_deliver {
       unset resp.http.CR;
     }
 }
+
+sub vcl_backend_error {
+    # Retry broken backend responses.
+    set bereq.http.X-Varnish-Backend-503 = "1";
+    return (retry);
+}

From 0159a6dbe97330150d2913c7d7a060151f83f7eb Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Wed, 29 May 2019 10:58:45 +0000
Subject: [PATCH 53/64] router: require oauth_read for searching

Search calls are generally expensive and allow unauthenticated users to
crawl the instance for user profiles or posts which contain specified
keywords.  An adversary can build a distributed search engine which not
only will consume significant instance resources, but also can be used
for undesirable purposes such as datamining.

Accordingly, require authenticated access to use the search API endpoints.
This acts as a nice balance as it allows guest users to make use of most
functionality available in Pleroma FE while ensuring that Pleroma
instances are reasonably protected from resource exhaustion.  It also
removes Pleroma as a potential vector in distributed search engines.
---
 lib/pleroma/web/router.ex | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 352268b96..08c74a742 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -414,7 +414,12 @@ defmodule Pleroma.Web.Router do
 
     get("/trends", MastodonAPIController, :empty_array)
 
-    get("/accounts/search", MastodonAPIController, :account_search)
+    scope [] do
+      pipe_through(:oauth_read)
+
+      get("/search", MastodonAPIController, :search)
+      get("/accounts/search", MastodonAPIController, :account_search)
+    end
 
     scope [] do
       pipe_through(:oauth_read_or_public)
@@ -431,14 +436,12 @@ defmodule Pleroma.Web.Router do
       get("/accounts/:id/following", MastodonAPIController, :following)
       get("/accounts/:id", MastodonAPIController, :user)
 
-      get("/search", MastodonAPIController, :search)
-
       get("/pleroma/accounts/:id/favourites", MastodonAPIController, :user_favourites)
     end
   end
 
   scope "/api/v2", Pleroma.Web.MastodonAPI do
-    pipe_through([:api, :oauth_read_or_public])
+    pipe_through([:api, :oauth_read])
     get("/search", MastodonAPIController, :search2)
   end
 
@@ -480,9 +483,14 @@ defmodule Pleroma.Web.Router do
       get("/statuses/show/:id", TwitterAPI.Controller, :fetch_status)
       get("/statusnet/conversation/:id", TwitterAPI.Controller, :fetch_conversation)
 
-      get("/search", TwitterAPI.Controller, :search)
       get("/statusnet/tags/timeline/:tag", TwitterAPI.Controller, :public_and_external_timeline)
     end
+
+    scope [] do
+      pipe_through(:oauth_read)
+
+      get("/search", TwitterAPI.Controller, :search)
+    end
   end
 
   scope "/api", Pleroma.Web do
@@ -500,7 +508,7 @@ defmodule Pleroma.Web.Router do
   end
 
   scope "/api", Pleroma.Web, as: :twitter_api_search do
-    pipe_through([:api, :oauth_read_or_public])
+    pipe_through([:api, :oauth_read])
     get("/pleroma/search_user", TwitterAPI.Controller, :search_user)
   end
 

From 672fddb7213b69f3ee8b9a7728426373090847ed Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 29 May 2019 08:06:26 -0500
Subject: [PATCH 54/64] Default search limit should be 40

https://docs.joinmastodon.org/api/rest/search/
---
 lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 0fe09c285..2110027c3 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -1084,7 +1084,7 @@ def status_search(user, query) do
       from([a, o] in Activity.with_preloaded_object(Activity),
         where: fragment("?->>'type' = 'Create'", a.data),
         where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients,
-        limit: 20
+        limit: 40
       )
 
     q =

From db94294dfb6f25ce0f08acfb78b3c4cc2241e9f5 Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Wed, 29 May 2019 14:04:58 +0000
Subject: [PATCH 55/64] [#936] fix tests

---
 .../admin_api/admin_api_controller_test.exs   | 176 ++++++++++--------
 1 file changed, 96 insertions(+), 80 deletions(-)

diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs
index c15c67e31..43dcf945a 100644
--- a/test/web/admin_api/admin_api_controller_test.exs
+++ b/test/web/admin_api/admin_api_controller_test.exs
@@ -437,27 +437,31 @@ test "renders users array for the first page", %{conn: conn, admin: admin} do
       user = insert(:user, local: false, tags: ["foo", "bar"])
       conn = get(conn, "/api/pleroma/admin/users?page=1")
 
+      users =
+        [
+          %{
+            "deactivated" => admin.info.deactivated,
+            "id" => admin.id,
+            "nickname" => admin.nickname,
+            "roles" => %{"admin" => true, "moderator" => false},
+            "local" => true,
+            "tags" => []
+          },
+          %{
+            "deactivated" => user.info.deactivated,
+            "id" => user.id,
+            "nickname" => user.nickname,
+            "roles" => %{"admin" => false, "moderator" => false},
+            "local" => false,
+            "tags" => ["foo", "bar"]
+          }
+        ]
+        |> Enum.sort_by(& &1["nickname"])
+
       assert json_response(conn, 200) == %{
                "count" => 2,
                "page_size" => 50,
-               "users" => [
-                 %{
-                   "deactivated" => admin.info.deactivated,
-                   "id" => admin.id,
-                   "nickname" => admin.nickname,
-                   "roles" => %{"admin" => true, "moderator" => false},
-                   "local" => true,
-                   "tags" => []
-                 },
-                 %{
-                   "deactivated" => user.info.deactivated,
-                   "id" => user.id,
-                   "nickname" => user.nickname,
-                   "roles" => %{"admin" => false, "moderator" => false},
-                   "local" => false,
-                   "tags" => ["foo", "bar"]
-                 }
-               ]
+               "users" => users
              }
     end
 
@@ -659,35 +663,39 @@ test "only local users with no query", %{admin: old_admin} do
         |> assign(:user, admin)
         |> get("/api/pleroma/admin/users?filters=local")
 
+      users =
+        [
+          %{
+            "deactivated" => user.info.deactivated,
+            "id" => user.id,
+            "nickname" => user.nickname,
+            "roles" => %{"admin" => false, "moderator" => false},
+            "local" => true,
+            "tags" => []
+          },
+          %{
+            "deactivated" => admin.info.deactivated,
+            "id" => admin.id,
+            "nickname" => admin.nickname,
+            "roles" => %{"admin" => true, "moderator" => false},
+            "local" => true,
+            "tags" => []
+          },
+          %{
+            "deactivated" => false,
+            "id" => old_admin.id,
+            "local" => true,
+            "nickname" => old_admin.nickname,
+            "roles" => %{"admin" => true, "moderator" => false},
+            "tags" => []
+          }
+        ]
+        |> Enum.sort_by(& &1["nickname"])
+
       assert json_response(conn, 200) == %{
                "count" => 3,
                "page_size" => 50,
-               "users" => [
-                 %{
-                   "deactivated" => user.info.deactivated,
-                   "id" => user.id,
-                   "nickname" => user.nickname,
-                   "roles" => %{"admin" => false, "moderator" => false},
-                   "local" => true,
-                   "tags" => []
-                 },
-                 %{
-                   "deactivated" => admin.info.deactivated,
-                   "id" => admin.id,
-                   "nickname" => admin.nickname,
-                   "roles" => %{"admin" => true, "moderator" => false},
-                   "local" => true,
-                   "tags" => []
-                 },
-                 %{
-                   "deactivated" => false,
-                   "id" => old_admin.id,
-                   "local" => true,
-                   "nickname" => old_admin.nickname,
-                   "roles" => %{"admin" => true, "moderator" => false},
-                   "tags" => []
-                 }
-               ]
+               "users" => users
              }
     end
 
@@ -698,27 +706,31 @@ test "load only admins", %{conn: conn, admin: admin} do
 
       conn = get(conn, "/api/pleroma/admin/users?filters=is_admin")
 
+      users =
+        [
+          %{
+            "deactivated" => false,
+            "id" => admin.id,
+            "nickname" => admin.nickname,
+            "roles" => %{"admin" => true, "moderator" => false},
+            "local" => admin.local,
+            "tags" => []
+          },
+          %{
+            "deactivated" => false,
+            "id" => second_admin.id,
+            "nickname" => second_admin.nickname,
+            "roles" => %{"admin" => true, "moderator" => false},
+            "local" => second_admin.local,
+            "tags" => []
+          }
+        ]
+        |> Enum.sort_by(& &1["nickname"])
+
       assert json_response(conn, 200) == %{
                "count" => 2,
                "page_size" => 50,
-               "users" => [
-                 %{
-                   "deactivated" => false,
-                   "id" => admin.id,
-                   "nickname" => admin.nickname,
-                   "roles" => %{"admin" => true, "moderator" => false},
-                   "local" => admin.local,
-                   "tags" => []
-                 },
-                 %{
-                   "deactivated" => false,
-                   "id" => second_admin.id,
-                   "nickname" => second_admin.nickname,
-                   "roles" => %{"admin" => true, "moderator" => false},
-                   "local" => second_admin.local,
-                   "tags" => []
-                 }
-               ]
+               "users" => users
              }
     end
 
@@ -753,27 +765,31 @@ test "load users with tags list", %{conn: conn} do
 
       conn = get(conn, "/api/pleroma/admin/users?tags[]=first&tags[]=second")
 
+      users =
+        [
+          %{
+            "deactivated" => false,
+            "id" => user1.id,
+            "nickname" => user1.nickname,
+            "roles" => %{"admin" => false, "moderator" => false},
+            "local" => user1.local,
+            "tags" => ["first"]
+          },
+          %{
+            "deactivated" => false,
+            "id" => user2.id,
+            "nickname" => user2.nickname,
+            "roles" => %{"admin" => false, "moderator" => false},
+            "local" => user2.local,
+            "tags" => ["second"]
+          }
+        ]
+        |> Enum.sort_by(& &1["nickname"])
+
       assert json_response(conn, 200) == %{
                "count" => 2,
                "page_size" => 50,
-               "users" => [
-                 %{
-                   "deactivated" => false,
-                   "id" => user1.id,
-                   "nickname" => user1.nickname,
-                   "roles" => %{"admin" => false, "moderator" => false},
-                   "local" => user1.local,
-                   "tags" => ["first"]
-                 },
-                 %{
-                   "deactivated" => false,
-                   "id" => user2.id,
-                   "nickname" => user2.nickname,
-                   "roles" => %{"admin" => false, "moderator" => false},
-                   "local" => user2.local,
-                   "tags" => ["second"]
-                 }
-               ]
+               "users" => users
              }
     end
 

From 6aec0d1b58c12db2fa669fcff043338174290a35 Mon Sep 17 00:00:00 2001
From: kaniini <nenolod@gmail.com>
Date: Wed, 29 May 2019 22:10:16 +0000
Subject: [PATCH 56/64] Revert "Merge branch
 'feature/search-authenticated-only' into 'develop'"

This reverts merge request !1209
---
 lib/pleroma/web/router.ex | 20 ++++++--------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 08c74a742..352268b96 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -414,12 +414,7 @@ defmodule Pleroma.Web.Router do
 
     get("/trends", MastodonAPIController, :empty_array)
 
-    scope [] do
-      pipe_through(:oauth_read)
-
-      get("/search", MastodonAPIController, :search)
-      get("/accounts/search", MastodonAPIController, :account_search)
-    end
+    get("/accounts/search", MastodonAPIController, :account_search)
 
     scope [] do
       pipe_through(:oauth_read_or_public)
@@ -436,12 +431,14 @@ defmodule Pleroma.Web.Router do
       get("/accounts/:id/following", MastodonAPIController, :following)
       get("/accounts/:id", MastodonAPIController, :user)
 
+      get("/search", MastodonAPIController, :search)
+
       get("/pleroma/accounts/:id/favourites", MastodonAPIController, :user_favourites)
     end
   end
 
   scope "/api/v2", Pleroma.Web.MastodonAPI do
-    pipe_through([:api, :oauth_read])
+    pipe_through([:api, :oauth_read_or_public])
     get("/search", MastodonAPIController, :search2)
   end
 
@@ -483,13 +480,8 @@ defmodule Pleroma.Web.Router do
       get("/statuses/show/:id", TwitterAPI.Controller, :fetch_status)
       get("/statusnet/conversation/:id", TwitterAPI.Controller, :fetch_conversation)
 
-      get("/statusnet/tags/timeline/:tag", TwitterAPI.Controller, :public_and_external_timeline)
-    end
-
-    scope [] do
-      pipe_through(:oauth_read)
-
       get("/search", TwitterAPI.Controller, :search)
+      get("/statusnet/tags/timeline/:tag", TwitterAPI.Controller, :public_and_external_timeline)
     end
   end
 
@@ -508,7 +500,7 @@ defmodule Pleroma.Web.Router do
   end
 
   scope "/api", Pleroma.Web, as: :twitter_api_search do
-    pipe_through([:api, :oauth_read])
+    pipe_through([:api, :oauth_read_or_public])
     get("/pleroma/search_user", TwitterAPI.Controller, :search_user)
   end
 

From 99f70c7e2011ab29746a8a61d3a4b354f4513045 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Thu, 30 May 2019 15:33:58 +0700
Subject: [PATCH 57/64] Use Pleroma.Config everywhere

---
 lib/healthcheck.ex                            |  4 +-
 lib/pleroma/emoji.ex                          |  4 +-
 lib/pleroma/html.ex                           | 20 ++----
 lib/pleroma/http/http.ex                      |  7 +-
 lib/pleroma/plugs/federating_plug.ex          |  2 +-
 lib/pleroma/reverse_proxy.ex                  |  2 +-
 lib/pleroma/user.ex                           |  4 +-
 lib/pleroma/web/activity_pub/activity_pub.ex  | 12 ++--
 .../activity_pub/activity_pub_controller.ex   |  2 +-
 lib/pleroma/web/activity_pub/mrf.ex           |  4 +-
 lib/pleroma/web/endpoint.ex                   |  2 +-
 lib/pleroma/web/media_proxy/media_proxy.ex    | 34 +++++-----
 .../web/nodeinfo/nodeinfo_controller.ex       | 66 ++++++++-----------
 lib/pleroma/web/templates/layout/app.html.eex |  4 +-
 .../mastodon_api/mastodon/index.html.eex      |  2 +-
 .../web/twitter_api/twitter_api_controller.ex |  2 +-
 test/web/plugs/federating_plug_test.exs       | 12 +---
 .../twitter_api_controller_test.exs           | 48 +++-----------
 18 files changed, 80 insertions(+), 151 deletions(-)

diff --git a/lib/healthcheck.ex b/lib/healthcheck.ex
index 646fb3b9d..32aafc210 100644
--- a/lib/healthcheck.ex
+++ b/lib/healthcheck.ex
@@ -29,13 +29,13 @@ def system_info do
   end
 
   defp assign_db_info(healthcheck) do
-    database = Application.get_env(:pleroma, Repo)[:database]
+    database = Pleroma.Config.get([Repo, :database])
 
     query =
       "select state, count(pid) from pg_stat_activity where datname = '#{database}' group by state;"
 
     result = Repo.query!(query)
-    pool_size = Application.get_env(:pleroma, Repo)[:pool_size]
+    pool_size = Pleroma.Config.get([Repo, :pool_size])
 
     db_info =
       Enum.reduce(result.rows, %{active: 0, idle: 0}, fn [state, cnt], states ->
diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex
index 6390cce4c..7d12eff7f 100644
--- a/lib/pleroma/emoji.ex
+++ b/lib/pleroma/emoji.ex
@@ -22,7 +22,7 @@ defmodule Pleroma.Emoji do
 
   @ets __MODULE__.Ets
   @ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}]
-  @groups Application.get_env(:pleroma, :emoji)[:groups]
+  @groups Pleroma.Config.get([:emoji, :groups])
 
   @doc false
   def start_link do
@@ -112,7 +112,7 @@ defp load do
 
     # Compat thing for old custom emoji handling & default emoji,
     # it should run even if there are no emoji packs
-    shortcode_globs = Application.get_env(:pleroma, :emoji)[:shortcode_globs] || []
+    shortcode_globs = Pleroma.Config.get([:emoji, :shortcode_globs], [])
 
     emojis =
       (load_from_file("config/emoji.txt") ++
diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex
index d1da746de..e5e78ee4f 100644
--- a/lib/pleroma/html.ex
+++ b/lib/pleroma/html.ex
@@ -104,7 +104,6 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do
   paragraphs, breaks and links are allowed through the filter.
   """
 
-  @markup Application.get_env(:pleroma, :markup)
   @valid_schemes Pleroma.Config.get([:uri_schemes, :valid_schemes], [])
 
   require HtmlSanitizeEx.Scrubber.Meta
@@ -142,9 +141,7 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do
   Meta.allow_tag_with_these_attributes("span", [])
 
   # allow inline images for custom emoji
-  @allow_inline_images Keyword.get(@markup, :allow_inline_images)
-
-  if @allow_inline_images do
+  if Pleroma.Config.get([:markup, :allow_inline_images]) do
     # restrict img tags to http/https only, because of MediaProxy.
     Meta.allow_tag_with_uri_attributes("img", ["src"], ["http", "https"])
 
@@ -168,7 +165,6 @@ defmodule Pleroma.HTML.Scrubber.Default do
   # credo:disable-for-previous-line
   # No idea how to fix this one…
 
-  @markup Application.get_env(:pleroma, :markup)
   @valid_schemes Pleroma.Config.get([:uri_schemes, :valid_schemes], [])
 
   Meta.remove_cdata_sections_before_scrub()
@@ -213,7 +209,7 @@ defmodule Pleroma.HTML.Scrubber.Default do
   Meta.allow_tag_with_this_attribute_values("span", "class", ["h-card"])
   Meta.allow_tag_with_these_attributes("span", [])
 
-  @allow_inline_images Keyword.get(@markup, :allow_inline_images)
+  @allow_inline_images Pleroma.Config.get([:markup, :allow_inline_images])
 
   if @allow_inline_images do
     # restrict img tags to http/https only, because of MediaProxy.
@@ -228,9 +224,7 @@ defmodule Pleroma.HTML.Scrubber.Default do
     ])
   end
 
-  @allow_tables Keyword.get(@markup, :allow_tables)
-
-  if @allow_tables do
+  if Pleroma.Config.get([:markup, :allow_tables]) do
     Meta.allow_tag_with_these_attributes("table", [])
     Meta.allow_tag_with_these_attributes("tbody", [])
     Meta.allow_tag_with_these_attributes("td", [])
@@ -239,9 +233,7 @@ defmodule Pleroma.HTML.Scrubber.Default do
     Meta.allow_tag_with_these_attributes("tr", [])
   end
 
-  @allow_headings Keyword.get(@markup, :allow_headings)
-
-  if @allow_headings do
+  if Pleroma.Config.get([:markup, :allow_headings]) do
     Meta.allow_tag_with_these_attributes("h1", [])
     Meta.allow_tag_with_these_attributes("h2", [])
     Meta.allow_tag_with_these_attributes("h3", [])
@@ -249,9 +241,7 @@ defmodule Pleroma.HTML.Scrubber.Default do
     Meta.allow_tag_with_these_attributes("h5", [])
   end
 
-  @allow_fonts Keyword.get(@markup, :allow_fonts)
-
-  if @allow_fonts do
+  if Pleroma.Config.get([:markup, :allow_fonts]) do
     Meta.allow_tag_with_these_attributes("font", ["face"])
   end
 
diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex
index c5f720bc9..c96ee7353 100644
--- a/lib/pleroma/http/http.ex
+++ b/lib/pleroma/http/http.ex
@@ -65,12 +65,9 @@ defp process_sni_options(options, url) do
   end
 
   def process_request_options(options) do
-    config = Application.get_env(:pleroma, :http, [])
-    proxy = Keyword.get(config, :proxy_url, nil)
-
-    case proxy do
+    case Pleroma.Config.get([:http, :proxy_url]) do
       nil -> options
-      _ -> options ++ [proxy: proxy]
+      proxy -> options ++ [proxy: proxy]
     end
   end
 
diff --git a/lib/pleroma/plugs/federating_plug.ex b/lib/pleroma/plugs/federating_plug.ex
index effc154bf..4dc4e9279 100644
--- a/lib/pleroma/plugs/federating_plug.ex
+++ b/lib/pleroma/plugs/federating_plug.ex
@@ -10,7 +10,7 @@ def init(options) do
   end
 
   def call(conn, _opts) do
-    if Keyword.get(Application.get_env(:pleroma, :instance), :federating) do
+    if Pleroma.Config.get([:instance, :federating]) do
       conn
     else
       conn
diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex
index 6e5feb4c3..983e156f5 100644
--- a/lib/pleroma/reverse_proxy.ex
+++ b/lib/pleroma/reverse_proxy.ex
@@ -61,7 +61,7 @@ defmodule Pleroma.ReverseProxy do
   * `http`: options for [hackney](https://github.com/benoitc/hackney).
 
   """
-  @hackney Application.get_env(:pleroma, :hackney, :hackney)
+  @hackney Pleroma.Config.get(:hackney, :hackney)
 
   @default_hackney_options []
 
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 653dec95f..5c91dc7d4 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -366,9 +366,7 @@ def follow_all(follower, followeds) do
   end
 
   def follow(%User{} = follower, %User{info: info} = followed) do
-    user_config = Application.get_env(:pleroma, :user)
-    deny_follow_blocked = Keyword.get(user_config, :deny_follow_blocked)
-
+    deny_follow_blocked = Pleroma.Config.get([:user, :deny_follow_blocked])
     ap_followers = followed.follower_address
 
     cond do
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index aa0229db7..bdd7e78d2 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -399,16 +399,12 @@ def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, local \\ tru
   end
 
   def block(blocker, blocked, activity_id \\ nil, local \\ true) do
-    ap_config = Application.get_env(:pleroma, :activitypub)
-    unfollow_blocked = Keyword.get(ap_config, :unfollow_blocked)
-    outgoing_blocks = Keyword.get(ap_config, :outgoing_blocks)
+    outgoing_blocks = Pleroma.Config.get([:activitypub, :outgoing_blocks])
+    unfollow_blocked = Pleroma.Config.get([:activitypub, :unfollow_blocked])
 
-    with true <- unfollow_blocked do
+    if unfollow_blocked do
       follow_activity = fetch_latest_follow(blocker, blocked)
-
-      if follow_activity do
-        unfollow(blocker, blocked, nil, local)
-      end
+      if follow_activity, do: unfollow(blocker, blocked, nil, local)
     end
 
     with true <- outgoing_blocks,
diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index ad2ca1e54..0182bda46 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -27,7 +27,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   plug(:relay_active? when action in [:relay])
 
   def relay_active?(conn, _) do
-    if Keyword.get(Application.get_env(:pleroma, :instance), :allow_relay) do
+    if Pleroma.Config.get([:instance, :allow_relay]) do
       conn
     else
       conn
diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex
index 1aaa20050..3bf7955f3 100644
--- a/lib/pleroma/web/activity_pub/mrf.ex
+++ b/lib/pleroma/web/activity_pub/mrf.ex
@@ -17,9 +17,7 @@ def filter(object) do
   end
 
   def get_policies do
-    Application.get_env(:pleroma, :instance, [])
-    |> Keyword.get(:rewrite_policy, [])
-    |> get_policies()
+    Pleroma.Config.get([:instance, :rewrite_policy], []) |> get_policies()
   end
 
   defp get_policies(policy) when is_atom(policy), do: [policy]
diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex
index 8cd7a2270..bd76e4295 100644
--- a/lib/pleroma/web/endpoint.ex
+++ b/lib/pleroma/web/endpoint.ex
@@ -66,7 +66,7 @@ defmodule Pleroma.Web.Endpoint do
     parsers: [:urlencoded, :multipart, :json],
     pass: ["*/*"],
     json_decoder: Jason,
-    length: Application.get_env(:pleroma, :instance) |> Keyword.get(:upload_limit),
+    length: Pleroma.Config.get([:instance, :upload_limit]),
     body_reader: {Pleroma.Web.Plugs.DigestPlug, :read_body, []}
   )
 
diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex
index 5762e767b..cee6d8481 100644
--- a/lib/pleroma/web/media_proxy/media_proxy.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy.ex
@@ -12,25 +12,27 @@ def url(""), do: nil
   def url("/" <> _ = url), do: url
 
   def url(url) do
-    config = Application.get_env(:pleroma, :media_proxy, [])
-    domain = URI.parse(url).host
-
-    cond do
-      !Keyword.get(config, :enabled, false) or String.starts_with?(url, Pleroma.Web.base_url()) ->
-        url
-
-      Enum.any?(Pleroma.Config.get([:media_proxy, :whitelist]), fn pattern ->
-        String.equivalent?(domain, pattern)
-      end) ->
-        url
-
-      true ->
-        encode_url(url)
+    if !enabled?() or local?(url) or whitelisted?(url) do
+      url
+    else
+      encode_url(url)
     end
   end
 
+  defp enabled?, do: Pleroma.Config.get([:media_proxy, :enabled], false)
+
+  defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())
+
+  defp whitelisted?(url) do
+    %{host: domain} = URI.parse(url)
+
+    Enum.any?(Pleroma.Config.get([:media_proxy, :whitelist]), fn pattern ->
+      String.equivalent?(domain, pattern)
+    end)
+  end
+
   def encode_url(url) do
-    secret = Application.get_env(:pleroma, Pleroma.Web.Endpoint)[:secret_key_base]
+    secret = Pleroma.Config.get([Pleroma.Web.Endpoint, :secret_key_base])
 
     # Must preserve `%2F` for compatibility with S3
     # https://git.pleroma.social/pleroma/pleroma/issues/580
@@ -52,7 +54,7 @@ def encode_url(url) do
   end
 
   def decode_url(sig, url) do
-    secret = Application.get_env(:pleroma, Pleroma.Web.Endpoint)[:secret_key_base]
+    secret = Pleroma.Config.get([Pleroma.Web.Endpoint, :secret_key_base])
     sig = Base.url_decode64!(sig, @base64_opts)
     local_sig = :crypto.hmac(:sha, secret, url)
 
diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex
index 45f90c579..59f3d4e11 100644
--- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex
+++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex
@@ -32,20 +32,15 @@ def schemas(conn, _params) do
   # returns a nodeinfo 2.0 map, since 2.1 just adds a repository field
   # under software.
   def raw_nodeinfo do
-    instance = Application.get_env(:pleroma, :instance)
-    media_proxy = Application.get_env(:pleroma, :media_proxy)
-    suggestions = Application.get_env(:pleroma, :suggestions)
-    chat = Application.get_env(:pleroma, :chat)
-    gopher = Application.get_env(:pleroma, :gopher)
     stats = Stats.get_stats()
 
     mrf_simple =
-      Application.get_env(:pleroma, :mrf_simple)
+      Config.get(:mrf_simple)
       |> Enum.into(%{})
 
     # This horror is needed to convert regex sigils to strings
     mrf_keyword =
-      Application.get_env(:pleroma, :mrf_keyword, [])
+      Config.get(:mrf_keyword, [])
       |> Enum.map(fn {key, value} ->
         {key,
          Enum.map(value, fn
@@ -74,14 +69,7 @@ def raw_nodeinfo do
       MRF.get_policies()
       |> Enum.map(fn policy -> to_string(policy) |> String.split(".") |> List.last() end)
 
-    quarantined = Keyword.get(instance, :quarantined_instances)
-
-    quarantined =
-      if is_list(quarantined) do
-        quarantined
-      else
-        []
-      end
+    quarantined = Config.get([:instance, :quarantined_instances], [])
 
     staff_accounts =
       User.all_superusers()
@@ -92,7 +80,7 @@ def raw_nodeinfo do
       |> Enum.into(%{}, fn {k, v} -> {k, length(v)} end)
 
     federation_response =
-      if Keyword.get(instance, :mrf_transparency) do
+      if Config.get([:instance, :mrf_transparency]) do
         %{
           mrf_policies: mrf_policies,
           mrf_simple: mrf_simple,
@@ -109,22 +97,22 @@ def raw_nodeinfo do
         "pleroma_api",
         "mastodon_api",
         "mastodon_api_streaming",
-        if Keyword.get(media_proxy, :enabled) do
+        if Config.get([:media_proxy, :enabled]) do
           "media_proxy"
         end,
-        if Keyword.get(gopher, :enabled) do
+        if Config.get([:gopher, :enabled]) do
           "gopher"
         end,
-        if Keyword.get(chat, :enabled) do
+        if Config.get([:chat, :enabled]) do
           "chat"
         end,
-        if Keyword.get(suggestions, :enabled) do
+        if Config.get([:suggestions, :enabled]) do
           "suggestions"
         end,
-        if Keyword.get(instance, :allow_relay) do
+        if Config.get([:instance, :allow_relay]) do
           "relay"
         end,
-        if Keyword.get(instance, :safe_dm_mentions) do
+        if Config.get([:instance, :safe_dm_mentions]) do
           "safe_dm_mentions"
         end
       ]
@@ -141,7 +129,7 @@ def raw_nodeinfo do
         inbound: [],
         outbound: []
       },
-      openRegistrations: Keyword.get(instance, :registrations_open),
+      openRegistrations: Config.get([:instance, :registrations_open]),
       usage: %{
         users: %{
           total: stats.user_count || 0
@@ -149,29 +137,29 @@ def raw_nodeinfo do
         localPosts: stats.status_count || 0
       },
       metadata: %{
-        nodeName: Keyword.get(instance, :name),
-        nodeDescription: Keyword.get(instance, :description),
-        private: !Keyword.get(instance, :public, true),
+        nodeName: Config.get([:instance, :name]),
+        nodeDescription: Config.get([:instance, :description]),
+        private: !Config.get([:instance, :public], true),
         suggestions: %{
-          enabled: Keyword.get(suggestions, :enabled, false),
-          thirdPartyEngine: Keyword.get(suggestions, :third_party_engine, ""),
-          timeout: Keyword.get(suggestions, :timeout, 5000),
-          limit: Keyword.get(suggestions, :limit, 23),
-          web: Keyword.get(suggestions, :web, "")
+          enabled: Config.get([:suggestions, :enabled], false),
+          thirdPartyEngine: Config.get([:suggestions, :third_party_engine], ""),
+          timeout: Config.get([:suggestions, :timeout], 5000),
+          limit: Config.get([:suggestions, :limit], 23),
+          web: Config.get([:suggestions, :web], "")
         },
         staffAccounts: staff_accounts,
         federation: federation_response,
-        postFormats: Keyword.get(instance, :allowed_post_formats),
+        postFormats: Config.get([:instance, :allowed_post_formats]),
         uploadLimits: %{
-          general: Keyword.get(instance, :upload_limit),
-          avatar: Keyword.get(instance, :avatar_upload_limit),
-          banner: Keyword.get(instance, :banner_upload_limit),
-          background: Keyword.get(instance, :background_upload_limit)
+          general: Config.get([:instance, :upload_limit]),
+          avatar: Config.get([:instance, :avatar_upload_limit]),
+          banner: Config.get([:instance, :banner_upload_limit]),
+          background: Config.get([:instance, :background_upload_limit])
         },
-        accountActivationRequired: Keyword.get(instance, :account_activation_required, false),
-        invitesEnabled: Keyword.get(instance, :invites_enabled, false),
+        accountActivationRequired: Config.get([:instance, :account_activation_required], false),
+        invitesEnabled: Config.get([:instance, :invites_enabled], false),
         features: features,
-        restrictedNicknames: Pleroma.Config.get([Pleroma.User, :restricted_nicknames])
+        restrictedNicknames: Config.get([Pleroma.User, :restricted_nicknames])
       }
     }
   end
diff --git a/lib/pleroma/web/templates/layout/app.html.eex b/lib/pleroma/web/templates/layout/app.html.eex
index 3389c91cc..85ec4d76c 100644
--- a/lib/pleroma/web/templates/layout/app.html.eex
+++ b/lib/pleroma/web/templates/layout/app.html.eex
@@ -4,7 +4,7 @@
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width,initial-scale=1,minimal-ui" />
     <title>
-    <%= Application.get_env(:pleroma, :instance)[:name] %>
+    <%= Pleroma.Config.get([:instance, :name]) %>
     </title>
     <style>
       body {
@@ -194,7 +194,7 @@
   </head>
   <body>
     <div class="container">
-      <h1><%= Application.get_env(:pleroma, :instance)[:name] %></h1>
+      <h1><%= Pleroma.Config.get([:instance, :name]) %></h1>
       <%= render @view_module, @view_template, assigns %>
     </div>
   </body>
diff --git a/lib/pleroma/web/templates/mastodon_api/mastodon/index.html.eex b/lib/pleroma/web/templates/mastodon_api/mastodon/index.html.eex
index 5659c7828..ac63811d1 100644
--- a/lib/pleroma/web/templates/mastodon_api/mastodon/index.html.eex
+++ b/lib/pleroma/web/templates/mastodon_api/mastodon/index.html.eex
@@ -4,7 +4,7 @@
 <meta charset='utf-8'>
 <meta content='width=device-width, initial-scale=1' name='viewport'>
 <title>
-<%= Application.get_env(:pleroma, :instance)[:name] %>
+<%= Pleroma.Config.get([:instance, :name]) %>
 </title>
 <link rel="icon" type="image/png" href="/favicon.png"/>
 <script crossorigin='anonymous' src="/packs/locales.js"></script>
diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
index 31e86685a..1b6b33e69 100644
--- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
@@ -728,7 +728,7 @@ defp forbidden_json_reply(conn, error_message) do
   def only_if_public_instance(%{assigns: %{user: %User{}}} = conn, _), do: conn
 
   def only_if_public_instance(conn, _) do
-    if Keyword.get(Application.get_env(:pleroma, :instance), :public) do
+    if Pleroma.Config.get([:instance, :public]) do
       conn
     else
       conn
diff --git a/test/web/plugs/federating_plug_test.exs b/test/web/plugs/federating_plug_test.exs
index 612db7e32..530562325 100644
--- a/test/web/plugs/federating_plug_test.exs
+++ b/test/web/plugs/federating_plug_test.exs
@@ -6,11 +6,7 @@ defmodule Pleroma.Web.FederatingPlugTest do
   use Pleroma.Web.ConnCase
 
   test "returns and halt the conn when federating is disabled" do
-    instance =
-      Application.get_env(:pleroma, :instance)
-      |> Keyword.put(:federating, false)
-
-    Application.put_env(:pleroma, :instance, instance)
+    Pleroma.Config.put([:instance, :federating], false)
 
     conn =
       build_conn()
@@ -19,11 +15,7 @@ test "returns and halt the conn when federating is disabled" do
     assert conn.status == 404
     assert conn.halted
 
-    instance =
-      Application.get_env(:pleroma, :instance)
-      |> Keyword.put(:federating, true)
-
-    Application.put_env(:pleroma, :instance, instance)
+    Pleroma.Config.put([:instance, :federating], true)
   end
 
   test "does nothing when federating is enabled" do
diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs
index e194f14fb..bcd0f522d 100644
--- a/test/web/twitter_api/twitter_api_controller_test.exs
+++ b/test/web/twitter_api/twitter_api_controller_test.exs
@@ -144,41 +144,25 @@ test "returns statuses", %{conn: conn} do
     end
 
     test "returns 403 to unauthenticated request when the instance is not public", %{conn: conn} do
-      instance =
-        Application.get_env(:pleroma, :instance)
-        |> Keyword.put(:public, false)
-
-      Application.put_env(:pleroma, :instance, instance)
+      Pleroma.Config.put([:instance, :public], false)
 
       conn
       |> get("/api/statuses/public_timeline.json")
       |> json_response(403)
 
-      instance =
-        Application.get_env(:pleroma, :instance)
-        |> Keyword.put(:public, true)
-
-      Application.put_env(:pleroma, :instance, instance)
+      Pleroma.Config.put([:instance, :public], true)
     end
 
     test "returns 200 to authenticated request when the instance is not public",
          %{conn: conn, user: user} do
-      instance =
-        Application.get_env(:pleroma, :instance)
-        |> Keyword.put(:public, false)
-
-      Application.put_env(:pleroma, :instance, instance)
+      Pleroma.Config.put([:instance, :public], false)
 
       conn
       |> with_credentials(user.nickname, "test")
       |> get("/api/statuses/public_timeline.json")
       |> json_response(200)
 
-      instance =
-        Application.get_env(:pleroma, :instance)
-        |> Keyword.put(:public, true)
-
-      Application.put_env(:pleroma, :instance, instance)
+      Pleroma.Config.put([:instance, :public], true)
     end
 
     test "returns 200 to unauthenticated request when the instance is public", %{conn: conn} do
@@ -214,41 +198,25 @@ test "returns 200 to authenticated request when the instance is public",
     setup [:valid_user]
 
     test "returns 403 to unauthenticated request when the instance is not public", %{conn: conn} do
-      instance =
-        Application.get_env(:pleroma, :instance)
-        |> Keyword.put(:public, false)
-
-      Application.put_env(:pleroma, :instance, instance)
+      Pleroma.Config.put([:instance, :public], false)
 
       conn
       |> get("/api/statuses/public_and_external_timeline.json")
       |> json_response(403)
 
-      instance =
-        Application.get_env(:pleroma, :instance)
-        |> Keyword.put(:public, true)
-
-      Application.put_env(:pleroma, :instance, instance)
+      Pleroma.Config.put([:instance, :public], true)
     end
 
     test "returns 200 to authenticated request when the instance is not public",
          %{conn: conn, user: user} do
-      instance =
-        Application.get_env(:pleroma, :instance)
-        |> Keyword.put(:public, false)
-
-      Application.put_env(:pleroma, :instance, instance)
+      Pleroma.Config.put([:instance, :public], false)
 
       conn
       |> with_credentials(user.nickname, "test")
       |> get("/api/statuses/public_and_external_timeline.json")
       |> json_response(200)
 
-      instance =
-        Application.get_env(:pleroma, :instance)
-        |> Keyword.put(:public, true)
-
-      Application.put_env(:pleroma, :instance, instance)
+      Pleroma.Config.put([:instance, :public], true)
     end
 
     test "returns 200 to unauthenticated request when the instance is public", %{conn: conn} do

From 1690be991ede33998203226998b8023a247a4f84 Mon Sep 17 00:00:00 2001
From: Sergey Suprunenko <suprunenko.s@gmail.com>
Date: Thu, 30 May 2019 21:03:31 +0000
Subject: [PATCH 58/64] Replace missing non-nullable Card attributes with empty
 strings

---
 CHANGELOG.md                                  |  1 +
 .../web/mastodon_api/views/status_view.ex     |  4 +-
 lib/pleroma/web/rich_media/parser.ex          |  5 +-
 .../fixtures/rich_media/ogp-missing-data.html |  8 ++
 test/fixtures/rich_media/ogp.html             |  1 +
 test/support/http_request_mock.ex             |  8 ++
 .../mastodon_api_controller_test.exs          | 81 ++++++++++++++-----
 test/web/rich_media/parser_test.exs           |  2 +
 8 files changed, 87 insertions(+), 23 deletions(-)
 create mode 100644 test/fixtures/rich_media/ogp-missing-data.html

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f689160e9..b4abb6eb8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -112,6 +112,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: Correct `reblogged`, `favourited`, and `bookmarked` values in the reblog status JSON
 - Mastodon API: Exposing default scope of the user to anyone
 - Mastodon API: Make `irreversible` field default to `false` [`POST /api/v1/filters`]
+- Mastodon API: Replace missing non-nullable Card attributes with empty strings
 - User-Agent is now sent correctly for all HTTP requests.
 - MRF: Simple policy now properly delists imported or relayed statuses
 
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index e55f9b96e..84ab20a1c 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -290,8 +290,8 @@ def render("card.json", %{rich_media: rich_media, page_url: page_url}) do
       provider_url: page_url_data.scheme <> "://" <> page_url_data.host,
       url: page_url,
       image: image_url |> MediaProxy.url(),
-      title: rich_media[:title],
-      description: rich_media[:description],
+      title: rich_media[:title] || "",
+      description: rich_media[:description] || "",
       pleroma: %{
         opengraph: rich_media
       }
diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index 62e8fa610..e4595800c 100644
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -37,7 +37,10 @@ defp parse_url(url) do
     try do
       {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options)
 
-      html |> maybe_parse() |> clean_parsed_data() |> check_parsed_data()
+      html
+      |> maybe_parse()
+      |> clean_parsed_data()
+      |> check_parsed_data()
     rescue
       e ->
         {:error, "Parsing error: #{inspect(e)}"}
diff --git a/test/fixtures/rich_media/ogp-missing-data.html b/test/fixtures/rich_media/ogp-missing-data.html
new file mode 100644
index 000000000..5746dc2f4
--- /dev/null
+++ b/test/fixtures/rich_media/ogp-missing-data.html
@@ -0,0 +1,8 @@
+<html prefix="og: http://ogp.me/ns#">
+  <head>
+    <title>Pleroma</title>
+    <meta property="og:title" content="Pleroma" />
+    <meta property="og:type" content="website" />
+    <meta property="og:url" content="https://pleroma.social/" />
+  </head>
+</html>
diff --git a/test/fixtures/rich_media/ogp.html b/test/fixtures/rich_media/ogp.html
index c886b5871..4b5a33595 100644
--- a/test/fixtures/rich_media/ogp.html
+++ b/test/fixtures/rich_media/ogp.html
@@ -5,5 +5,6 @@
     <meta property="og:type" content="video.movie" />
     <meta property="og:url" content="http://www.imdb.com/title/tt0117500/" />
     <meta property="og:image" content="http://ia.media-imdb.com/images/rock.jpg" />
+    <meta property="og:description" content="Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer.">
   </head>
 </html>
diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex
index 5b355bfe6..66d7d5ba9 100644
--- a/test/support/http_request_mock.ex
+++ b/test/support/http_request_mock.ex
@@ -728,6 +728,14 @@ def get("http://example.com/ogp", _, _, _) do
     {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}}
   end
 
+  def get("http://example.com/ogp-missing-data", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/rich_media/ogp-missing-data.html")
+     }}
+  end
+
   def get("http://example.com/malformed", _, _, _) do
     {:ok,
      %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/malformed-data.html")}}
diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs
index 1d9f5a816..93ef630f2 100644
--- a/test/web/mastodon_api/mastodon_api_controller_test.exs
+++ b/test/web/mastodon_api/mastodon_api_controller_test.exs
@@ -2684,33 +2684,50 @@ test "max pinned statuses", %{conn: conn, user: user, activity: activity_one} do
                |> post("/api/v1/statuses/#{activity_two.id}/pin")
                |> json_response(400)
     end
+  end
 
-    test "Status rich-media Card", %{conn: conn, user: user} do
+  describe "cards" do
+    setup do
       Pleroma.Config.put([:rich_media, :enabled], true)
+
+      on_exit(fn ->
+        Pleroma.Config.put([:rich_media, :enabled], false)
+      end)
+
+      user = insert(:user)
+      %{user: user}
+    end
+
+    test "returns rich-media card", %{conn: conn, user: user} do
       {:ok, activity} = CommonAPI.post(user, %{"status" => "http://example.com/ogp"})
 
+      card_data = %{
+        "image" => "http://ia.media-imdb.com/images/rock.jpg",
+        "provider_name" => "www.imdb.com",
+        "provider_url" => "http://www.imdb.com",
+        "title" => "The Rock",
+        "type" => "link",
+        "url" => "http://www.imdb.com/title/tt0117500/",
+        "description" =>
+          "Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer.",
+        "pleroma" => %{
+          "opengraph" => %{
+            "image" => "http://ia.media-imdb.com/images/rock.jpg",
+            "title" => "The Rock",
+            "type" => "video.movie",
+            "url" => "http://www.imdb.com/title/tt0117500/",
+            "description" =>
+              "Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer."
+          }
+        }
+      }
+
       response =
         conn
         |> get("/api/v1/statuses/#{activity.id}/card")
         |> json_response(200)
 
-      assert response == %{
-               "image" => "http://ia.media-imdb.com/images/rock.jpg",
-               "provider_name" => "www.imdb.com",
-               "provider_url" => "http://www.imdb.com",
-               "title" => "The Rock",
-               "type" => "link",
-               "url" => "http://www.imdb.com/title/tt0117500/",
-               "description" => nil,
-               "pleroma" => %{
-                 "opengraph" => %{
-                   "image" => "http://ia.media-imdb.com/images/rock.jpg",
-                   "title" => "The Rock",
-                   "type" => "video.movie",
-                   "url" => "http://www.imdb.com/title/tt0117500/"
-                 }
-               }
-             }
+      assert response == card_data
 
       # works with private posts
       {:ok, activity} =
@@ -2722,9 +2739,33 @@ test "Status rich-media Card", %{conn: conn, user: user} do
         |> get("/api/v1/statuses/#{activity.id}/card")
         |> json_response(200)
 
-      assert response_two == response
+      assert response_two == card_data
+    end
 
-      Pleroma.Config.put([:rich_media, :enabled], false)
+    test "replaces missing description with an empty string", %{conn: conn, user: user} do
+      {:ok, activity} = CommonAPI.post(user, %{"status" => "http://example.com/ogp-missing-data"})
+
+      response =
+        conn
+        |> get("/api/v1/statuses/#{activity.id}/card")
+        |> json_response(:ok)
+
+      assert response == %{
+               "type" => "link",
+               "title" => "Pleroma",
+               "description" => "",
+               "image" => nil,
+               "provider_name" => "pleroma.social",
+               "provider_url" => "https://pleroma.social",
+               "url" => "https://pleroma.social/",
+               "pleroma" => %{
+                 "opengraph" => %{
+                   "title" => "Pleroma",
+                   "type" => "website",
+                   "url" => "https://pleroma.social/"
+                 }
+               }
+             }
     end
   end
 
diff --git a/test/web/rich_media/parser_test.exs b/test/web/rich_media/parser_test.exs
index 47b127cf9..3a9cc1854 100644
--- a/test/web/rich_media/parser_test.exs
+++ b/test/web/rich_media/parser_test.exs
@@ -44,6 +44,8 @@ test "parses ogp" do
               %{
                 image: "http://ia.media-imdb.com/images/rock.jpg",
                 title: "The Rock",
+                description:
+                  "Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer.",
                 type: "video.movie",
                 url: "http://www.imdb.com/title/tt0117500/"
               }}

From 75a37a68d4080a3781256cbe17b1e1b6c9944b18 Mon Sep 17 00:00:00 2001
From: feld <feld@feld.me>
Date: Thu, 30 May 2019 21:30:42 +0000
Subject: [PATCH 59/64] Install docs should clone from master branch

---
 docs/installation/alpine_linux_en.md | 2 +-
 docs/installation/arch_linux_en.md   | 2 +-
 docs/installation/centos7_en.md      | 2 +-
 docs/installation/debian_based_en.md | 2 +-
 docs/installation/debian_based_jp.md | 2 +-
 docs/installation/gentoo_en.md       | 2 +-
 docs/installation/netbsd_en.md       | 2 +-
 docs/installation/openbsd_en.md      | 2 +-
 docs/installation/openbsd_fi.md      | 2 +-
 9 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/docs/installation/alpine_linux_en.md b/docs/installation/alpine_linux_en.md
index c493816d6..e1d69c873 100644
--- a/docs/installation/alpine_linux_en.md
+++ b/docs/installation/alpine_linux_en.md
@@ -87,7 +87,7 @@ sudo adduser -S -s /bin/false -h /opt/pleroma -H pleroma
 ```shell
 sudo mkdir -p /opt/pleroma
 sudo chown -R pleroma:pleroma /opt/pleroma
-sudo -Hu pleroma git clone https://git.pleroma.social/pleroma/pleroma /opt/pleroma
+sudo -Hu pleroma git clone -b master https://git.pleroma.social/pleroma/pleroma /opt/pleroma
 ```
 
 * Change to the new directory:
diff --git a/docs/installation/arch_linux_en.md b/docs/installation/arch_linux_en.md
index 2b040cfbc..26e1ab86a 100644
--- a/docs/installation/arch_linux_en.md
+++ b/docs/installation/arch_linux_en.md
@@ -66,7 +66,7 @@ sudo useradd -r -s /bin/false -m -d /var/lib/pleroma -U pleroma
 ```shell
 sudo mkdir -p /opt/pleroma
 sudo chown -R pleroma:pleroma /opt/pleroma
-sudo -Hu pleroma git clone https://git.pleroma.social/pleroma/pleroma /opt/pleroma
+sudo -Hu pleroma git clone -b master https://git.pleroma.social/pleroma/pleroma /opt/pleroma
 ```
 
 * Change to the new directory:
diff --git a/docs/installation/centos7_en.md b/docs/installation/centos7_en.md
index 76de21ed8..19bff7461 100644
--- a/docs/installation/centos7_en.md
+++ b/docs/installation/centos7_en.md
@@ -143,7 +143,7 @@ sudo useradd -r -s /bin/false -m -d /var/lib/pleroma -U pleroma
 ```shell
 sudo mkdir -p /opt/pleroma
 sudo chown -R pleroma:pleroma /opt/pleroma
-sudo -Hu pleroma git clone https://git.pleroma.social/pleroma/pleroma /opt/pleroma
+sudo -Hu pleroma git clone -b master https://git.pleroma.social/pleroma/pleroma /opt/pleroma
 ```
 
 * Change to the new directory:
diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md
index 9c0ef92d4..7d39ca5f9 100644
--- a/docs/installation/debian_based_en.md
+++ b/docs/installation/debian_based_en.md
@@ -68,7 +68,7 @@ sudo useradd -r -s /bin/false -m -d /var/lib/pleroma -U pleroma
 ```shell
 sudo mkdir -p /opt/pleroma
 sudo chown -R pleroma:pleroma /opt/pleroma
-sudo -Hu pleroma git clone https://git.pleroma.social/pleroma/pleroma /opt/pleroma
+sudo -Hu pleroma git clone -b master https://git.pleroma.social/pleroma/pleroma /opt/pleroma
 ```
 
 * Change to the new directory:
diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md
index 41cce6792..7b4c1bd86 100644
--- a/docs/installation/debian_based_jp.md
+++ b/docs/installation/debian_based_jp.md
@@ -69,7 +69,7 @@ cd ~
 
 *  Gitリポジトリをクローンします。
 ```
-git clone https://git.pleroma.social/pleroma/pleroma
+ git clone -b master https://git.pleroma.social/pleroma/pleroma
 ```
 
 *  新しいディレクトリに移動します。
diff --git a/docs/installation/gentoo_en.md b/docs/installation/gentoo_en.md
index fccaad378..b7c42a477 100644
--- a/docs/installation/gentoo_en.md
+++ b/docs/installation/gentoo_en.md
@@ -106,7 +106,7 @@ It is highly recommended you use your own fork for the `https://path/to/repo` pa
 
 ```shell
  pleroma$ cd ~
- pleroma$ git clone https://path/to/repo
+ pleroma$ git clone -b master https://path/to/repo
 ```
 
 * Change to the new directory:
diff --git a/docs/installation/netbsd_en.md b/docs/installation/netbsd_en.md
index e0ac98359..a096d5354 100644
--- a/docs/installation/netbsd_en.md
+++ b/docs/installation/netbsd_en.md
@@ -58,7 +58,7 @@ Clone the repository:
 
 ```
 $ cd /home/pleroma
-$ git clone https://git.pleroma.social/pleroma/pleroma.git
+$ git clone -b master https://git.pleroma.social/pleroma/pleroma.git
 ```
 
 Configure Pleroma. Note that you need a domain name at this point:
diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md
index 633b08e6c..46e7feffe 100644
--- a/docs/installation/openbsd_en.md
+++ b/docs/installation/openbsd_en.md
@@ -29,7 +29,7 @@ This creates a "pleroma" login class and sets higher values than default for dat
 Create the \_pleroma user, assign it the pleroma login class and create its home directory (/home/\_pleroma/): `useradd -m -L pleroma _pleroma`
 
 #### Clone pleroma's directory
-Enter a shell as the \_pleroma user. As root, run `su _pleroma -;cd`. Then clone the repository with `git clone https://git.pleroma.social/pleroma/pleroma.git`. Pleroma is now installed in /home/\_pleroma/pleroma/, it will be configured and started at the end of this guide.
+Enter a shell as the \_pleroma user. As root, run `su _pleroma -;cd`. Then clone the repository with ` git clone -b master https://git.pleroma.social/pleroma/pleroma.git`. Pleroma is now installed in /home/\_pleroma/pleroma/, it will be configured and started at the end of this guide.
 
 #### Postgresql
 Start a shell as the \_postgresql user (as root run `su _postgresql -` then run the `initdb` command to initialize postgresql:  
diff --git a/docs/installation/openbsd_fi.md b/docs/installation/openbsd_fi.md
index fa6faa62d..39819a8c8 100644
--- a/docs/installation/openbsd_fi.md
+++ b/docs/installation/openbsd_fi.md
@@ -44,7 +44,7 @@ Vaihda pleroma-käyttäjään ja mene kotihakemistoosi:
 
 Lataa pleroman lähdekoodi:
 
-`$ git clone https://git.pleroma.social/pleroma/pleroma.git`
+`$ git clone -b master https://git.pleroma.social/pleroma/pleroma.git`
 
 `$ cd pleroma`
 

From 88414151b41e6e060caf7c90be3341a986351395 Mon Sep 17 00:00:00 2001
From: feld <feld@feld.me>
Date: Thu, 30 May 2019 21:35:31 +0000
Subject: [PATCH 60/64] Formatting

---
 docs/installation/debian_based_jp.md | 2 +-
 docs/installation/openbsd_en.md      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md
index 7b4c1bd86..84b9666c8 100644
--- a/docs/installation/debian_based_jp.md
+++ b/docs/installation/debian_based_jp.md
@@ -69,7 +69,7 @@ cd ~
 
 *  Gitリポジトリをクローンします。
 ```
- git clone -b master https://git.pleroma.social/pleroma/pleroma
+git clone -b master https://git.pleroma.social/pleroma/pleroma
 ```
 
 *  新しいディレクトリに移動します。
diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md
index 46e7feffe..fcba38b2c 100644
--- a/docs/installation/openbsd_en.md
+++ b/docs/installation/openbsd_en.md
@@ -29,7 +29,7 @@ This creates a "pleroma" login class and sets higher values than default for dat
 Create the \_pleroma user, assign it the pleroma login class and create its home directory (/home/\_pleroma/): `useradd -m -L pleroma _pleroma`
 
 #### Clone pleroma's directory
-Enter a shell as the \_pleroma user. As root, run `su _pleroma -;cd`. Then clone the repository with ` git clone -b master https://git.pleroma.social/pleroma/pleroma.git`. Pleroma is now installed in /home/\_pleroma/pleroma/, it will be configured and started at the end of this guide.
+Enter a shell as the \_pleroma user. As root, run `su _pleroma -;cd`. Then clone the repository with `git clone -b master https://git.pleroma.social/pleroma/pleroma.git`. Pleroma is now installed in /home/\_pleroma/pleroma/, it will be configured and started at the end of this guide.
 
 #### Postgresql
 Start a shell as the \_postgresql user (as root run `su _postgresql -` then run the `initdb` command to initialize postgresql:  

From a9eaa558853460b811d134b49fb00b017b772e94 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 31 May 2019 14:17:05 +0300
Subject: [PATCH 61/64] Fix fix_explicit_addressing moving follower collection
 to cc and add tests for it

---
 .../web/activity_pub/transmogrifier.ex        |  5 ++-
 test/web/activity_pub/transmogrifier_test.exs | 40 +++++++++++++++++++
 2 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 5edd8ccc7..d8fa2728d 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -93,7 +93,10 @@ def fix_explicit_addressing(object) do
       object
       |> Utils.determine_explicit_mentions()
 
-    explicit_mentions = explicit_mentions ++ ["https://www.w3.org/ns/activitystreams#Public"]
+    follower_collection = User.get_cached_by_ap_id(Containment.get_actor(object)).follower_address
+
+    explicit_mentions =
+      explicit_mentions ++ ["https://www.w3.org/ns/activitystreams#Public", follower_collection]
 
     object
     |> fix_explicit_addressing(explicit_mentions)
diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs
index c24b50f8c..ee71de8d0 100644
--- a/test/web/activity_pub/transmogrifier_test.exs
+++ b/test/web/activity_pub/transmogrifier_test.exs
@@ -1209,4 +1209,44 @@ test "successfully reserializes a message with AS2 objects in IR" do
       {:ok, _} = Transmogrifier.prepare_outgoing(activity.data)
     end
   end
+
+  describe "fix_explicit_addressing" do
+    test "moves non-explicitly mentioned actors to cc" do
+      user = insert(:user)
+
+      explicitly_mentioned_actors = [
+        "https://pleroma.gold/users/user1",
+        "https://pleroma.gold/user2"
+      ]
+
+      object = %{
+        "actor" => user.ap_id,
+        "to" => explicitly_mentioned_actors ++ ["https://social.beepboop.ga/users/dirb"],
+        "cc" => [],
+        "tag" =>
+          Enum.map(explicitly_mentioned_actors, fn href ->
+            %{"type" => "Mention", "href" => href}
+          end)
+      }
+
+      fixed_object = Transmogrifier.fix_explicit_addressing(object)
+      assert Enum.all?(explicitly_mentioned_actors, &(&1 in fixed_object["to"]))
+      refute "https://social.beepboop.ga/users/dirb" in fixed_object["to"]
+      assert "https://social.beepboop.ga/users/dirb" in fixed_object["cc"]
+    end
+
+    test "does not move actor's follower collection to cc" do
+      user = insert(:user)
+
+      object = %{
+        "actor" => user.ap_id,
+        "to" => [user.follower_address],
+        "cc" => []
+      }
+
+      fixed_object = Transmogrifier.fix_explicit_addressing(object)
+      assert user.follower_address in fixed_object["to"]
+      refute user.follower_address in fixed_object["cc"]
+    end
+  end
 end

From d9c0650ff9afd66c15d960b727dc2e6ed37477a3 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 31 May 2019 15:25:17 +0300
Subject: [PATCH 62/64] Mastodon API: Fix lists leaking private posts

Our previous list visibility resolver grabbed posts if either follower
collection of the user in a list who is followed is in `to` or if
follower collection of the user in a list was in `cc`. This not only
missed unlisted posts but also lead to leaking private posts when
`fix_explicit_addressing` mistakingly started putting follower collections
to `cc` (also fixed in this MR).

Reported by @kurisu@iscute.moe via a DM
---
 CHANGELOG.md                                 |  4 +++
 lib/pleroma/web/activity_pub/activity_pub.ex | 27 ++++++++----------
 test/web/activity_pub/activity_pub_test.exs  | 29 ++++++++++++++++++++
 3 files changed, 44 insertions(+), 16 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f689160e9..716cf2a46 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -118,6 +118,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## Removed
 - Configuration: `config :pleroma, :fe` in favor of the more flexible `config :pleroma, :frontend_configurations`
 
+## [0.9.99999] - 2019-05-31
+### Security
+- Mastodon API: Fix lists leaking private posts
+
 ## [0.9.9999] - 2019-04-05
 ### Security
 - Mastodon API: Fix content warnings skipping HTML sanitization
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index aa0229db7..498b1a342 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -653,20 +653,6 @@ defp restrict_tag(query, %{"tag" => tag}) when is_binary(tag) do
 
   defp restrict_tag(query, _), do: query
 
-  defp restrict_to_cc(query, recipients_to, recipients_cc) do
-    from(
-      activity in query,
-      where:
-        fragment(
-          "(?->'to' \\?| ?) or (?->'cc' \\?| ?)",
-          activity.data,
-          ^recipients_to,
-          activity.data,
-          ^recipients_cc
-        )
-    )
-  end
-
   defp restrict_recipients(query, [], _user), do: query
 
   defp restrict_recipients(query, recipients, nil) do
@@ -889,9 +875,18 @@ def fetch_activities(recipients, opts \\ %{}) do
     |> Enum.reverse()
   end
 
-  def fetch_activities_bounded(recipients_to, recipients_cc, opts \\ %{}) do
+  def fetch_activities_bounded_query(query, recipients, recipients_with_public) do
+    from(activity in query,
+      where:
+        fragment("? && ?", activity.recipients, ^recipients) or
+          (fragment("? && ?", activity.recipients, ^recipients_with_public) and
+             "https://www.w3.org/ns/activitystreams#Public" in activity.recipients)
+    )
+  end
+
+  def fetch_activities_bounded(recipients, recipients_with_public, opts \\ %{}) do
     fetch_activities_query([], opts)
-    |> restrict_to_cc(recipients_to, recipients_cc)
+    |> fetch_activities_bounded_query(recipients, recipients_with_public)
     |> Pagination.fetch_paginated(opts)
     |> Enum.reverse()
   end
diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs
index f743f380b..76586ee4a 100644
--- a/test/web/activity_pub/activity_pub_test.exs
+++ b/test/web/activity_pub/activity_pub_test.exs
@@ -1186,4 +1186,33 @@ test "it can create a Flag activity" do
   def data_uri do
     File.read!("test/fixtures/avatar_data_uri")
   end
+
+  describe "fetch_activities_bounded" do
+    test "fetches private posts for followed users" do
+      user = insert(:user)
+
+      {:ok, activity} =
+        CommonAPI.post(user, %{
+          "status" => "thought I looked cute might delete later :3",
+          "visibility" => "private"
+        })
+
+      [result] = ActivityPub.fetch_activities_bounded([user.follower_address], [])
+      assert result.id == activity.id
+    end
+
+    test "fetches only public posts for other users" do
+      user = insert(:user)
+      {:ok, activity} = CommonAPI.post(user, %{"status" => "#cofe", "visibility" => "public"})
+
+      {:ok, _private_activity} =
+        CommonAPI.post(user, %{
+          "status" => "why is tenshi eating a corndog so cute?",
+          "visibility" => "private"
+        })
+
+      [result] = ActivityPub.fetch_activities_bounded([], [user.follower_address])
+      assert result.id == activity.id
+    end
+  end
 end

From 6872c5f88f211dd40ff605268f1a0a22acc199c3 Mon Sep 17 00:00:00 2001
From: feld <feld@feld.me>
Date: Fri, 31 May 2019 18:46:57 +0000
Subject: [PATCH 63/64] Mastodon API search default value for "resolve" is
 false

https://docs.joinmastodon.org/api/rest/accounts/
https://docs.joinmastodon.org/api/rest/search/
---
 lib/pleroma/user.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 5c91dc7d4..474cd8c1a 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -758,7 +758,7 @@ def search_query(query, for_user) do
 
     from(s in subquery(boost_search_rank_query(distinct_query, for_user)),
       order_by: [desc: s.search_rank],
-      limit: 20
+      limit: 40
     )
   end
 

From abedcca349bf5eefd75fb1e15d5b6ca54401fb27 Mon Sep 17 00:00:00 2001
From: feld <feld@feld.me>
Date: Fri, 31 May 2019 20:36:33 +0000
Subject: [PATCH 64/64] Sample Mongoose IM config.

Change domain, ensure paths to PEM files are correct, and this is good to go.
---
 installation/pleroma-mongooseim.cfg | 932 ++++++++++++++++++++++++++++
 1 file changed, 932 insertions(+)
 create mode 100755 installation/pleroma-mongooseim.cfg

diff --git a/installation/pleroma-mongooseim.cfg b/installation/pleroma-mongooseim.cfg
new file mode 100755
index 000000000..d7567321f
--- /dev/null
+++ b/installation/pleroma-mongooseim.cfg
@@ -0,0 +1,932 @@
+%%%
+%%%               ejabberd configuration file
+%%%
+%%%'
+
+%%% The parameters used in this configuration file are explained in more detail
+%%% in the ejabberd Installation and Operation Guide.
+%%% Please consult the Guide in case of doubts, it is included with
+%%% your copy of ejabberd, and is also available online at
+%%% http://www.process-one.net/en/ejabberd/docs/
+
+%%% This configuration file contains Erlang terms.
+%%% In case you want to understand the syntax, here are the concepts:
+%%%
+%%%  - The character to comment a line is %
+%%%
+%%%  - Each term ends in a dot, for example:
+%%%      override_global.
+%%%
+%%%  - A tuple has a fixed definition, its elements are
+%%%    enclosed in {}, and separated with commas:
+%%%      {loglevel, 4}.
+%%%
+%%%  - A list can have as many elements as you want,
+%%%    and is enclosed in [], for example:
+%%%      [http_poll, web_admin, tls]
+%%%
+%%%    Pay attention that list elements are delimited with commas,
+%%%    but no comma is allowed after the last list element. This will
+%%%    give a syntax error unlike in more lenient languages (e.g. Python).
+%%%
+%%%  - A keyword of ejabberd is a word in lowercase.
+%%%    Strings are enclosed in "" and can contain spaces, dots, ...
+%%%      {language, "en"}.
+%%%      {ldap_rootdn, "dc=example,dc=com"}.
+%%%
+%%%  - This term includes a tuple, a keyword, a list, and two strings:
+%%%      {hosts, ["jabber.example.net", "im.example.com"]}.
+%%%
+%%%  - This config is preprocessed during release generation by a tool which
+%%%    interprets double curly braces as substitution markers, so avoid this
+%%%    syntax in this file (though it's valid Erlang).
+%%%
+%%%    So this is OK (though arguably looks quite ugly):
+%%%      { {s2s_addr, "example-host.net"}, {127,0,0,1} }.
+%%%
+%%%    And I can't give an example of what's not OK exactly because
+%%%    of this rule.
+%%%
+
+
+%%%.   =======================
+%%%'   OVERRIDE STORED OPTIONS
+
+%%
+%% Override the old values stored in the database.
+%%
+
+%%
+%% Override global options (shared by all ejabberd nodes in a cluster).
+%%
+%%override_global.
+
+%%
+%% Override local options (specific for this particular ejabberd node).
+%%
+%%override_local.
+
+%%
+%% Remove the Access Control Lists before new ones are added.
+%%
+%%override_acls.
+
+
+%%%.   =========
+%%%'   DEBUGGING
+
+%%
+%% loglevel: Verbosity of log files generated by ejabberd.
+%% 0: No ejabberd log at all (not recommended)
+%% 1: Critical
+%% 2: Error
+%% 3: Warning
+%% 4: Info
+%% 5: Debug
+%%
+{loglevel, 3}.
+
+%%%.   ================
+%%%'   SERVED HOSTNAMES
+
+%%
+%% hosts: Domains served by ejabberd.
+%% You can define one or several, for example:
+%% {hosts, ["example.net", "example.com", "example.org"]}.
+%%
+{hosts, ["pleroma.soykaf.com"] }.
+
+%%
+%% route_subdomains: Delegate subdomains to other XMPP servers.
+%% For example, if this ejabberd serves example.org and you want
+%% to allow communication with an XMPP server called im.example.org.
+%%
+%%{route_subdomains, s2s}.
+
+
+%%%.   ===============
+%%%'   LISTENING PORTS
+
+%%
+%% listen: The ports ejabberd will listen on, which service each is handled
+%% by and what options to start it with.
+%%
+{listen,
+ [
+  %% BOSH and WS endpoints over HTTP
+  { 5280, ejabberd_cowboy, [
+      {num_acceptors, 10},
+      {transport_options, [{max_connections, 1024}]},
+      {modules, [
+
+          {"_", "/http-bind", mod_bosh},
+          {"_", "/ws-xmpp", mod_websockets, [{ejabberd_service, [
+                                        {access, all},
+                                        {shaper_rule, fast},
+                                        {ip, {127, 0, 0, 1}},
+                                        {password, "secret"}]}
+          %% Uncomment to enable connection dropping or/and server-side pings
+          %{timeout, 600000}, {ping_rate, 2000}
+          ]}
+          %% Uncomment to serve static files
+          %{"_", "/static/[...]", cowboy_static,
+          %  {dir, "/var/www", [{mimetypes, cow_mimetypes, all}]}
+          %},
+
+          %% Example usage of mod_revproxy
+
+          %% {"_", "/[...]", mod_revproxy, [{timeout, 5000},
+          %%                                % time limit for upstream to respond
+          %%                                {body_length, 8000000},
+          %%                                % maximum body size (may be infinity)
+          %%                                {custom_headers, [{<<"header">>,<<"value">>}]}
+          %%                                % list of extra headers that are send to upstream
+          %%                               ]}
+
+          %% Example usage of mod_cowboy
+
+          %% {"_", "/[...]", mod_cowboy, [{http, mod_revproxy,
+          %%                                [{timeout, 5000},
+          %%                                 % time limit for upstream to respond
+          %%                                 {body_length, 8000000},
+          %%                                 % maximum body size (may be infinity)
+          %%                                 {custom_headers, [{<<"header">>,<<"value">>}]}
+          %%                                 % list of extra headers that are send to upstream
+          %%                                ]},
+          %%                               {ws, xmpp, mod_websockets}
+          %%                             ]}
+      ]}
+  ]},
+
+  %% BOSH and WS endpoints over HTTPS
+  { 5285, ejabberd_cowboy, [
+        {num_acceptors, 10},
+        {transport_options, [{max_connections, 1024}]},
+        {ssl, [{certfile, "priv/ssl/fullchain.pem"}, {keyfile, "priv/ssl/privkey.pem"}, {password, ""}]},
+        {modules, [
+            {"_", "/http-bind", mod_bosh},
+            {"_", "/ws-xmpp", mod_websockets, [
+            %% Uncomment to enable connection dropping or/and server-side pings
+            %{timeout, 600000}, {ping_rate, 60000}
+            ]}
+            %% Uncomment to serve static files
+            %{"_", "/static/[...]", cowboy_static,
+            %  {dir, "/var/www", [{mimetypes, cow_mimetypes, all}]}
+            %},
+        ]}
+    ]},
+
+  %% MongooseIM HTTP API it's important to start it on localhost
+  %% or some private interface only (not accessible from the outside)
+  %% At least start it on different port which will be hidden behind firewall
+
+  { {8088, "127.0.0.1"} , ejabberd_cowboy, [
+      {num_acceptors, 10},
+      {transport_options, [{max_connections, 1024}]},
+      {modules, [
+          {"localhost", "/api", mongoose_api_admin, []}
+      ]}
+  ]},
+
+  { 8089 , ejabberd_cowboy, [
+      {num_acceptors, 10},
+      {transport_options, [{max_connections, 1024}]},
+      {protocol_options, [{compress, true}]},
+      {ssl, [{certfile, "priv/ssl/fullchain.pem"}, {keyfile, "priv/ssl/privkey.pem"}, {password, ""}]},
+      {modules, [
+          {"_", "/api/sse", lasse_handler, [mongoose_client_api_sse]},
+          {"_", "/api/messages/[:with]", mongoose_client_api_messages, []},
+          {"_", "/api/contacts/[:jid]", mongoose_client_api_contacts, []},
+          {"_", "/api/rooms/[:id]",    mongoose_client_api_rooms, []},
+          {"_", "/api/rooms/[:id]/config",    mongoose_client_api_rooms_config, []},
+          {"_", "/api/rooms/:id/users/[:user]",    mongoose_client_api_rooms_users, []},
+          {"_", "/api/rooms/[:id]/messages",    mongoose_client_api_rooms_messages, []}
+      ]}
+  ]},
+
+  %% Following HTTP API is deprected, the new one abouve should be used instead
+
+  { {5288, "127.0.0.1"} , ejabberd_cowboy, [
+      {num_acceptors, 10},
+      {transport_options, [{max_connections, 1024}]},
+      {modules, [
+          {"localhost", "/api", mongoose_api, [{handlers, [mongoose_api_metrics,
+                                                           mongoose_api_users]}]}
+      ]}
+  ]},
+
+  { 5222, ejabberd_c2s, [
+
+			%%
+			%% If TLS is compiled in and you installed a SSL
+			%% certificate, specify the full path to the
+			%% file and uncomment this line:
+			%%
+                        {certfile, "priv/ssl/both.pem"}, starttls,
+                        
+                        %%{zlib, 10000},
+			%% https://www.openssl.org/docs/apps/ciphers.html#CIPHER_STRINGS
+			%% {ciphers, "DEFAULT:!EXPORT:!LOW:!SSLv2"},
+			{access, c2s},
+			{shaper, c2s_shaper},
+			{max_stanza_size, 65536},
+			{protocol_options, ["no_sslv3"]}
+                        
+		       ]},
+
+  
+
+  %%
+  %% To enable the old SSL connection method on port 5223:
+  %%
+  %%{5223, ejabberd_c2s, [
+  %%			{access, c2s},
+  %%			{shaper, c2s_shaper},
+  %%			{certfile, "/path/to/ssl.pem"}, tls,
+  %%			{max_stanza_size, 65536}
+  %%		       ]},
+
+  { 5269, ejabberd_s2s_in, [
+			   {shaper, s2s_shaper},
+			   {max_stanza_size, 131072},
+			   {protocol_options, ["no_sslv3"]}
+			   
+			  ]}
+
+  %%
+  %% ejabberd_service: Interact with external components (transports, ...)
+  %%
+  ,{8888, ejabberd_service, [
+                {access, all},
+                {shaper_rule, fast},
+                {ip, {127, 0, 0, 1}},
+                {password, "secret"}
+           ]}
+
+  %%
+  %% ejabberd_stun: Handles STUN Binding requests
+  %%
+  %%{ {3478, udp}, ejabberd_stun, []}
+
+ ]}.
+
+%%
+%% s2s_use_starttls: Enable STARTTLS + Dialback for S2S connections.
+%% Allowed values are: false optional required required_trusted
+%% You must specify a certificate file.
+%%
+{s2s_use_starttls, optional}.
+%%
+%% s2s_certfile: Specify a certificate file.
+%%
+{s2s_certfile, "priv/ssl/both.pem"}.
+
+%% https://www.openssl.org/docs/apps/ciphers.html#CIPHER_STRINGS
+%% {s2s_ciphers, "DEFAULT:!EXPORT:!LOW:!SSLv2"}.
+
+%%
+%% domain_certfile: Specify a different certificate for each served hostname.
+%%
+%%{domain_certfile, "example.org", "/path/to/example_org.pem"}.
+%%{domain_certfile, "example.com", "/path/to/example_com.pem"}.
+
+%%
+%% S2S whitelist or blacklist
+%%
+%% Default s2s policy for undefined hosts.
+%%
+{s2s_default_policy, deny }.
+
+%%
+%% Allow or deny communication with specific servers.
+%%
+%%{ {s2s_host, "goodhost.org"}, allow}.
+%%{ {s2s_host, "badhost.org"}, deny}.
+
+{outgoing_s2s_port, 5269 }.
+
+%%
+%% IP addresses predefined for specific hosts to skip DNS lookups.
+%% Ports defined here take precedence over outgoing_s2s_port.
+%% Examples:
+%%
+%% { {s2s_addr, "example-host.net"}, {127,0,0,1} }.
+%% { {s2s_addr, "example-host.net"}, { {127,0,0,1}, 5269 } }.
+%% { {s2s_addr, "example-host.net"}, { {127,0,0,1}, 5269 } }.
+
+%%
+%% Outgoing S2S options
+%%
+%% Preferred address families (which to try first) and connect timeout
+%% in milliseconds.
+%%
+%%{outgoing_s2s_options, [ipv4, ipv6], 10000}.
+%%
+%%%.   ==============
+%%%'   SESSION BACKEND
+
+%%{sm_backend, {mnesia, []}}.
+
+%% Requires {redis, global, default, ..., ...} outgoing pool
+%%{sm_backend, {redis, []}}.
+
+{sm_backend, {mnesia, []} }.
+
+
+%%%.   ==============
+%%%'   AUTHENTICATION
+
+%% Advertised SASL mechanisms
+{sasl_mechanisms, [cyrsasl_plain]}.
+
+%%
+%% auth_method: Method used to authenticate the users.
+%% The default method is the internal.
+%% If you want to use a different method,
+%% comment this line and enable the correct ones.
+%%
+%% {auth_method, internal }.
+{auth_method, http }.
+{auth_opts, [
+             {http, global, auth, [{workers, 50}], [{server, "https://pleroma.soykaf.com"}]},
+             {password_format, plain} % default
+             %% {password_format, scram}
+             
+             %% {scram_iterations, 4096} % default
+             
+             %%
+             %% For auth_http:
+             %% {basic_auth, "user:password"}
+             %% {path_prefix, "/"} % default
+             %% auth_http requires {http, Host | global, auth, ..., ...} outgoing pool.
+             %%
+             %% For auth_external
+             %%{extauth_program, "/path/to/authentication/script"}.
+             %%
+             %% For auth_jwt
+             %% {jwt_secret_source, "/path/to/file"},
+             %% {jwt_algorithm, "RS256"},
+             %% {jwt_username_key, user}
+             %% For cyrsasl_external
+             %% {authenticate_with_cn, false}
+             {cyrsasl_external, standard}
+            ]}.
+
+%%
+%% Authentication using external script
+%% Make sure the script is executable by ejabberd.
+%%
+%%{auth_method, external}.
+
+%%
+%% Authentication using RDBMS
+%% Remember to setup a database in the next section.
+%%
+%%{auth_method, rdbms}.
+
+%%
+%% Authentication using LDAP
+%%
+%%{auth_method, ldap}.
+%%
+
+%% List of LDAP servers:
+%%{ldap_servers, ["localhost"]}.
+%%
+%% Encryption of connection to LDAP servers:
+%%{ldap_encrypt, none}.
+%%{ldap_encrypt, tls}.
+%%
+%% Port to connect to on LDAP servers:
+%%{ldap_port, 389}.
+%%{ldap_port, 636}.
+%%
+%% LDAP manager:
+%%{ldap_rootdn, "dc=example,dc=com"}.
+%%
+%% Password of LDAP manager:
+%%{ldap_password, "******"}.
+%%
+%% Search base of LDAP directory:
+%%{ldap_base, "dc=example,dc=com"}.
+%%
+%% LDAP attribute that holds user ID:
+%%{ldap_uids, [{"mail", "%u@mail.example.org"}]}.
+%%
+%% LDAP filter:
+%%{ldap_filter, "(objectClass=shadowAccount)"}.
+
+%%
+%% Anonymous login support:
+%%   auth_method: anonymous
+%%   anonymous_protocol: sasl_anon | login_anon | both
+%%   allow_multiple_connections: true | false
+%%
+%%{host_config, "public.example.org", [{auth_method, anonymous},
+%%                                     {allow_multiple_connections, false},
+%%                                     {anonymous_protocol, sasl_anon}]}.
+%%
+%% To use both anonymous and internal authentication:
+%%
+%%{host_config, "public.example.org", [{auth_method, [internal, anonymous]}]}.
+
+
+%%%.   ==============
+%%%'   OUTGOING CONNECTIONS (e.g. DB)
+
+%% Here you may configure all outgoing connections used by MongooseIM,
+%% e.g. to RDBMS (such as MySQL), Riak or external HTTP components.
+%% Default MongooseIM configuration uses only Mnesia (non-Mnesia extensions are disabled),
+%% so no options here are uncommented out of the box.
+%% This section includes configuration examples; for comprehensive guide
+%% please consult MongooseIM documentation, page "Outgoing connections":
+%% - doc/advanced-configuration/outgoing-connections.md
+%% - https://mongooseim.readthedocs.io/en/latest/advanced-configuration/outgoing-connections/
+
+
+{outgoing_pools, [
+%  {riak, global, default, [{workers, 5}], [{address, "127.0.0.1"}, {port, 8087}]},
+%  {elastic, global, default, [], [{host, "elastic.host.com"}, {port, 9042}]},
+  {http, global, auth, [{workers, 50}], [{server, "https://pleroma.soykaf.com"}]}
+%  {cassandra, global, default, [{workers, 100}], [{servers, [{"server1", 9042}]}, {keyspace, "big_mongooseim"}]},
+%  {rdbms, global, default, [{workers, 10}], [{server, {mysql, "server", 3306, "database", "username", "password"}}]}
+]}.
+
+%% More examples that may be added to outgoing_pools list:
+%%
+%% == MySQL ==
+%%  {rdbms, global, default, [{workers, 10}],
+%%   [{server, {mysql, "server", 3306, "database", "username", "password"}},
+%%    {keepalive_interval, 10}]},
+%% keepalive_interval is optional
+
+%% == PostgreSQL ==
+%%  {rdbms, global, default, [{workers, 10}],
+%%   [{server, {pgsql, "server", 5432, "database", "username", "password"}}]},
+
+%% == ODBC (MSSQL) ==
+%%  {rdbms, global, default, [{workers, 10}],
+%%   [{server, "DSN=mongooseim;UID=mongooseim;PWD=mongooseim"}]},
+
+%% == Elastic Search ==
+%%  {elastic, global, default, [], [{host, "elastic.host.com"}, {port, 9042}]},
+
+%% == Riak ==
+%%  {riak, global, default, [{workers, 20}], [{address, "127.0.0.1"}, {port, 8087}]},
+
+%% == HTTP ==
+%%  {http, global, conn1, [{workers, 50}], [{server, "http://server:8080"}]},
+
+%% == Cassandra ==
+%%  {cassandra, global, default, [{workers, 100}],
+%%    [
+%%      {servers, [
+%%                 {"cassandra_server1.example.com", 9042},
+%%                 {"cassandra_server2.example.com", 9042},
+%%                 {"cassandra_server3.example.com", 9042},
+%%                 {"cassandra_server4.example.com", 9042}
+%%                ]},
+%%      {keyspace, "big_mongooseim"}
+%%    ]}
+
+%% == Extra options ==
+%%
+%% If you use PostgreSQL, have a large database, and need a
+%% faster but inexact replacement for "select count(*) from users"
+%%
+%%{pgsql_users_number_estimate, true}.
+%%
+%% rdbms_server_type specifies what database is used over the RDBMS layer
+%% Can take values mssql, pgsql, mysql
+%% In some cases (for example for MAM with pgsql) it is required to set proper value.
+%%
+%% {rdbms_server_type, pgsql}.
+
+%%%.   ===============
+%%%'   TRAFFIC SHAPERS
+
+%%
+%% The "normal" shaper limits traffic speed to 1000 B/s
+%%
+{shaper, normal, {maxrate, 1000}}.
+
+%%
+%% The "fast" shaper limits traffic speed to 50000 B/s
+%%
+{shaper, fast, {maxrate, 50000}}.
+
+%%
+%% This option specifies the maximum number of elements in the queue
+%% of the FSM. Refer to the documentation for details.
+%%
+{max_fsm_queue, 1000}.
+
+%%%.   ====================
+%%%'   ACCESS CONTROL LISTS
+
+%%
+%% The 'admin' ACL grants administrative privileges to XMPP accounts.
+%% You can put here as many accounts as you want.
+%%
+%{acl, admin, {user, "alice", "localhost"}}.
+%{acl, admin, {user, "a", "localhost"}}.
+
+%%
+%% Blocked users
+%%
+%%{acl, blocked, {user, "baduser", "example.org"}}.
+%%{acl, blocked, {user, "test"}}.
+
+%%
+%% Local users: don't modify this line.
+%%
+{acl, local, {user_regexp, ""}}.
+
+%%
+%% More examples of ACLs
+%%
+%%{acl, jabberorg, {server, "jabber.org"}}.
+%%{acl, aleksey, {user, "aleksey", "jabber.ru"}}.
+%%{acl, test, {user_regexp, "^test"}}.
+%%{acl, test, {user_glob, "test*"}}.
+
+%%
+%% Define specific ACLs in a virtual host.
+%%
+%%{host_config, "localhost",
+%% [
+%%  {acl, admin, {user, "bob-local", "localhost"}}
+%% ]
+%%}.
+
+%%%.   ============
+%%%'   ACCESS RULES
+
+%% Maximum number of simultaneous sessions allowed for a single user:
+{access, max_user_sessions, [{10, all}]}.
+
+%% Maximum number of offline messages that users can have:
+{access, max_user_offline_messages, [{5000, admin}, {100, all}]}.
+
+%% This rule allows access only for local users:
+{access, local, [{allow, local}]}.
+
+%% Only non-blocked users can use c2s connections:
+{access, c2s, [{deny, blocked},
+	       {allow, all}]}.
+
+%% For C2S connections, all users except admins use the "normal" shaper
+{access, c2s_shaper, [{none, admin},
+		      {normal, all}]}.
+
+%% All S2S connections use the "fast" shaper
+{access, s2s_shaper, [{fast, all}]}.
+
+%% Admins of this server are also admins of the MUC service:
+{access, muc_admin, [{allow, admin}]}.
+
+%% Only accounts of the local ejabberd server can create rooms:
+{access, muc_create, [{allow, local}]}.
+
+%% All users are allowed to use the MUC service:
+{access, muc, [{allow, all}]}.
+
+%% In-band registration allows registration of any possible username.
+%% To disable in-band registration, replace 'allow' with 'deny'.
+{access, register, [{allow, all}]}.
+
+%% By default the frequency of account registrations from the same IP
+%% is limited to 1 account every 10 minutes. To disable, specify: infinity
+{registration_timeout, infinity}.
+
+%% Default settings for MAM.
+%% To set non-standard value, replace 'default' with 'allow' or 'deny'.
+%% Only user can access his/her archive by default.
+%% An online user can read room's archive by default.
+%% Only an owner can change settings and purge messages by default.
+%% Empty list (i.e. `[]`) means `[{deny, all}]`.
+{access, mam_set_prefs, [{default, all}]}.
+{access, mam_get_prefs, [{default, all}]}.
+{access, mam_lookup_messages, [{default, all}]}.
+{access, mam_purge_single_message, [{default, all}]}.
+{access, mam_purge_multiple_messages, [{default, all}]}.
+
+%% 1 command of the specified type per second.
+{shaper, mam_shaper, {maxrate, 1}}.
+%% This shaper is primeraly for Mnesia overload protection during stress testing.
+%% The limit is 1000 operations of each type per second.
+{shaper, mam_global_shaper, {maxrate, 1000}}.
+
+{access, mam_set_prefs_shaper, [{mam_shaper, all}]}.
+{access, mam_get_prefs_shaper, [{mam_shaper, all}]}.
+{access, mam_lookup_messages_shaper, [{mam_shaper, all}]}.
+{access, mam_purge_single_message_shaper, [{mam_shaper, all}]}.
+{access, mam_purge_multiple_messages_shaper, [{mam_shaper, all}]}.
+
+{access, mam_set_prefs_global_shaper, [{mam_global_shaper, all}]}.
+{access, mam_get_prefs_global_shaper, [{mam_global_shaper, all}]}.
+{access, mam_lookup_messages_global_shaper, [{mam_global_shaper, all}]}.
+{access, mam_purge_single_message_global_shaper, [{mam_global_shaper, all}]}.
+{access, mam_purge_multiple_messages_global_shaper, [{mam_global_shaper, all}]}.
+
+%%
+%% Define specific Access Rules in a virtual host.
+%%
+%%{host_config, "localhost",
+%% [
+%%  {access, c2s, [{allow, admin}, {deny, all}]},
+%%  {access, register, [{deny, all}]}
+%% ]
+%%}.
+
+%%%.   ================
+%%%'   DEFAULT LANGUAGE
+
+%%
+%% language: Default language used for server messages.
+%%
+{language, "en"}.
+
+%%
+%% Set a different default language in a virtual host.
+%%
+%%{host_config, "localhost",
+%% [{language, "ru"}]
+%%}.
+
+%%%.   ================
+%%%'   MISCELLANEOUS
+
+{all_metrics_are_global, false }.
+
+%%%.   ========
+%%%'   SERVICES
+
+%% Unlike modules, services are started per node and provide either features which are not
+%% related to any particular host, or backend stuff which is used by modules.
+%% This is handled by `mongoose_service` module.
+
+{services,
+    [
+        {service_admin_extra, [{submods, [node, accounts, sessions, vcard,
+                                          roster, last, private, stanza, stats]}]}
+    ]
+}.
+
+%%%.   =======
+%%%'   MODULES
+
+%%
+%% Modules enabled in all mongooseim virtual hosts.
+%% For list of possible modules options, check documentation.
+%%
+{modules,
+ [
+
+  %% The format for a single route is as follows:
+  %% {Host, Path, Method, Upstream}
+  %%
+  %% "_" can be used as wildcard for Host, Path and Method
+  %% Upstream can be either host (just http(s)://host:port) or uri
+  %% The difference is that host upstreams append whole path while
+  %% uri upstreams append only remainder that follows the matched Path
+  %% (this behaviour is similar to nginx's proxy_pass rules)
+  %%
+  %% Bindings can be used to match certain parts of host or path.
+  %% They will be later overlaid with parts of the upstream uri.
+  %%
+  %% {mod_revproxy,
+  %%    [{routes, [{"www.erlang-solutions.com", "/admin", "_",
+  %%                "https://www.erlang-solutions.com/"},
+  %%               {":var.com", "/:var", "_", "http://localhost:8080/"},
+  %%               {":domain.com", "/", "_", "http://localhost:8080/:domain"}]
+  %%     }]},
+
+% {mod_http_upload, [
+    %% Set max file size in bytes. Defaults to 10 MB.
+    %% Disabled if value is `undefined`.
+%   {max_file_size, 1024},
+    %% Use S3 storage backend
+%   {backend, s3},
+    %% Set options for S3 backend
+%   {s3, [
+%     {bucket_url, "http://s3-eu-west-1.amazonaws.com/konbucket2"},
+%     {region, "eu-west-1"},
+%     {access_key_id, "AKIAIAOAONIULXQGMOUA"},
+%     {secret_access_key, "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"}
+%   ]}
+% ]},
+
+  {mod_adhoc, []},
+  
+  {mod_disco, [{users_can_see_hidden_services, false}]},
+  {mod_commands, []},
+  {mod_muc_commands, []},
+  {mod_muc_light_commands, []},
+  {mod_last, []},
+  {mod_stream_management, [
+                           % default 100
+                           % size of a buffer of unacked messages
+                           % {buffer_max, 100}
+
+                           % default 1 - server sends the ack request after each stanza
+                           % {ack_freq, 1}
+
+                           % default: 600 seconds
+                           % {resume_timeout, 600}
+                          ]},
+  %% {mod_muc_light, [{host, "muclight.@HOST@"}]},
+  %% {mod_muc, [{host, "muc.@HOST@"},
+  %%            {access, muc},
+  %%            {access_create, muc_create}
+  %%           ]},
+  %% {mod_muc_log, [
+  %%                {outdir, "/tmp/muclogs"},
+  %%                {access_log, muc}
+  %%               ]},
+  {mod_offline, [{access_max_user_messages, max_user_offline_messages}]},
+  {mod_privacy, []},
+  {mod_blocking, []},
+  {mod_private, []},
+% {mod_private, [{backend, mnesia}]},
+% {mod_private, [{backend, rdbms}]},
+% {mod_register, [
+%		  %%
+%		  %% Set the minimum informational entropy for passwords.
+%		  %%
+%		  %%{password_strength, 32},
+%
+%		  %%
+%		  %% After successful registration, the user receives
+%		  %% a message with this subject and body.
+%		  %%
+%		  {welcome_message, {""}},
+%
+%		  %%
+%		  %% When a user registers, send a notification to
+%		  %% these XMPP accounts.
+%		  %%
+%
+%
+%		  %%
+%		  %% Only clients in the server machine can register accounts
+%		  %%
+%		  {ip_access, [{allow, "127.0.0.0/8"},
+%			       {deny, "0.0.0.0/0"}]},
+%
+%		  %%
+%		  %% Local c2s or remote s2s users cannot register accounts
+%		  %%
+%		  %%{access_from, deny},
+%
+%		  {access, register}
+%		 ]},
+  {mod_roster, []},
+  {mod_sic, []},
+  {mod_vcard, [%{matches, 1},
+%{search, true},
+%{ldap_search_operator, 'or'}, %% either 'or' or 'and'
+%{ldap_binary_search_fields, [<<"PHOTO">>]},
+%% list of binary search fields (as in vcard after mapping)
+{host, "vjud.@HOST@"}
+]},
+  {mod_bosh, []},
+  {mod_carboncopy, []}
+
+  %%
+  %% Message Archive Management (MAM, XEP-0313) for registered users and
+  %% Multi-User chats (MUCs).
+  %%
+
+% {mod_mam_meta, [
+    %% Use RDBMS backend (default)
+%   {backend, rdbms},
+
+    %% Do not store user preferences (default)
+%   {user_prefs_store, false},
+    %% Store user preferences in RDBMS
+%   {user_prefs_store, rdbms},
+    %% Store user preferences in Mnesia (recommended).
+    %% The preferences store will be called each time, as a message is routed.
+    %% That is why Mnesia is better suited for this job.
+%   {user_prefs_store, mnesia},
+
+    %% Enables a pool of asynchronous writers. (default)
+    %% Messages will be grouped together based on archive id.
+%   {async_writer, true},
+
+    %% Cache information about users (default)
+%   {cache_users, true},
+
+    %% Enable archivization for private messages (default)
+%   {pm, [
+      %% Top-level options can be overriden here if needed, for example:
+%     {async_writer, false}
+%   ]},
+
+    %%
+    %% Message Archive Management (MAM) for multi-user chats (MUC).
+    %% Enable XEP-0313 for "muc.@HOST@".
+    %%
+%   {muc, [
+%     {host, "muc.@HOST@"}
+      %% As with pm, top-level options can be overriden for MUC archive
+%   ]},
+%
+    %% Do not use a <stanza-id/> element (by default stanzaid is used)
+%   no_stanzaid_element,
+% ]},
+
+
+  %%
+  %% MAM configuration examples
+  %%
+
+  %% Only MUC, no user-defined preferences, good performance.
+% {mod_mam_meta, [
+%   {backend, rdbms},
+%   {pm, false},
+%   {muc, [
+%     {host, "muc.@HOST@"}
+%   ]}
+% ]},
+
+  %% Only archives for c2c messages, good performance.
+% {mod_mam_meta, [
+%   {backend, rdbms},
+%   {pm, [
+%     {user_prefs_store, mnesia}
+%   ]}
+% ]},
+
+  %% Basic configuration for c2c messages, bad performance, easy to debug.
+% {mod_mam_meta, [
+%   {backend, rdbms},
+%   {async_writer, false},
+%   {cache_users, false}
+% ]},
+
+  %% Cassandra archive for c2c and MUC conversations.
+  %% No custom settings supported (always archive).
+% {mod_mam_meta, [
+%   {backend, cassandra},
+%   {user_prefs_store, cassandra},
+%   {muc, [{host, "muc.@HOST@"}]}
+% ]}
+
+% {mod_event_pusher, [
+%   {backends, [
+%     %%
+%     %% Configuration for Amazon SNS notifications.
+%     %%
+%     {sns, [
+%       %% AWS credentials, region and host configuration
+%       {access_key_id, "AKIAJAZYHOIPY6A2PESA"},
+%       {secret_access_key, "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"},
+%       {region, "eu-west-1"},
+%       {account_id, "251423380551"},
+%       {region, "eu-west-1"},
+%       {sns_host, "sns.eu-west-1.amazonaws.com"},
+%
+%       %% Messages from this MUC host will be sent to the SNS topic
+%       {muc_host, "muc.@HOST@"},
+%
+%       %% Plugin module for defining custom message attributes and user identification
+%       {plugin_module, mod_event_pusher_sns_defaults},
+%
+%       %% Topic name configurations. Removing a topic will disable this specific SNS notification
+%       {presence_updates_topic, "user_presence_updated-dev-1"},  %% For presence updates
+%       {pm_messages_topic, "user_message_sent-dev-1"},           %% For private chat messages
+%       {muc_messages_topic, "user_messagegroup_sent-dev-1"}      %% For group chat messages
+%
+%       %% Pool options
+%       {pool_size, 100}, %% Worker pool size for publishing notifications
+%       {publish_retry_count, 2}, %% Retry count in case of publish error
+%       {publish_retry_time_ms, 50} %% Base exponential backoff time (in ms) for publish errors
+%      ]}
+%   ]}
+
+]}.
+
+
+%%
+%% Enable modules with custom options in a specific virtual host
+%%
+%%{host_config, "localhost",
+%% [{ {add, modules},
+%%   [
+%%    {mod_some_module, []}
+%%   ]
+%%  }
+%% ]}.
+
+%%%.
+%%%'
+
+%%% $Id$
+
+%%% Local Variables:
+%%% mode: erlang
+%%% End:
+%%% vim: set filetype=erlang tabstop=8 foldmarker=%%%',%%%. foldmethod=marker:
+%%%.