From 5c028b8f92aacb296afbd59130d848883f0c3a10 Mon Sep 17 00:00:00 2001 From: Sachin Joshi Date: Fri, 17 May 2019 12:20:31 +0545 Subject: [PATCH 01/94] user creation admin api will create multiple users --- CHANGELOG.md | 1 + .../web/admin_api/admin_api_controller.ex | 37 +++++---- .../web/admin_api/views/account_view.ex | 46 ++++++++++ lib/pleroma/web/router.ex | 2 +- .../admin_api/admin_api_controller_test.exs | 83 ++++++++++++++++++- 5 files changed, 149 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0e849285..5ee853c76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Configuration: Added `extra_cookie_attrs` for setting non-standard cookie attributes. Defaults to ["SameSite=Lax"] so that remote follows work. - Timelines: Messages involving people you have blocked will be excluded from the timeline in all cases instead of just repeats. - Admin API: Move the user related API to `api/pleroma/admin/users` +- Admin API: `POST /api/pleroma/admin/users` will take list of users - Pleroma API: Support for emoji tags in `/api/pleroma/emoji` resulting in a breaking API change - Mastodon API: Support for `exclude_types`, `limit` and `min_id` in `/api/v1/notifications` - Mastodon API: Add `languages` and `registrations` to `/api/v1/instance` diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index e00b33aba..6048ed35b 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -46,24 +46,31 @@ def user_unfollow(conn, %{"follower" => follower_nick, "followed" => followed_ni |> json("ok") end - def user_create( - conn, - %{"nickname" => nickname, "email" => email, "password" => password} - ) do - user_data = %{ - nickname: nickname, - name: nickname, - email: email, - password: password, - password_confirmation: password, - bio: "." - } + def users_create(conn, %{"users" => users}) do + result = + Enum.map(users, fn %{"nickname" => nickname, "email" => email, "password" => password} -> + user_data = %{ + nickname: nickname, + name: nickname, + email: email, + password: password, + password_confirmation: password, + bio: "." + } - changeset = User.register_changeset(%User{}, user_data, need_confirmation: false) - {:ok, user} = User.register(changeset) + changeset = User.register_changeset(%User{}, user_data, need_confirmation: false) + + case User.register(changeset) do + {:ok, user} -> + AccountView.render("created.json", %{user: user}) + + {:error, changeset} -> + AccountView.render("create-error.json", %{changeset: changeset}) + end + end) conn - |> json(user.nickname) + |> json(result) end def user_show(conn, %{"nickname" => nickname}) do diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex index 28bb667d8..e1825c5f1 100644 --- a/lib/pleroma/web/admin_api/views/account_view.ex +++ b/lib/pleroma/web/admin_api/views/account_view.ex @@ -44,4 +44,50 @@ def render("invites.json", %{invites: invites}) do invites: render_many(invites, AccountView, "invite.json", as: :invite) } end + + def render("created.json", %{user: user}) do + %{ + type: "success", + code: 201, + data: %{ + nickname: user.nickname, + email: user.email + } + } + end + + def render("create-error.json", %{changeset: %Ecto.Changeset{changes: changes, errors: errors}}) do + %{ + type: "error", + code: 409, + error: parse_error(errors), + data: %{ + nickname: Map.get(changes, :nickname), + email: Map.get(changes, :email) + } + } + end + + defp parse_error([]), do: "" + + defp parse_error(errors) do + ## when nickname is duplicate ap_id constraint error is raised + nickname_error = Keyword.get(errors, :nickname) || Keyword.get(errors, :ap_id) + email_error = Keyword.get(errors, :email) + password_error = Keyword.get(errors, :password) + + cond do + nickname_error -> + "nickname #{elem(nickname_error, 0)}" + + email_error -> + "email #{elem(email_error, 0)}" + + password_error -> + "password #{elem(password_error, 0)}" + + true -> + "" + end + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 7fef82f82..bbc2fda9b 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -156,7 +156,7 @@ defmodule Pleroma.Web.Router do post("/user", AdminAPIController, :user_create) delete("/users", AdminAPIController, :user_delete) - post("/users", AdminAPIController, :user_create) + post("/users", AdminAPIController, :users_create) patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation) put("/users/tag", AdminAPIController, :tag_users) delete("/users/tag", AdminAPIController, :untag_users) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 6c1897b5a..a0c9fd56f 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -31,12 +31,87 @@ test "Create" do |> assign(:user, admin) |> put_req_header("accept", "application/json") |> post("/api/pleroma/admin/users", %{ - "nickname" => "lain", - "email" => "lain@example.org", - "password" => "test" + "users" => [ + %{ + "nickname" => "lain", + "email" => "lain@example.org", + "password" => "test" + } + ] }) - assert json_response(conn, 200) == "lain" + assert json_response(conn, 200) == [ + %{ + "code" => 201, + "data" => %{ + "email" => "lain@example.org", + "nickname" => "lain" + }, + "type" => "success" + } + ] + end + + test "Cannot create user with exisiting email" do + admin = insert(:user, info: %{is_admin: true}) + user = insert(:user) + + conn = + build_conn() + |> assign(:user, admin) + |> put_req_header("accept", "application/json") + |> post("/api/pleroma/admin/users", %{ + "users" => [ + %{ + "nickname" => "lain", + "email" => user.email, + "password" => "test" + } + ] + }) + + assert json_response(conn, 200) == [ + %{ + "code" => 409, + "data" => %{ + "email" => user.email, + "nickname" => "lain" + }, + "error" => "email has already been taken", + "type" => "error" + } + ] + end + + test "Cannot create user with exisiting nickname" do + admin = insert(:user, info: %{is_admin: true}) + user = insert(:user) + + conn = + build_conn() + |> assign(:user, admin) + |> put_req_header("accept", "application/json") + |> post("/api/pleroma/admin/users", %{ + "users" => [ + %{ + "nickname" => user.nickname, + "email" => "someuser@plerama.social", + "password" => "test" + } + ] + }) + + assert json_response(conn, 200) == [ + %{ + "code" => 409, + "data" => %{ + "email" => "someuser@plerama.social", + "nickname" => user.nickname + }, + "error" => "nickname has already been taken", + "type" => "error" + } + ] end end From 5534d4c67675901ab272ee47355ad43dfae99033 Mon Sep 17 00:00:00 2001 From: Sachin Joshi Date: Sat, 1 Jun 2019 11:17:53 +0545 Subject: [PATCH 02/94] make bulk user creation from admin works as a transaction --- lib/pleroma/user.ex | 8 ++- .../web/admin_api/admin_api_controller.ex | 45 +++++++++---- .../web/admin_api/views/account_view.ex | 2 +- .../admin_api/admin_api_controller_test.exs | 66 ++++++++++++++++++- 4 files changed, 104 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index c6a562a61..722e8ff6b 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -276,7 +276,13 @@ defp autofollow_users(user) do @doc "Inserts provided changeset, performs post-registration actions (confirmation email sending etc.)" def register(%Ecto.Changeset{} = changeset) do with {:ok, user} <- Repo.insert(changeset), - {:ok, user} <- autofollow_users(user), + {:ok, user} <- post_register_action(user) do + {:ok, user} + end + end + + def post_register_action(%User{} = user) do + with {:ok, user} <- autofollow_users(user), {:ok, user} <- set_cache(user), {:ok, _} <- Pleroma.User.WelcomeMessage.post_welcome_message_to_user(user), {:ok, _} <- try_send_confirmation_email(user) do diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 6048ed35b..60fd4e571 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -47,7 +47,7 @@ def user_unfollow(conn, %{"follower" => follower_nick, "followed" => followed_ni end def users_create(conn, %{"users" => users}) do - result = + changesets = Enum.map(users, fn %{"nickname" => nickname, "email" => email, "password" => password} -> user_data = %{ nickname: nickname, @@ -58,19 +58,40 @@ def users_create(conn, %{"users" => users}) do bio: "." } - changeset = User.register_changeset(%User{}, user_data, need_confirmation: false) - - case User.register(changeset) do - {:ok, user} -> - AccountView.render("created.json", %{user: user}) - - {:error, changeset} -> - AccountView.render("create-error.json", %{changeset: changeset}) - end + User.register_changeset(%User{}, user_data, need_confirmation: false) + end) + |> Enum.reduce(Ecto.Multi.new(), fn changeset, multi -> + Ecto.Multi.insert(multi, Ecto.UUID.generate(), changeset) end) - conn - |> json(result) + case Pleroma.Repo.transaction(changesets) do + {:ok, users} -> + res = + users + |> Map.values() + |> Enum.map(fn user -> + {:ok, user} = User.post_register_action(user) + user + end) + |> Enum.map(&AccountView.render("created.json", %{user: &1})) + + conn + |> json(res) + + {:error, id, changeset, _} -> + res = + Enum.map(changesets.operations, fn + {current_id, {:changeset, _current_changeset, _}} when current_id == id -> + AccountView.render("create-error.json", %{changeset: changeset}) + + {_, {:changeset, current_changeset, _}} -> + AccountView.render("create-error.json", %{changeset: current_changeset}) + end) + + conn + |> put_status(:conflict) + |> json(res) + end end def user_show(conn, %{"nickname" => nickname}) do diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex index e1825c5f1..cccdeff7e 100644 --- a/lib/pleroma/web/admin_api/views/account_view.ex +++ b/lib/pleroma/web/admin_api/views/account_view.ex @@ -48,7 +48,7 @@ def render("invites.json", %{invites: invites}) do def render("created.json", %{user: user}) do %{ type: "success", - code: 201, + code: 200, data: %{ nickname: user.nickname, email: user.email diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index a0c9fd56f..019905137 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -36,18 +36,31 @@ test "Create" do "nickname" => "lain", "email" => "lain@example.org", "password" => "test" + }, + %{ + "nickname" => "lain2", + "email" => "lain2@example.org", + "password" => "test" } ] }) assert json_response(conn, 200) == [ %{ - "code" => 201, + "code" => 200, "data" => %{ "email" => "lain@example.org", "nickname" => "lain" }, "type" => "success" + }, + %{ + "code" => 200, + "data" => %{ + "email" => "lain2@example.org", + "nickname" => "lain2" + }, + "type" => "success" } ] end @@ -70,7 +83,7 @@ test "Cannot create user with exisiting email" do ] }) - assert json_response(conn, 200) == [ + assert json_response(conn, 409) == [ %{ "code" => 409, "data" => %{ @@ -101,7 +114,7 @@ test "Cannot create user with exisiting nickname" do ] }) - assert json_response(conn, 200) == [ + assert json_response(conn, 409) == [ %{ "code" => 409, "data" => %{ @@ -113,6 +126,53 @@ test "Cannot create user with exisiting nickname" do } ] end + + test "Multiple user creation works in transaction" do + admin = insert(:user, info: %{is_admin: true}) + user = insert(:user) + + conn = + build_conn() + |> assign(:user, admin) + |> put_req_header("accept", "application/json") + |> post("/api/pleroma/admin/users", %{ + "users" => [ + %{ + "nickname" => "newuser", + "email" => "newuser@pleroma.social", + "password" => "test" + }, + %{ + "nickname" => "lain", + "email" => user.email, + "password" => "test" + } + ] + }) + + assert json_response(conn, 409) == [ + %{ + "code" => 409, + "data" => %{ + "email" => user.email, + "nickname" => "lain" + }, + "error" => "email has already been taken", + "type" => "error" + }, + %{ + "code" => 409, + "data" => %{ + "email" => "newuser@pleroma.social", + "nickname" => "newuser" + }, + "error" => "", + "type" => "error" + } + ] + + assert User.get_by_nickname("newuser") === nil + end end describe "/api/pleroma/admin/users/:nickname" do From e394fc2eefdd7a4c7edd5fb3c04b445215d4a86c Mon Sep 17 00:00:00 2001 From: Sachin Joshi Date: Sun, 2 Jun 2019 09:48:45 +0545 Subject: [PATCH 03/94] fix the flaky test for users creation by admin --- .../admin_api/admin_api_controller_test.exs | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 9721a4034..86b160246 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -47,24 +47,8 @@ test "Create" do ] }) - assert json_response(conn, 200) == [ - %{ - "code" => 200, - "data" => %{ - "email" => "lain@example.org", - "nickname" => "lain" - }, - "type" => "success" - }, - %{ - "code" => 200, - "data" => %{ - "email" => "lain2@example.org", - "nickname" => "lain2" - }, - "type" => "success" - } - ] + response = json_response(conn, 200) |> Enum.map(&Map.get(&1, "type")) + assert response == ["success", "success"] end test "Cannot create user with exisiting email" do From 8ba7a151adf77c5cc47d6e1364a6078cc4bdef98 Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Mon, 22 Jul 2019 16:45:54 +0200 Subject: [PATCH 04/94] Cleanup: fix a comment --- test/web/mastodon_api/mastodon_api_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index ce2e44499..b5279412f 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -405,7 +405,7 @@ test "direct timeline", %{conn: conn} do assert %{"visibility" => "direct"} = status assert status["url"] != direct.data["id"] - # User should be able to see his own direct message + # User should be able to see their own direct message res_conn = build_conn() |> assign(:user, user_one) From b72940277470c67802b979e4cab44f277e8fffb3 Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Mon, 22 Jul 2019 09:10:30 +0200 Subject: [PATCH 05/94] Make test.exs read config in the same way as dev.exs This way, if your test.secret.exs has an error, you'll actually see it. --- config/test.exs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/config/test.exs b/config/test.exs index 92dca18bc..3f606aa81 100644 --- a/config/test.exs +++ b/config/test.exs @@ -82,11 +82,10 @@ config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock -try do +if File.exists?("./config/test.secret.exs") do import_config "test.secret.exs" -rescue - _ -> - IO.puts( - "You may want to create test.secret.exs to declare custom database connection parameters." - ) +else + IO.puts( + "You may want to create test.secret.exs to declare custom database connection parameters." + ) end From 666514194a325e2463c05bae516b89d7c5f59316 Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Mon, 22 Jul 2019 14:16:20 +0200 Subject: [PATCH 06/94] Add activity expirations table Add a table to store activity expirations. An activity can have zero or one expirations. The expiration has a scheduled_at field which stores the time at which the activity should expire and be deleted. --- lib/pleroma/activity.ex | 3 ++ lib/pleroma/activity_expiration.ex | 31 +++++++++++++++++++ .../20190716100804_add_expirations_table.exs | 10 ++++++ test/activity_expiration_test.exs | 21 +++++++++++++ test/activity_test.exs | 9 ++++++ test/support/factory.ex | 19 +++++++++++- 6 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 lib/pleroma/activity_expiration.ex create mode 100644 priv/repo/migrations/20190716100804_add_expirations_table.exs create mode 100644 test/activity_expiration_test.exs diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 46552c7be..be4850560 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Activity do use Ecto.Schema alias Pleroma.Activity + alias Pleroma.ActivityExpiration alias Pleroma.Bookmark alias Pleroma.Notification alias Pleroma.Object @@ -59,6 +60,8 @@ defmodule Pleroma.Activity do # typical case. has_one(:object, Object, on_delete: :nothing, foreign_key: :id) + has_one(:expiration, ActivityExpiration, on_delete: :delete_all) + timestamps() end diff --git a/lib/pleroma/activity_expiration.ex b/lib/pleroma/activity_expiration.ex new file mode 100644 index 000000000..d3d95f9e9 --- /dev/null +++ b/lib/pleroma/activity_expiration.ex @@ -0,0 +1,31 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ActivityExpiration do + use Ecto.Schema + + alias Pleroma.Activity + alias Pleroma.ActivityExpiration + alias Pleroma.FlakeId + alias Pleroma.Repo + + import Ecto.Query + + @type t :: %__MODULE__{} + + schema "activity_expirations" do + belongs_to(:activity, Activity, type: FlakeId) + field(:scheduled_at, :naive_datetime) + end + + def due_expirations(offset \\ 0) do + naive_datetime = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(offset, :millisecond) + + ActivityExpiration + |> where([exp], exp.scheduled_at < ^naive_datetime) + |> Repo.all() + end +end diff --git a/priv/repo/migrations/20190716100804_add_expirations_table.exs b/priv/repo/migrations/20190716100804_add_expirations_table.exs new file mode 100644 index 000000000..fbde8f9d6 --- /dev/null +++ b/priv/repo/migrations/20190716100804_add_expirations_table.exs @@ -0,0 +1,10 @@ +defmodule Pleroma.Repo.Migrations.AddExpirationsTable do + use Ecto.Migration + + def change do + create_if_not_exists table(:activity_expirations) do + add(:activity_id, references(:activities, type: :uuid, on_delete: :delete_all)) + add(:scheduled_at, :naive_datetime, null: false) + end + end +end diff --git a/test/activity_expiration_test.exs b/test/activity_expiration_test.exs new file mode 100644 index 000000000..20566a186 --- /dev/null +++ b/test/activity_expiration_test.exs @@ -0,0 +1,21 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ActivityExpirationTest do + use Pleroma.DataCase + alias Pleroma.ActivityExpiration + import Pleroma.Factory + + test "finds activities due to be deleted only" do + activity = insert(:note_activity) + expiration_due = insert(:expiration_in_the_past, %{activity_id: activity.id}) + activity2 = insert(:note_activity) + insert(:expiration_in_the_future, %{activity_id: activity2.id}) + + expirations = ActivityExpiration.due_expirations() + + assert length(expirations) == 1 + assert hd(expirations) == expiration_due + end +end diff --git a/test/activity_test.exs b/test/activity_test.exs index b27f6fd36..785c4b3cf 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -164,4 +164,13 @@ test "find all statuses for unauthenticated users when `limit_to_local_content` Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) end end + + test "add an activity with an expiration" do + activity = insert(:note_activity) + insert(:expiration_in_the_future, %{activity_id: activity.id}) + + Pleroma.ActivityExpiration + |> where([a], a.activity_id == ^activity.id) + |> Repo.one!() + end end diff --git a/test/support/factory.ex b/test/support/factory.ex index c751546ce..7b52b1328 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Factory do @@ -142,6 +142,23 @@ def note_activity_factory(attrs \\ %{}) do |> Map.merge(attrs) end + defp expiration_offset_by_minutes(attrs, minutes) do + %Pleroma.ActivityExpiration{} + |> Map.merge(attrs) + |> Map.put( + :scheduled_at, + NaiveDateTime.add(NaiveDateTime.utc_now(), :timer.minutes(minutes), :millisecond) + ) + end + + def expiration_in_the_past_factory(attrs \\ %{}) do + expiration_offset_by_minutes(attrs, -60) + end + + def expiration_in_the_future_factory(attrs \\ %{}) do + expiration_offset_by_minutes(attrs, 60) + end + def article_activity_factory do article = insert(:article) From 378f5f0fbe21c2533719fed9afe8313586fda5d5 Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Mon, 22 Jul 2019 14:18:58 +0200 Subject: [PATCH 07/94] Add activity expiration worker This is a worker that runs every minute and deletes expired activities. It's based heavily on the scheduled activities worker. --- config/config.exs | 3 ++ docs/config.md | 4 ++ lib/pleroma/activity_expiration_worker.ex | 62 +++++++++++++++++++++++ lib/pleroma/application.ex | 4 ++ test/activity_expiration_worker_test.exs | 17 +++++++ 5 files changed, 90 insertions(+) create mode 100644 lib/pleroma/activity_expiration_worker.ex create mode 100644 test/activity_expiration_worker_test.exs diff --git a/config/config.exs b/config/config.exs index 569411866..2887353fb 100644 --- a/config/config.exs +++ b/config/config.exs @@ -447,6 +447,7 @@ max_retries: 5 config :pleroma_job_queue, :queues, + activity_expiration: 10, federator_incoming: 50, federator_outgoing: 50, web_push: 50, @@ -536,6 +537,8 @@ status_id_action: {60_000, 3}, password_reset: {1_800_000, 5} +config :pleroma, Pleroma.ActivityExpiration, enabled: true + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/docs/config.md b/docs/config.md index 02f86dc16..a20ed704f 100644 --- a/docs/config.md +++ b/docs/config.md @@ -484,6 +484,10 @@ config :auto_linker, * `total_user_limit`: the number of scheduled activities a user is allowed to create in total (Default: `300`) * `enabled`: whether scheduled activities are sent to the job queue to be executed +## Pleroma.ActivityExpiration + +# `enabled`: whether expired activities will be sent to the job queue to be deleted + ## Pleroma.Web.Auth.Authenticator * `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator diff --git a/lib/pleroma/activity_expiration_worker.ex b/lib/pleroma/activity_expiration_worker.ex new file mode 100644 index 000000000..a341f58df --- /dev/null +++ b/lib/pleroma/activity_expiration_worker.ex @@ -0,0 +1,62 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ActivityExpirationWorker do + alias Pleroma.Activity + alias Pleroma.ActivityExpiration + alias Pleroma.Config + alias Pleroma.Repo + alias Pleroma.User + alias Pleroma.Web.CommonAPI + require Logger + use GenServer + import Ecto.Query + + @schedule_interval :timer.minutes(1) + + def start_link do + GenServer.start_link(__MODULE__, nil) + end + + @impl true + def init(_) do + if Config.get([ActivityExpiration, :enabled]) do + schedule_next() + {:ok, nil} + else + :ignore + end + end + + def perform(:execute, expiration_id) do + try do + expiration = + ActivityExpiration + |> where([e], e.id == ^expiration_id) + |> Repo.one!() + + activity = Activity.get_by_id_with_object(expiration.activity_id) + user = User.get_by_ap_id(activity.object.data["actor"]) + CommonAPI.delete(activity.id, user) + rescue + error -> + Logger.error("#{__MODULE__} Couldn't delete expired activity: #{inspect(error)}") + end + end + + @impl true + def handle_info(:perform, state) do + ActivityExpiration.due_expirations(@schedule_interval) + |> Enum.each(fn expiration -> + PleromaJobQueue.enqueue(:activity_expiration, __MODULE__, [:execute, expiration.id]) + end) + + schedule_next() + {:noreply, state} + end + + defp schedule_next do + Process.send_after(self(), :perform, @schedule_interval) + end +end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 035331491..42e4a1dfa 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -115,6 +115,10 @@ def start(_type, _args) do %{ id: Pleroma.ScheduledActivityWorker, start: {Pleroma.ScheduledActivityWorker, :start_link, []} + }, + %{ + id: Pleroma.ActivityExpirationWorker, + start: {Pleroma.ActivityExpirationWorker, :start_link, []} } ] ++ hackney_pool_children() ++ diff --git a/test/activity_expiration_worker_test.exs b/test/activity_expiration_worker_test.exs new file mode 100644 index 000000000..939d912f1 --- /dev/null +++ b/test/activity_expiration_worker_test.exs @@ -0,0 +1,17 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ActivityExpirationWorkerTest do + use Pleroma.DataCase + alias Pleroma.Activity + import Pleroma.Factory + + test "deletes an activity" do + activity = insert(:note_activity) + expiration = insert(:expiration_in_the_past, %{activity_id: activity.id}) + Pleroma.ActivityExpirationWorker.perform(:execute, expiration.id) + + refute Repo.get(Activity, activity.id) + end +end From 704960b3c135d2e050308c68f5ccf5d7b7df40f8 Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Mon, 22 Jul 2019 16:46:20 +0200 Subject: [PATCH 08/94] Add support for activity expiration to common and Masto API The "expires_at" parameter accepts an ISO8601-formatted date which defines when the activity will expire. At this point the API will not give you any feedback about if your post will expire or not. --- docs/api/differences_in_mastoapi_responses.md | 1 + lib/pleroma/activity_expiration.ex | 19 ++++++++++++ lib/pleroma/web/common_api/common_api.ex | 29 +++++++++++++------ test/support/factory.ex | 10 ++++--- test/web/common_api/common_api_test.exs | 17 +++++++++++ .../mastodon_api_controller_test.exs | 19 ++++++++++++ 6 files changed, 82 insertions(+), 13 deletions(-) diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index 1907d70c8..7d5be4713 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -79,6 +79,7 @@ Additional parameters can be added to the JSON body/Form data: - `content_type`: string, contain the MIME type of the status, it is transformed into HTML by the backend. You can get the list of the supported MIME types with the nodeinfo endpoint. - `to`: A list of nicknames (like `lain@soykaf.club` or `lain` on the local server) that will be used to determine who is going to be addressed by this post. Using this will disable the implicit addressing by mentioned names in the `status` body, only the people in the `to` list will be addressed. The normal rules for for post visibility are not affected by this and will still apply. - `visibility`: string, besides standard MastoAPI values (`direct`, `private`, `unlisted` or `public`) it can be used to address a List by setting it to `list:LIST_ID`. +- `expires_on`: datetime (iso8601), sets when the posted activity should expire. When a posted activity expires it will be deleted from the server, and a delete request for it will be federated. ## PATCH `/api/v1/update_credentials` diff --git a/lib/pleroma/activity_expiration.ex b/lib/pleroma/activity_expiration.ex index d3d95f9e9..a0af5255b 100644 --- a/lib/pleroma/activity_expiration.ex +++ b/lib/pleroma/activity_expiration.ex @@ -10,6 +10,7 @@ defmodule Pleroma.ActivityExpiration do alias Pleroma.FlakeId alias Pleroma.Repo + import Ecto.Changeset import Ecto.Query @type t :: %__MODULE__{} @@ -19,6 +20,24 @@ defmodule Pleroma.ActivityExpiration do field(:scheduled_at, :naive_datetime) end + def changeset(%ActivityExpiration{} = expiration, attrs) do + expiration + |> cast(attrs, [:scheduled_at]) + |> validate_required([:scheduled_at]) + end + + def get_by_activity_id(activity_id) do + ActivityExpiration + |> where([exp], exp.activity_id == ^activity_id) + |> Repo.one() + end + + def create(%Activity{} = activity, scheduled_at) do + %ActivityExpiration{activity_id: activity.id} + |> changeset(%{scheduled_at: scheduled_at}) + |> Repo.insert() + end + def due_expirations(offset \\ 0) do naive_datetime = NaiveDateTime.utc_now() diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 44af6a773..0f287af4e 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.Activity + alias Pleroma.ActivityExpiration alias Pleroma.Formatter alias Pleroma.Object alias Pleroma.ThreadMute @@ -218,6 +219,7 @@ def post(user, %{"status" => status} = data) do context <- make_context(in_reply_to), cw <- data["spoiler_text"] || "", sensitive <- data["sensitive"] || Enum.member?(tags, {"#nsfw", "nsfw"}), + {:ok, expires_at} <- Ecto.Type.cast(:naive_datetime, data["expires_at"]), full_payload <- String.trim(status <> cw), :ok <- validate_character_limit(full_payload, attachments, limit), object <- @@ -243,15 +245,24 @@ def post(user, %{"status" => status} = data) do preview? = Pleroma.Web.ControllerHelper.truthy_param?(data["preview"]) || false direct? = visibility == "direct" - %{ - to: to, - actor: user, - context: context, - object: object, - additional: %{"cc" => cc, "directMessage" => direct?} - } - |> maybe_add_list_data(user, visibility) - |> ActivityPub.create(preview?) + result = + %{ + to: to, + actor: user, + context: context, + object: object, + additional: %{"cc" => cc, "directMessage" => direct?} + } + |> maybe_add_list_data(user, visibility) + |> ActivityPub.create(preview?) + + if expires_at do + with {:ok, activity} <- result do + ActivityExpiration.create(activity, expires_at) + end + end + + result else {:private_to_public, true} -> {:error, dgettext("errors", "The message visibility must be direct")} diff --git a/test/support/factory.ex b/test/support/factory.ex index 7b52b1328..63fe3a66d 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -143,12 +143,14 @@ def note_activity_factory(attrs \\ %{}) do end defp expiration_offset_by_minutes(attrs, minutes) do + scheduled_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(minutes), :millisecond) + |> NaiveDateTime.truncate(:second) + %Pleroma.ActivityExpiration{} |> Map.merge(attrs) - |> Map.put( - :scheduled_at, - NaiveDateTime.add(NaiveDateTime.utc_now(), :timer.minutes(minutes), :millisecond) - ) + |> Map.put(:scheduled_at, scheduled_at) end def expiration_in_the_past_factory(attrs \\ %{}) do diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 16b3f121d..210314a4a 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -160,6 +160,23 @@ test "it returns error when character limit is exceeded" do Pleroma.Config.put([:instance, :limit], limit) end + + test "it can handle activities that expire" do + user = insert(:user) + + expires_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.truncate(:second) + |> NaiveDateTime.add(1_000_000, :second) + + expires_at_iso8601 = expires_at |> NaiveDateTime.to_iso8601() + + assert {:ok, activity} = + CommonAPI.post(user, %{"status" => "chai", "expires_at" => expires_at_iso8601}) + + assert expiration = Pleroma.ActivityExpiration.get_by_activity_id(activity.id) + assert expiration.scheduled_at == expires_at + end end describe "reactions" do diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index b5279412f..24482a4a2 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do alias Ecto.Changeset alias Pleroma.Activity + alias Pleroma.ActivityExpiration alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo @@ -151,6 +152,24 @@ test "posting a status", %{conn: conn} do assert %{"id" => third_id} = json_response(conn_three, 200) refute id == third_id + + # An activity that will expire: + expires_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(120), :millisecond) + |> NaiveDateTime.truncate(:second) + + conn_four = + conn + |> post("api/v1/statuses", %{ + "status" => "oolong", + "expires_at" => expires_at + }) + + assert %{"id" => fourth_id} = json_response(conn_four, 200) + assert activity = Activity.get_by_id(fourth_id) + assert expiration = ActivityExpiration.get_by_activity_id(fourth_id) + assert expiration.scheduled_at == expires_at end test "replying to a status", %{conn: conn} do From 36012ef6c1dfea2489e61063e14783fa3fb52700 Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Tue, 23 Jul 2019 16:33:45 +0200 Subject: [PATCH 09/94] Require that ephemeral posts live for at least one hour If we didn't put some kind of lifetime requirement on these, I guess you could annoy people by sending large numbers of ephemeral posts that provoke notifications but then disappear before anyone can read them. --- lib/pleroma/activity_expiration.ex | 18 ++++++++++++++++++ lib/pleroma/web/common_api/common_api.ex | 14 ++++++++++++-- test/activity_expiration_test.exs | 6 ++++++ test/support/factory.ex | 2 +- 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/activity_expiration.ex b/lib/pleroma/activity_expiration.ex index a0af5255b..bf57abca4 100644 --- a/lib/pleroma/activity_expiration.ex +++ b/lib/pleroma/activity_expiration.ex @@ -14,6 +14,7 @@ defmodule Pleroma.ActivityExpiration do import Ecto.Query @type t :: %__MODULE__{} + @min_activity_lifetime :timer.hours(1) schema "activity_expirations" do belongs_to(:activity, Activity, type: FlakeId) @@ -24,6 +25,7 @@ def changeset(%ActivityExpiration{} = expiration, attrs) do expiration |> cast(attrs, [:scheduled_at]) |> validate_required([:scheduled_at]) + |> validate_scheduled_at() end def get_by_activity_id(activity_id) do @@ -47,4 +49,20 @@ def due_expirations(offset \\ 0) do |> where([exp], exp.scheduled_at < ^naive_datetime) |> Repo.all() end + + def validate_scheduled_at(changeset) do + validate_change(changeset, :scheduled_at, fn _, scheduled_at -> + if not expires_late_enough?(scheduled_at) do + [scheduled_at: "an ephemeral activity must live for at least one hour"] + else + [] + end + end) + end + + def expires_late_enough?(scheduled_at) do + now = NaiveDateTime.utc_now() + diff = NaiveDateTime.diff(scheduled_at, now, :millisecond) + diff >= @min_activity_lifetime + end end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 0f287af4e..261d60392 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -196,6 +196,16 @@ def get_replied_to_visibility(activity) do end end + defp check_expiry_date(expiry_str) do + {:ok, expiry} = Ecto.Type.cast(:naive_datetime, expiry_str) + + if is_nil(expiry) || ActivityExpiration.expires_late_enough?(expiry) do + {:ok, expiry} + else + {:error, "Expiry date is too soon"} + end + end + def post(user, %{"status" => status} = data) do limit = Pleroma.Config.get([:instance, :limit]) @@ -219,7 +229,7 @@ def post(user, %{"status" => status} = data) do context <- make_context(in_reply_to), cw <- data["spoiler_text"] || "", sensitive <- data["sensitive"] || Enum.member?(tags, {"#nsfw", "nsfw"}), - {:ok, expires_at} <- Ecto.Type.cast(:naive_datetime, data["expires_at"]), + {:ok, expires_at} <- check_expiry_date(data["expires_at"]), full_payload <- String.trim(status <> cw), :ok <- validate_character_limit(full_payload, attachments, limit), object <- @@ -258,7 +268,7 @@ def post(user, %{"status" => status} = data) do if expires_at do with {:ok, activity} <- result do - ActivityExpiration.create(activity, expires_at) + {:ok, _} = ActivityExpiration.create(activity, expires_at) end end diff --git a/test/activity_expiration_test.exs b/test/activity_expiration_test.exs index 20566a186..4948fae16 100644 --- a/test/activity_expiration_test.exs +++ b/test/activity_expiration_test.exs @@ -18,4 +18,10 @@ test "finds activities due to be deleted only" do assert length(expirations) == 1 assert hd(expirations) == expiration_due end + + test "denies expirations that don't live long enough" do + activity = insert(:note_activity) + now = NaiveDateTime.utc_now() + assert {:error, _} = ActivityExpiration.create(activity, now) + end end diff --git a/test/support/factory.ex b/test/support/factory.ex index 63fe3a66d..7a2ddcada 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -158,7 +158,7 @@ def expiration_in_the_past_factory(attrs \\ %{}) do end def expiration_in_the_future_factory(attrs \\ %{}) do - expiration_offset_by_minutes(attrs, 60) + expiration_offset_by_minutes(attrs, 61) end def article_activity_factory do From 3cb471ec0688b81c8ef37dd27f2b82e6c858431f Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Wed, 24 Jul 2019 12:43:20 +0200 Subject: [PATCH 10/94] Expose expires_at datetime in mastoAPI only for the activity actor In the "pleroma" section of the MastoAPI for status activities you can see an expires_at item that states when the activity will expire, or nothing if the activity will not expire. The expires_at date is only visible to the person who posted the activity. This is the conservative approach in case some attacker decides to write a logger for expiring posts. However, in the future of OCAP, signed requests, and all that stuff, this attack might not be that likely. Some other pleroma dev should remove the restriction in the code at that time, if they're satisfied with the security implications of doing so. --- docs/api/differences_in_mastoapi_responses.md | 1 + lib/pleroma/web/mastodon_api/views/status_view.ex | 13 ++++++++++++- .../mastodon_api/mastodon_api_controller_test.exs | 3 ++- test/web/mastodon_api/status_view_test.exs | 3 ++- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index 7d5be4713..168a13f4e 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -25,6 +25,7 @@ Has these additional fields under the `pleroma` object: - `in_reply_to_account_acct`: the `acct` property of User entity for replied user (if any) - `content`: a map consisting of alternate representations of the `content` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain` - `spoiler_text`: a map consisting of alternate representations of the `spoiler_text` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain` +- `expires_on`: a datetime (iso8601) that states when the post will expire (be deleted automatically), or empty if the post won't expire ## Attachments diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index de9425959..7264dcafb 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do use Pleroma.Web, :view alias Pleroma.Activity + alias Pleroma.ActivityExpiration alias Pleroma.HTML alias Pleroma.Object alias Pleroma.Repo @@ -165,6 +166,15 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity bookmarked = Activity.get_bookmark(activity, opts[:for]) != nil + client_posted_this_activity = opts[:for] && user.id == opts[:for].id + + expires_at = + with true <- client_posted_this_activity, + expiration when not is_nil(expiration) <- + ActivityExpiration.get_by_activity_id(activity.id) do + expiration.scheduled_at + end + thread_muted? = case activity.thread_muted? do thread_muted? when is_boolean(thread_muted?) -> thread_muted? @@ -262,7 +272,8 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity conversation_id: get_context_id(activity), in_reply_to_account_acct: reply_to_user && reply_to_user.nickname, content: %{"text/plain" => content_plaintext}, - spoiler_text: %{"text/plain" => summary_plaintext} + spoiler_text: %{"text/plain" => summary_plaintext}, + expires_at: expires_at } } end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 24482a4a2..e59908979 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -166,10 +166,11 @@ test "posting a status", %{conn: conn} do "expires_at" => expires_at }) - assert %{"id" => fourth_id} = json_response(conn_four, 200) + assert fourth_response = %{"id" => fourth_id} = json_response(conn_four, 200) assert activity = Activity.get_by_id(fourth_id) assert expiration = ActivityExpiration.get_by_activity_id(fourth_id) assert expiration.scheduled_at == expires_at + assert fourth_response["pleroma"]["expires_at"] == NaiveDateTime.to_iso8601(expires_at) end test "replying to a status", %{conn: conn} do diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index 3447c5b1f..073c69659 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -133,7 +133,8 @@ test "a note activity" do conversation_id: convo_id, in_reply_to_account_acct: nil, content: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["content"])}, - spoiler_text: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["summary"])} + spoiler_text: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["summary"])}, + expires_at: nil } } From 91d9fdc7decc664483625c11e44d4e053dd9c585 Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Wed, 24 Jul 2019 13:02:28 +0200 Subject: [PATCH 11/94] Update changelog to document expiring posts feature --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35a5a6c21..75d236af5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] +### Added +- Expiring/ephemeral activites. All activities can have expires_on value set, which controls when they should be deleted automatically. +- Mastodon API: in post_status, expires_at datetime parameter lets you set when an activity should expire +- Mastodon API: all status JSON responses contain a `pleroma.expires_in` item which states the number of minutes until an activity expires. The value is only shown to the user who created the activity. To everyone else it's empty. +- Configuration: `ActivityExpiration.enabled` controls whether expired activites will get deleted at the appropriate time. Enabled by default. + ### Changed - **Breaking:** Configuration: A setting to explicitly disable the mailer was added, defaulting to true, if you are using a mailer add `config :pleroma, Pleroma.Emails.Mailer, enabled: true` to your config - Configuration: OpenGraph and TwitterCard providers enabled by default From 2981821db834448bf9b2ba26590314e36201664c Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Wed, 24 Jul 2019 16:51:09 +0200 Subject: [PATCH 12/94] squash! Expose expires_at datetime in mastoAPI only for the activity actor NOTE: rewrite the commit msg --- docs/api/differences_in_mastoapi_responses.md | 2 +- lib/pleroma/web/mastodon_api/views/status_view.ex | 10 +++++++--- test/web/mastodon_api/mastodon_api_controller_test.exs | 2 +- test/web/mastodon_api/status_view_test.exs | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index 168a13f4e..829468b13 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -25,7 +25,7 @@ Has these additional fields under the `pleroma` object: - `in_reply_to_account_acct`: the `acct` property of User entity for replied user (if any) - `content`: a map consisting of alternate representations of the `content` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain` - `spoiler_text`: a map consisting of alternate representations of the `spoiler_text` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain` -- `expires_on`: a datetime (iso8601) that states when the post will expire (be deleted automatically), or empty if the post won't expire +- `expires_in`: the number of minutes until a post will expire (be deleted automatically), or empty if the post won't expire ## Attachments diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 7264dcafb..4a3686d72 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -168,11 +168,15 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity client_posted_this_activity = opts[:for] && user.id == opts[:for].id - expires_at = + expires_in = with true <- client_posted_this_activity, expiration when not is_nil(expiration) <- ActivityExpiration.get_by_activity_id(activity.id) do - expiration.scheduled_at + expires_in_seconds = + expiration.scheduled_at + |> NaiveDateTime.diff(NaiveDateTime.utc_now(), :second) + + round(expires_in_seconds / 60) end thread_muted? = @@ -273,7 +277,7 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity in_reply_to_account_acct: reply_to_user && reply_to_user.nickname, content: %{"text/plain" => content_plaintext}, spoiler_text: %{"text/plain" => summary_plaintext}, - expires_at: expires_at + expires_in: expires_in } } end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index e59908979..a9d38c06e 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -170,7 +170,7 @@ test "posting a status", %{conn: conn} do assert activity = Activity.get_by_id(fourth_id) assert expiration = ActivityExpiration.get_by_activity_id(fourth_id) assert expiration.scheduled_at == expires_at - assert fourth_response["pleroma"]["expires_at"] == NaiveDateTime.to_iso8601(expires_at) + assert fourth_response["pleroma"]["expires_in"] > 0 end test "replying to a status", %{conn: conn} do diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index 073c69659..eb0874ab2 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -134,7 +134,7 @@ test "a note activity" do in_reply_to_account_acct: nil, content: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["content"])}, spoiler_text: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["summary"])}, - expires_at: nil + expires_in: nil } } From 877575d0da830724e822eac2de243391aaea7ec8 Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Wed, 24 Jul 2019 17:07:51 +0200 Subject: [PATCH 13/94] fixup! Update changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75d236af5..f64506637 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Added - Expiring/ephemeral activites. All activities can have expires_on value set, which controls when they should be deleted automatically. -- Mastodon API: in post_status, expires_at datetime parameter lets you set when an activity should expire -- Mastodon API: all status JSON responses contain a `pleroma.expires_in` item which states the number of minutes until an activity expires. The value is only shown to the user who created the activity. To everyone else it's empty. +- Mastodon API: in post_status, the expires_in parameter lets you set the number of minutes until an activity expires. It must be at least one hour. +- Mastodon API: all status JSON responses contain a `pleroma.expires_on` item which states when an activity will expire. The value is only shown to the user who created the activity. To everyone else it's empty. - Configuration: `ActivityExpiration.enabled` controls whether expired activites will get deleted at the appropriate time. Enabled by default. ### Changed From 2c83eb0b157b2f574f55341e9171f0b5ab7bd3b2 Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Wed, 24 Jul 2019 17:09:59 +0200 Subject: [PATCH 14/94] Revert "squash! Expose expires_at datetime in mastoAPI only for the activity actor" This reverts commit 2981821db834448bf9b2ba26590314e36201664c. --- docs/api/differences_in_mastoapi_responses.md | 2 +- lib/pleroma/web/mastodon_api/views/status_view.ex | 10 +++------- test/web/mastodon_api/mastodon_api_controller_test.exs | 2 +- test/web/mastodon_api/status_view_test.exs | 2 +- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index 829468b13..168a13f4e 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -25,7 +25,7 @@ Has these additional fields under the `pleroma` object: - `in_reply_to_account_acct`: the `acct` property of User entity for replied user (if any) - `content`: a map consisting of alternate representations of the `content` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain` - `spoiler_text`: a map consisting of alternate representations of the `spoiler_text` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain` -- `expires_in`: the number of minutes until a post will expire (be deleted automatically), or empty if the post won't expire +- `expires_on`: a datetime (iso8601) that states when the post will expire (be deleted automatically), or empty if the post won't expire ## Attachments diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 4a3686d72..7264dcafb 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -168,15 +168,11 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity client_posted_this_activity = opts[:for] && user.id == opts[:for].id - expires_in = + expires_at = with true <- client_posted_this_activity, expiration when not is_nil(expiration) <- ActivityExpiration.get_by_activity_id(activity.id) do - expires_in_seconds = - expiration.scheduled_at - |> NaiveDateTime.diff(NaiveDateTime.utc_now(), :second) - - round(expires_in_seconds / 60) + expiration.scheduled_at end thread_muted? = @@ -277,7 +273,7 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity in_reply_to_account_acct: reply_to_user && reply_to_user.nickname, content: %{"text/plain" => content_plaintext}, spoiler_text: %{"text/plain" => summary_plaintext}, - expires_in: expires_in + expires_at: expires_at } } end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index a9d38c06e..e59908979 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -170,7 +170,7 @@ test "posting a status", %{conn: conn} do assert activity = Activity.get_by_id(fourth_id) assert expiration = ActivityExpiration.get_by_activity_id(fourth_id) assert expiration.scheduled_at == expires_at - assert fourth_response["pleroma"]["expires_in"] > 0 + assert fourth_response["pleroma"]["expires_at"] == NaiveDateTime.to_iso8601(expires_at) end test "replying to a status", %{conn: conn} do diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index eb0874ab2..073c69659 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -134,7 +134,7 @@ test "a note activity" do in_reply_to_account_acct: nil, content: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["content"])}, spoiler_text: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["summary"])}, - expires_in: nil + expires_at: nil } } From 0e2b5a3e6aed7947909c2a1ff1618403546f1572 Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Wed, 24 Jul 2019 17:25:11 +0200 Subject: [PATCH 15/94] WIP --- .../mastodon_api_controller_test.exs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index e59908979..fbe0ab375 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -154,23 +154,27 @@ test "posting a status", %{conn: conn} do refute id == third_id # An activity that will expire: - expires_at = - NaiveDateTime.utc_now() - |> NaiveDateTime.add(:timer.minutes(120), :millisecond) - |> NaiveDateTime.truncate(:second) + expires_in = 120 conn_four = conn |> post("api/v1/statuses", %{ "status" => "oolong", - "expires_at" => expires_at + "expires_in" => expires_in }) assert fourth_response = %{"id" => fourth_id} = json_response(conn_four, 200) assert activity = Activity.get_by_id(fourth_id) assert expiration = ActivityExpiration.get_by_activity_id(fourth_id) - assert expiration.scheduled_at == expires_at - assert fourth_response["pleroma"]["expires_at"] == NaiveDateTime.to_iso8601(expires_at) + + estimated_expires_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(expires_in), :millisecond) + |> NaiveDateTime.truncate(:second) + + # This assert will fail if the test takes longer than a minute. I sure hope it never does: + assert abs(NaiveDateTime.diff(expiration.scheduled_at, estimated_expires_at, :second)) < 60 + assert fourth_response["pleroma"]["expires_at"] == NaiveDateTime.to_iso8601(expiration.scheduled_at) end test "replying to a status", %{conn: conn} do From 7483679a7b6ff63c9c61c3df3e9e37f2c24012ff Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 31 Jul 2019 15:12:29 +0200 Subject: [PATCH 16/94] StatusView: Return direct conversation id. --- lib/pleroma/conversation/participation.ex | 8 ++++++++ .../web/mastodon_api/views/status_view.ex | 18 +++++++++++++++++- test/web/mastodon_api/status_view_test.exs | 18 +++++++++++++++++- 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index 5883e4183..77b3f61e9 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -65,6 +65,14 @@ def for_user(user, params \\ %{}) do |> Pleroma.Pagination.fetch_paginated(params) end + def for_user_and_conversation(user, conversation) do + from(p in __MODULE__, + where: p.user_id == ^user.id, + where: p.conversation_id == ^conversation.id + ) + |> Repo.one() + end + def for_user_with_last_activity_id(user, params \\ %{}) do for_user(user, params) |> Enum.map(fn participation -> diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 80df9b2ac..a862554b1 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do use Pleroma.Web, :view alias Pleroma.Activity + alias Pleroma.Conversation + alias Pleroma.Conversation.Participation alias Pleroma.HTML alias Pleroma.Object alias Pleroma.Repo @@ -225,6 +227,19 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity object.data["url"] || object.data["external_url"] || object.data["id"] end + direct_conversation_id = + with {_, true} <- {:include_id, opts[:with_direct_conversation_id]}, + {_, %User{} = for_user} <- {:for_user, opts[:for]}, + %{data: %{"context" => context}} when is_binary(context) <- activity, + %Conversation{} = conversation <- Conversation.get_for_ap_id(context), + %Participation{id: participation_id} <- + Participation.for_user_and_conversation(for_user, conversation) do + participation_id + else + _e -> + nil + end + %{ id: to_string(activity.id), uri: object.data["id"], @@ -262,7 +277,8 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity conversation_id: get_context_id(activity), in_reply_to_account_acct: reply_to_user && reply_to_user.nickname, content: %{"text/plain" => content_plaintext}, - spoiler_text: %{"text/plain" => summary_plaintext} + spoiler_text: %{"text/plain" => summary_plaintext}, + direct_conversation_id: direct_conversation_id } } end diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index 0b167f839..c983b494f 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -23,6 +23,21 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do :ok end + test "returns the direct conversation id when given the `with_conversation_id` option" do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!", "visibility" => "direct"}) + + status = + StatusView.render("status.json", + activity: activity, + with_direct_conversation_id: true, + for: user + ) + + assert status[:pleroma][:direct_conversation_id] + end + test "returns a temporary ap_id based user for activities missing db users" do user = insert(:user) @@ -133,7 +148,8 @@ test "a note activity" do conversation_id: convo_id, in_reply_to_account_acct: nil, content: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["content"])}, - spoiler_text: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["summary"])} + spoiler_text: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["summary"])}, + direct_conversation_id: nil } } From fd4b7239cd6f44a25c9aa4195750e94e0612a3b1 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 1 Aug 2019 17:25:46 +0200 Subject: [PATCH 17/94] nothing From f88560accd801ac88c60344cef93ef00cf136069 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 2 Aug 2019 11:55:41 +0200 Subject: [PATCH 18/94] Conversations: Add recipient list to conversation participation. This enables to address the same group of people every time. --- lib/pleroma/conversation.ex | 11 ++++++ lib/pleroma/conversation/participation.ex | 4 +++ .../participation_recipient_ship.ex | 34 +++++++++++++++++++ lib/pleroma/user.ex | 7 ++++ .../20190205114625_create_thread_mutes.exs | 2 +- ...ersation_participation_recipient_ships.exs | 13 +++++++ test/conversation/participation_test.exs | 30 ++++++++++++++++ 7 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 lib/pleroma/conversation/participation_recipient_ship.ex create mode 100644 priv/repo/migrations/20190801154554_create_conversation_participation_recipient_ships.exs diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index bc97b39ca..fb0dfedca 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Conversation do alias Pleroma.Conversation.Participation + alias Pleroma.Conversation.Participation.RecipientShip alias Pleroma.Repo alias Pleroma.User use Ecto.Schema @@ -39,6 +40,15 @@ def get_for_ap_id(ap_id) do Repo.get_by(__MODULE__, ap_id: ap_id) end + def maybe_set_recipients(participation, activity) do + participation = Repo.preload(participation, :recipients) + + if participation.recipients |> Enum.empty?() do + recipients = User.get_all_by_ap_id(activity.recipients) + RecipientShip.create(recipients, participation) + end + end + @doc """ This will 1. Create a conversation if there isn't one already @@ -60,6 +70,7 @@ def create_or_bump_for(activity, opts \\ []) do {:ok, participation} = Participation.create_for_user_and_conversation(user, conversation, opts) + maybe_set_recipients(participation, activity) participation end) diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index 77b3f61e9..121efb671 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Conversation.Participation do use Ecto.Schema alias Pleroma.Conversation + alias Pleroma.Conversation.Participation.RecipientShip alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub @@ -17,6 +18,9 @@ defmodule Pleroma.Conversation.Participation do field(:read, :boolean, default: false) field(:last_activity_id, Pleroma.FlakeId, virtual: true) + has_many(:recipient_ships, RecipientShip) + has_many(:recipients, through: [:recipient_ships, :user]) + timestamps() end diff --git a/lib/pleroma/conversation/participation_recipient_ship.ex b/lib/pleroma/conversation/participation_recipient_ship.ex new file mode 100644 index 000000000..27c0c89cd --- /dev/null +++ b/lib/pleroma/conversation/participation_recipient_ship.ex @@ -0,0 +1,34 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Conversation.Participation.RecipientShip do + use Ecto.Schema + + alias Pleroma.Conversation.Participation + alias Pleroma.User + alias Pleroma.Repo + + import Ecto.Changeset + + schema "conversation_participation_recipient_ships" do + belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:participation, Participation) + end + + def creation_cng(struct, params) do + struct + |> cast(params, [:user_id, :participation_id]) + |> validate_required([:user_id, :participation_id]) + end + + def create(%User{} = user, participation), do: create([user], participation) + + def create(users, participation) do + Enum.each(users, fn user -> + %__MODULE__{} + |> creation_cng(%{user_id: user.id, participation_id: participation.id}) + |> Repo.insert!() + end) + end +end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 6e2fd3af8..a021e77f0 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -450,6 +450,13 @@ def get_by_ap_id(ap_id) do Repo.get_by(User, ap_id: ap_id) end + def get_all_by_ap_id(ap_ids) do + from(u in __MODULE__, + where: u.ap_id in ^ap_ids + ) + |> Repo.all() + end + # This is mostly an SPC migration fix. This guesses the user nickname by taking the last part # of the ap_id and the domain and tries to get that user def get_by_guessed_nickname(ap_id) do diff --git a/priv/repo/migrations/20190205114625_create_thread_mutes.exs b/priv/repo/migrations/20190205114625_create_thread_mutes.exs index 7e44db121..baaf07253 100644 --- a/priv/repo/migrations/20190205114625_create_thread_mutes.exs +++ b/priv/repo/migrations/20190205114625_create_thread_mutes.exs @@ -6,7 +6,7 @@ def change do add :user_id, references(:users, type: :uuid, on_delete: :delete_all) add :context, :string end - + create_if_not_exists unique_index(:thread_mutes, [:user_id, :context], name: :unique_index) end end diff --git a/priv/repo/migrations/20190801154554_create_conversation_participation_recipient_ships.exs b/priv/repo/migrations/20190801154554_create_conversation_participation_recipient_ships.exs new file mode 100644 index 000000000..c6e3469d5 --- /dev/null +++ b/priv/repo/migrations/20190801154554_create_conversation_participation_recipient_ships.exs @@ -0,0 +1,13 @@ +defmodule Pleroma.Repo.Migrations.CreateConversationParticipationRecipientShips do + use Ecto.Migration + + def change do + create_if_not_exists table(:conversation_participation_recipient_ships) do + add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) + add(:participation_id, references(:conversation_participations, on_delete: :delete_all)) + end + + create_if_not_exists index(:conversation_participation_recipient_ships, [:user_id]) + create_if_not_exists index(:conversation_participation_recipient_ships, [:participation_id]) + end +end diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs index 2a03e5d67..4a3c397bd 100644 --- a/test/conversation/participation_test.exs +++ b/test/conversation/participation_test.exs @@ -8,6 +8,36 @@ defmodule Pleroma.Conversation.ParticipationTest do alias Pleroma.Conversation.Participation alias Pleroma.Web.CommonAPI + test "for a new conversation, it sets the recipents of the participation" do + user = insert(:user) + other_user = insert(:user) + third_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{"status" => "Hey @#{other_user.nickname}.", "visibility" => "direct"}) + + [participation] = Participation.for_user(user) + participation = Pleroma.Repo.preload(participation, :recipients) + + assert length(participation.recipients) == 2 + assert user in participation.recipients + assert other_user in participation.recipients + + # Mentioning another user in the same conversation will not add a new recipients. + + {:ok, _activity} = + CommonAPI.post(user, %{ + "in_reply_to_status_id" => activity.id, + "status" => "Hey @#{third_user.nickname}.", + "visibility" => "direct" + }) + + [participation] = Participation.for_user(user) + participation = Pleroma.Repo.preload(participation, :recipients) + + assert length(participation.recipients) == 2 + end + test "it creates a participation for a conversation and a user" do user = insert(:user) conversation = insert(:conversation) From 56b1c3af13c9519e13da688bdbbfdd8d69cda4ac Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 2 Aug 2019 15:05:27 +0200 Subject: [PATCH 19/94] CommonAPI: Extend api with conversation replies. --- lib/pleroma/conversation/participation.ex | 6 +++++ lib/pleroma/web/common_api/common_api.ex | 20 ++++++++++----- lib/pleroma/web/common_api/utils.ex | 27 ++++++++++++++------ test/web/common_api/common_api_test.exs | 30 +++++++++++++++++++++++ 4 files changed, 70 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index 121efb671..f1e1a6958 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -93,4 +93,10 @@ def for_user_with_last_activity_id(user, params \\ %{}) do end) |> Enum.filter(& &1.last_activity_id) end + + def get(nil), do: nil + + def get(id) do + Repo.get(__MODULE__, id) + end end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 2db58324b..86e95cd0f 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.Activity + alias Pleroma.Conversation.Participation alias Pleroma.Formatter alias Pleroma.Object alias Pleroma.ThreadMute @@ -171,21 +172,25 @@ defp normalize_and_validate_choice_indices(choices, count) do end) end - def get_visibility(%{"visibility" => visibility}, in_reply_to) + def get_visibility(_, _, %Participation{}) do + {"direct", "direct"} + end + + def get_visibility(%{"visibility" => visibility}, in_reply_to, _) when visibility in ~w{public unlisted private direct}, do: {visibility, get_replied_to_visibility(in_reply_to)} - def get_visibility(%{"visibility" => "list:" <> list_id}, in_reply_to) do + def get_visibility(%{"visibility" => "list:" <> list_id}, in_reply_to, _) do visibility = {:list, String.to_integer(list_id)} {visibility, get_replied_to_visibility(in_reply_to)} end - def get_visibility(_, in_reply_to) when not is_nil(in_reply_to) do + def get_visibility(_, in_reply_to, _) when not is_nil(in_reply_to) do visibility = get_replied_to_visibility(in_reply_to) {visibility, visibility} end - def get_visibility(_, in_reply_to), do: {"public", get_replied_to_visibility(in_reply_to)} + def get_visibility(_, in_reply_to, _), do: {"public", get_replied_to_visibility(in_reply_to)} def get_replied_to_visibility(nil), do: nil @@ -201,7 +206,9 @@ def post(user, %{"status" => status} = data) do with status <- String.trim(status), attachments <- attachments_from_ids(data), in_reply_to <- get_replied_to_activity(data["in_reply_to_status_id"]), - {visibility, in_reply_to_visibility} <- get_visibility(data, in_reply_to), + in_reply_to_conversation <- Participation.get(data["in_reply_to_conversation_id"]), + {visibility, in_reply_to_visibility} <- + get_visibility(data, in_reply_to, in_reply_to_conversation), {_, false} <- {:private_to_public, in_reply_to_visibility == "direct" && visibility != "direct"}, {content_html, mentions, tags} <- @@ -214,7 +221,8 @@ def post(user, %{"status" => status} = data) do mentioned_users <- for({_, mentioned_user} <- mentions, do: mentioned_user.ap_id), addressed_users <- get_addressed_users(mentioned_users, data["to"]), {poll, poll_emoji} <- make_poll_data(data), - {to, cc} <- get_to_and_cc(user, addressed_users, in_reply_to, visibility), + {to, cc} <- + get_to_and_cc(user, addressed_users, in_reply_to, visibility, in_reply_to_conversation), context <- make_context(in_reply_to), cw <- data["spoiler_text"] || "", sensitive <- data["sensitive"] || Enum.member?(tags, {"#nsfw", "nsfw"}), diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index d80fffa26..e70ba7d43 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do alias Calendar.Strftime alias Pleroma.Activity alias Pleroma.Config + alias Pleroma.Conversation.Participation alias Pleroma.Formatter alias Pleroma.Object alias Pleroma.Plugs.AuthenticationPlug @@ -64,9 +65,21 @@ def attachments_from_ids_descs(ids, descs_str) do end) end - @spec get_to_and_cc(User.t(), list(String.t()), Activity.t() | nil, String.t()) :: + @spec get_to_and_cc( + User.t(), + list(String.t()), + Activity.t() | nil, + String.t(), + Participation.t() | nil + ) :: {list(String.t()), list(String.t())} - def get_to_and_cc(user, mentioned_users, inReplyTo, "public") do + + def get_to_and_cc(_, _, _, _, %Participation{} = participation) do + participation = Repo.preload(participation, :recipients) + {Enum.map(participation.recipients, & &1.ap_id), []} + end + + def get_to_and_cc(user, mentioned_users, inReplyTo, "public", _) do to = [Pleroma.Constants.as_public() | mentioned_users] cc = [user.follower_address] @@ -77,7 +90,7 @@ def get_to_and_cc(user, mentioned_users, inReplyTo, "public") do end end - def get_to_and_cc(user, mentioned_users, inReplyTo, "unlisted") do + def get_to_and_cc(user, mentioned_users, inReplyTo, "unlisted", _) do to = [user.follower_address | mentioned_users] cc = [Pleroma.Constants.as_public()] @@ -88,12 +101,12 @@ def get_to_and_cc(user, mentioned_users, inReplyTo, "unlisted") do end end - def get_to_and_cc(user, mentioned_users, inReplyTo, "private") do - {to, cc} = get_to_and_cc(user, mentioned_users, inReplyTo, "direct") + def get_to_and_cc(user, mentioned_users, inReplyTo, "private", _) do + {to, cc} = get_to_and_cc(user, mentioned_users, inReplyTo, "direct", nil) {[user.follower_address | to], cc} end - def get_to_and_cc(_user, mentioned_users, inReplyTo, "direct") do + def get_to_and_cc(_user, mentioned_users, inReplyTo, "direct", _) do if inReplyTo do {Enum.uniq([inReplyTo.data["actor"] | mentioned_users]), []} else @@ -101,7 +114,7 @@ def get_to_and_cc(_user, mentioned_users, inReplyTo, "direct") do end end - def get_to_and_cc(_user, mentions, _inReplyTo, {:list, _}), do: {mentions, []} + def get_to_and_cc(_user, mentions, _inReplyTo, {:list, _}, _), do: {mentions, []} def get_addressed_users(_, to) when is_list(to) do User.get_ap_ids_by_nicknames(to) diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 16b3f121d..e2a5bf117 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -5,6 +5,7 @@ defmodule Pleroma.Web.CommonAPITest do use Pleroma.DataCase alias Pleroma.Activity + alias Pleroma.Conversation.Participation alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub @@ -12,6 +13,35 @@ defmodule Pleroma.Web.CommonAPITest do import Pleroma.Factory + test "when replying to a conversation / participation, it only mentions the recipients explicitly declared in the participation" do + har = insert(:user) + jafnhar = insert(:user) + tridi = insert(:user) + + {:ok, activity} = + CommonAPI.post(har, %{ + "status" => "@#{jafnhar.nickname} hey", + "visibility" => "direct" + }) + + assert har.ap_id in activity.recipients + assert jafnhar.ap_id in activity.recipients + + [participation] = Participation.for_user(har) + + {:ok, activity} = + CommonAPI.post(har, %{ + "status" => "I don't really like @#{tridi.nickname}", + "visibility" => "direct", + "in_reply_to_status_id" => activity.id, + "in_reply_to_conversation_id" => participation.id + }) + + assert har.ap_id in activity.recipients + assert jafnhar.ap_id in activity.recipients + refute tridi.ap_id in activity.recipients + end + test "with the safe_dm_mention option set, it does not mention people beyond the initial tags" do har = insert(:user) jafnhar = insert(:user) From eee98aaa738c1aa5f2e4203a61b67648d62965c8 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 2 Aug 2019 19:53:08 +0200 Subject: [PATCH 20/94] Pleroma API: Add endpoint to get conversation statuses. --- lib/pleroma/web/controller_helper.ex | 76 +++++++++++++++++++ .../mastodon_api/mastodon_api_controller.ex | 68 +---------------- .../web/pleroma_api/pleroma_api_controller.ex | 49 ++++++++++++ lib/pleroma/web/router.ex | 9 +++ test/web/common_api/common_api_utils_test.exs | 16 ++-- .../pleroma_api_controller_test.exs | 45 +++++++++++ 6 files changed, 189 insertions(+), 74 deletions(-) create mode 100644 lib/pleroma/web/pleroma_api/pleroma_api_controller.ex create mode 100644 test/web/pleroma_api/pleroma_api_controller_test.exs diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index 8a753bb4f..eeac9f503 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -33,4 +33,80 @@ defp param_to_integer(val, default) when is_binary(val) do end defp param_to_integer(_, default), do: default + + def add_link_headers( + conn, + method, + activities, + param \\ nil, + params \\ %{}, + func3 \\ nil, + func4 \\ nil + ) do + params = + conn.params + |> Map.drop(["since_id", "max_id", "min_id"]) + |> Map.merge(params) + + last = List.last(activities) + + func3 = func3 || (&mastodon_api_url/3) + func4 = func4 || (&mastodon_api_url/4) + + if last do + max_id = last.id + + limit = + params + |> Map.get("limit", "20") + |> String.to_integer() + + min_id = + if length(activities) <= limit do + activities + |> List.first() + |> Map.get(:id) + else + activities + |> Enum.at(limit * -1) + |> Map.get(:id) + end + + {next_url, prev_url} = + if param do + { + func4.( + Pleroma.Web.Endpoint, + method, + param, + Map.merge(params, %{max_id: max_id}) + ), + func4.( + Pleroma.Web.Endpoint, + method, + param, + Map.merge(params, %{min_id: min_id}) + ) + } + else + { + func3.( + Pleroma.Web.Endpoint, + method, + Map.merge(params, %{max_id: max_id}) + ), + func3.( + Pleroma.Web.Endpoint, + method, + Map.merge(params, %{min_id: min_id}) + ) + } + end + + conn + |> put_resp_header("link", "<#{next_url}>; rel=\"next\", <#{prev_url}>; rel=\"prev\"") + else + conn + end + end end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 174e93468..0deeab2be 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -5,7 +5,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do use Pleroma.Web, :controller - import Pleroma.Web.ControllerHelper, only: [json_response: 3] + import Pleroma.Web.ControllerHelper, + only: [json_response: 3, add_link_headers: 5, add_link_headers: 4, add_link_headers: 3] alias Ecto.Changeset alias Pleroma.Activity @@ -342,71 +343,6 @@ def custom_emojis(conn, _params) do json(conn, mastodon_emoji) end - defp add_link_headers(conn, method, activities, param \\ nil, params \\ %{}) do - params = - conn.params - |> Map.drop(["since_id", "max_id", "min_id"]) - |> Map.merge(params) - - last = List.last(activities) - - if last do - max_id = last.id - - limit = - params - |> Map.get("limit", "20") - |> String.to_integer() - - min_id = - if length(activities) <= limit do - activities - |> List.first() - |> Map.get(:id) - else - activities - |> Enum.at(limit * -1) - |> Map.get(:id) - end - - {next_url, prev_url} = - if param do - { - mastodon_api_url( - Pleroma.Web.Endpoint, - method, - param, - Map.merge(params, %{max_id: max_id}) - ), - mastodon_api_url( - Pleroma.Web.Endpoint, - method, - param, - Map.merge(params, %{min_id: min_id}) - ) - } - else - { - mastodon_api_url( - Pleroma.Web.Endpoint, - method, - Map.merge(params, %{max_id: max_id}) - ), - mastodon_api_url( - Pleroma.Web.Endpoint, - method, - Map.merge(params, %{min_id: min_id}) - ) - } - end - - conn - |> put_resp_header("link", "<#{next_url}>; rel=\"next\", <#{prev_url}>; rel=\"prev\"") - else - conn - end - end - def home_timeline(%{assigns: %{user: user}} = conn, params) do params = params diff --git a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex new file mode 100644 index 000000000..b677892ed --- /dev/null +++ b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex @@ -0,0 +1,49 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do + use Pleroma.Web, :controller + + import Pleroma.Web.ControllerHelper, only: [add_link_headers: 7] + + alias Pleroma.Conversation.Participation + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.MastodonAPI.StatusView + alias Pleroma.Repo + + def conversation_statuses( + %{assigns: %{user: user}} = conn, + %{"id" => participation_id} = params + ) do + params = + params + |> Map.put("blocking_user", user) + |> Map.put("muting_user", user) + |> Map.put("user", user) + + participation = + participation_id + |> Participation.get() + |> Repo.preload(:conversation) + + if user.id == participation.user_id do + activities = + participation.conversation.ap_id + |> ActivityPub.fetch_activities_for_context(params) + |> Enum.reverse() + + conn + |> add_link_headers( + :conversation_statuses, + activities, + participation_id, + params, + nil, + &pleroma_api_url/4 + ) + |> put_view(StatusView) + |> render("index.json", %{activities: activities, for: user, as: :activity}) + end + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 0689d69fb..40298538a 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -257,6 +257,15 @@ defmodule Pleroma.Web.Router do end end + scope "/api/v1/pleroma", Pleroma.Web.PleromaAPI do + pipe_through(:authenticated_api) + + scope [] do + pipe_through(:oauth_write) + get("/conversations/:id/statuses", PleromaAPIController, :conversation_statuses) + end + end + scope "/api/v1", Pleroma.Web.MastodonAPI do pipe_through(:authenticated_api) diff --git a/test/web/common_api/common_api_utils_test.exs b/test/web/common_api/common_api_utils_test.exs index af320f31f..7510c8def 100644 --- a/test/web/common_api/common_api_utils_test.exs +++ b/test/web/common_api/common_api_utils_test.exs @@ -239,7 +239,7 @@ test "for public posts, not a reply" do mentioned_user = insert(:user) mentions = [mentioned_user.ap_id] - {to, cc} = Utils.get_to_and_cc(user, mentions, nil, "public") + {to, cc} = Utils.get_to_and_cc(user, mentions, nil, "public", nil) assert length(to) == 2 assert length(cc) == 1 @@ -256,7 +256,7 @@ test "for public posts, a reply" do {:ok, activity} = CommonAPI.post(third_user, %{"status" => "uguu"}) mentions = [mentioned_user.ap_id] - {to, cc} = Utils.get_to_and_cc(user, mentions, activity, "public") + {to, cc} = Utils.get_to_and_cc(user, mentions, activity, "public", nil) assert length(to) == 3 assert length(cc) == 1 @@ -272,7 +272,7 @@ test "for unlisted posts, not a reply" do mentioned_user = insert(:user) mentions = [mentioned_user.ap_id] - {to, cc} = Utils.get_to_and_cc(user, mentions, nil, "unlisted") + {to, cc} = Utils.get_to_and_cc(user, mentions, nil, "unlisted", nil) assert length(to) == 2 assert length(cc) == 1 @@ -289,7 +289,7 @@ test "for unlisted posts, a reply" do {:ok, activity} = CommonAPI.post(third_user, %{"status" => "uguu"}) mentions = [mentioned_user.ap_id] - {to, cc} = Utils.get_to_and_cc(user, mentions, activity, "unlisted") + {to, cc} = Utils.get_to_and_cc(user, mentions, activity, "unlisted", nil) assert length(to) == 3 assert length(cc) == 1 @@ -305,7 +305,7 @@ test "for private posts, not a reply" do mentioned_user = insert(:user) mentions = [mentioned_user.ap_id] - {to, cc} = Utils.get_to_and_cc(user, mentions, nil, "private") + {to, cc} = Utils.get_to_and_cc(user, mentions, nil, "private", nil) assert length(to) == 2 assert length(cc) == 0 @@ -321,7 +321,7 @@ test "for private posts, a reply" do {:ok, activity} = CommonAPI.post(third_user, %{"status" => "uguu"}) mentions = [mentioned_user.ap_id] - {to, cc} = Utils.get_to_and_cc(user, mentions, activity, "private") + {to, cc} = Utils.get_to_and_cc(user, mentions, activity, "private", nil) assert length(to) == 3 assert length(cc) == 0 @@ -336,7 +336,7 @@ test "for direct posts, not a reply" do mentioned_user = insert(:user) mentions = [mentioned_user.ap_id] - {to, cc} = Utils.get_to_and_cc(user, mentions, nil, "direct") + {to, cc} = Utils.get_to_and_cc(user, mentions, nil, "direct", nil) assert length(to) == 1 assert length(cc) == 0 @@ -351,7 +351,7 @@ test "for direct posts, a reply" do {:ok, activity} = CommonAPI.post(third_user, %{"status" => "uguu"}) mentions = [mentioned_user.ap_id] - {to, cc} = Utils.get_to_and_cc(user, mentions, activity, "direct") + {to, cc} = Utils.get_to_and_cc(user, mentions, activity, "direct", nil) assert length(to) == 2 assert length(cc) == 0 diff --git a/test/web/pleroma_api/pleroma_api_controller_test.exs b/test/web/pleroma_api/pleroma_api_controller_test.exs new file mode 100644 index 000000000..43104e36e --- /dev/null +++ b/test/web/pleroma_api/pleroma_api_controller_test.exs @@ -0,0 +1,45 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.PleromaAPIControllerTest do + use Pleroma.Web.ConnCase + + alias Pleroma.Conversation.Participation + alias Pleroma.Web.CommonAPI + + import Pleroma.Factory + + test "/api/v1/pleroma/conversations/:id/statuses", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + third_user = insert(:user) + + {:ok, _activity} = + CommonAPI.post(user, %{"status" => "Hi @#{third_user.nickname}!", "visibility" => "direct"}) + + {:ok, activity} = + CommonAPI.post(user, %{"status" => "Hi @#{other_user.nickname}!", "visibility" => "direct"}) + + [participation] = Participation.for_user(other_user) + + {:ok, activity_two} = + CommonAPI.post(other_user, %{ + "status" => "Hi!", + "in_reply_to_status_id" => activity.id, + "in_reply_to_conversation_id" => participation.id + }) + + result = + conn + |> assign(:user, other_user) + |> get("/api/v1/pleroma/conversations/#{participation.id}/statuses") + |> json_response(200) + + assert length(result) == 2 + + id_one = activity.id + id_two = activity_two.id + assert [%{"id" => ^id_one}, %{"id" => ^id_two}] = result + end +end From 3af6d14da769aa5adfdd6360b43c691fd8c8eed5 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 5 Aug 2019 15:09:19 +0200 Subject: [PATCH 21/94] Pleroma Conversations API: Add a way to set recipients. --- lib/pleroma/conversation/participation.ex | 20 ++++++++++ .../mastodon_api/views/conversation_view.ex | 13 +++++- .../web/pleroma_api/pleroma_api_controller.ex | 17 ++++++++ lib/pleroma/web/router.ex | 1 + .../mastodon_api/conversation_view_test.exs | 40 +++++++++++++++++++ .../pleroma_api_controller_test.exs | 31 ++++++++++++++ 6 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 test/web/mastodon_api/conversation_view_test.exs diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index f1e1a6958..acdee5517 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -99,4 +99,24 @@ def get(nil), do: nil def get(id) do Repo.get(__MODULE__, id) end + + def set_recipients(participation, user_ids) do + Repo.transaction(fn -> + query = + from(r in RecipientShip, + where: r.participation_id == ^participation.id + ) + + Repo.delete_all(query) + + users = + from(u in User, + where: u.id in ^user_ids + ) + |> Repo.all() + + RecipientShip.create(users, participation) + :ok + end) + end end diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex index 38bdec737..5adaecdb0 100644 --- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex +++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex @@ -12,7 +12,7 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do alias Pleroma.Web.MastodonAPI.StatusView def render("participation.json", %{participation: participation, user: user}) do - participation = Repo.preload(participation, conversation: :users) + participation = Repo.preload(participation, conversation: :users, recipients: []) last_activity_id = with nil <- participation.last_activity_id do @@ -37,11 +37,20 @@ def render("participation.json", %{participation: participation, user: user}) do as: :user }) + recipients = + AccountView.render("accounts.json", %{ + users: participation.recipients, + as: :user + }) + %{ id: participation.id |> to_string(), accounts: accounts, unread: !participation.read, - last_status: last_status + last_status: last_status, + pleroma: %{ + recipients: recipients + } } end end diff --git a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex index b677892ed..759d8aef0 100644 --- a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do alias Pleroma.Conversation.Participation alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.MastodonAPI.StatusView + alias Pleroma.Web.MastodonAPI.ConversationView alias Pleroma.Repo def conversation_statuses( @@ -46,4 +47,20 @@ def conversation_statuses( |> render("index.json", %{activities: activities, for: user, as: :activity}) end end + + def update_conversation( + %{assigns: %{user: user}} = conn, + %{"id" => participation_id, "recipients" => recipients} + ) do + participation = + participation_id + |> Participation.get() + + with true <- user.id == participation.user_id, + {:ok, _} <- Participation.set_recipients(participation, recipients) do + conn + |> put_view(ConversationView) + |> render("participation.json", %{participation: participation, user: user}) + end + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 40298538a..6cdef7e2f 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -263,6 +263,7 @@ defmodule Pleroma.Web.Router do scope [] do pipe_through(:oauth_write) get("/conversations/:id/statuses", PleromaAPIController, :conversation_statuses) + patch("/conversations/:id", PleromaAPIController, :update_conversation) end end diff --git a/test/web/mastodon_api/conversation_view_test.exs b/test/web/mastodon_api/conversation_view_test.exs new file mode 100644 index 000000000..2a4b41fa4 --- /dev/null +++ b/test/web/mastodon_api/conversation_view_test.exs @@ -0,0 +1,40 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.ConversationViewTest do + use Pleroma.DataCase + + alias Pleroma.Web.CommonAPI + alias Pleroma.Conversation.Participation + alias Pleroma.Web.MastodonAPI.ConversationView + + import Pleroma.Factory + + test "represents a Mastodon Conversation entity" do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{"status" => "hey @#{other_user.nickname}", "visibility" => "direct"}) + + [participation] = Participation.for_user_with_last_activity_id(user) + + assert participation + + conversation = + ConversationView.render("participation.json", %{participation: participation, user: user}) + + assert conversation.id == participation.id |> to_string() + assert conversation.last_status.id == activity.id + + assert [account] = conversation.accounts + assert account.id == other_user.id + + assert recipients = conversation.pleroma.recipients + recipient_ids = recipients |> Enum.map(& &1.id) + + assert user.id in recipient_ids + assert other_user.id in recipient_ids + end +end diff --git a/test/web/pleroma_api/pleroma_api_controller_test.exs b/test/web/pleroma_api/pleroma_api_controller_test.exs index 43104e36e..7989defe0 100644 --- a/test/web/pleroma_api/pleroma_api_controller_test.exs +++ b/test/web/pleroma_api/pleroma_api_controller_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIControllerTest do alias Pleroma.Conversation.Participation alias Pleroma.Web.CommonAPI + alias Pleroma.Repo import Pleroma.Factory @@ -42,4 +43,34 @@ test "/api/v1/pleroma/conversations/:id/statuses", %{conn: conn} do id_two = activity_two.id assert [%{"id" => ^id_one}, %{"id" => ^id_two}] = result end + + test "PATCH /api/v1/pleroma/conversations/:id", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, _activity} = CommonAPI.post(user, %{"status" => "Hi", "visibility" => "direct"}) + + [participation] = Participation.for_user(user) + + participation = Repo.preload(participation, :recipients) + + assert [user] == participation.recipients + assert other_user not in participation.recipients + + result = + conn + |> assign(:user, user) + |> patch("/api/v1/pleroma/conversations/#{participation.id}", %{ + "recipients" => [user.id, other_user.id] + }) + |> json_response(200) + + assert result["id"] == participation.id |> to_string + + assert recipients = result["pleroma"]["recipients"] + recipient_ids = Enum.map(recipients, & &1["id"]) + + assert user.id in recipient_ids + assert other_user.id in recipient_ids + end end From b64b6fee2a78fbfbc557b89550128494ca7d2894 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 5 Aug 2019 15:33:22 +0200 Subject: [PATCH 22/94] CommonAPI: Replies to conversations also get the correct context id. --- lib/pleroma/web/common_api/common_api.ex | 2 +- lib/pleroma/web/common_api/utils.ex | 8 ++++++-- test/web/common_api/common_api_test.exs | 15 +++++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 86e95cd0f..72da46263 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -223,7 +223,7 @@ def post(user, %{"status" => status} = data) do {poll, poll_emoji} <- make_poll_data(data), {to, cc} <- get_to_and_cc(user, addressed_users, in_reply_to, visibility, in_reply_to_conversation), - context <- make_context(in_reply_to), + context <- make_context(in_reply_to, in_reply_to_conversation), cw <- data["spoiler_text"] || "", sensitive <- data["sensitive"] || Enum.member?(tags, {"#nsfw", "nsfw"}), full_payload <- String.trim(status <> cw), diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index e70ba7d43..425b6d656 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -244,8 +244,12 @@ defp maybe_add_nsfw_tag({text, mentions, tags}, %{"sensitive" => sensitive}) defp maybe_add_nsfw_tag(data, _), do: data - def make_context(%Activity{data: %{"context" => context}}), do: context - def make_context(_), do: Utils.generate_context_id() + def make_context(_, %Participation{} = participation) do + Repo.preload(participation, :conversation).conversation.ap_id + end + + def make_context(%Activity{data: %{"context" => context}}, _), do: context + def make_context(_, _), do: Utils.generate_context_id() def maybe_add_attachments(parsed, _attachments, true = _no_links), do: parsed diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index e2a5bf117..454523349 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -9,10 +9,25 @@ defmodule Pleroma.Web.CommonAPITest do alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.CommonAPI import Pleroma.Factory + test "when replying to a conversation / participation, it will set the correct context id even if no explicit reply_to is given" do + user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{"status" => ".", "visibility" => "direct"}) + + [participation] = Participation.for_user(user) + + {:ok, convo_reply} = + CommonAPI.post(user, %{"status" => ".", "in_reply_to_conversation_id" => participation.id}) + + assert Visibility.is_direct?(convo_reply) + + assert activity.data["context"] == convo_reply.data["context"] + end + test "when replying to a conversation / participation, it only mentions the recipients explicitly declared in the participation" do har = insert(:user) jafnhar = insert(:user) From ddabe7784b47939120a838b9c65381fd2262c161 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 5 Aug 2019 15:58:14 +0200 Subject: [PATCH 23/94] Pleroma Conversations: Document differences. --- docs/api/differences_in_mastoapi_responses.md | 8 +++++ docs/api/pleroma_api.md | 29 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index 1907d70c8..79ca531b8 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -59,12 +59,19 @@ Has these additional fields under the `pleroma` object: - `show_role`: boolean, nullable, true when the user wants his role (e.g admin, moderator) to be shown - `no_rich_text` - boolean, nullable, true when html tags are stripped from all statuses requested from the API +## Conversations + +Has an additional field under the `pleroma` object: + +- `recipients`: The list of the recipients of this Conversation. These will be addressed when replying to this conversation. + ## Account Search Behavior has changed: - `/api/v1/accounts/search`: Does not require authentication + ## Notifications Has these additional fields under the `pleroma` object: @@ -79,6 +86,7 @@ Additional parameters can be added to the JSON body/Form data: - `content_type`: string, contain the MIME type of the status, it is transformed into HTML by the backend. You can get the list of the supported MIME types with the nodeinfo endpoint. - `to`: A list of nicknames (like `lain@soykaf.club` or `lain` on the local server) that will be used to determine who is going to be addressed by this post. Using this will disable the implicit addressing by mentioned names in the `status` body, only the people in the `to` list will be addressed. The normal rules for for post visibility are not affected by this and will still apply. - `visibility`: string, besides standard MastoAPI values (`direct`, `private`, `unlisted` or `public`) it can be used to address a List by setting it to `list:LIST_ID`. +- `in_reply_to_conversation_id`: Will reply to a given conversation, addressing only the people who are part of the recipient set of that conversation. Sets the visibility to `direct`. ## PATCH `/api/v1/update_credentials` diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md index 5698e88ac..4323e59f0 100644 --- a/docs/api/pleroma_api.md +++ b/docs/api/pleroma_api.md @@ -319,3 +319,32 @@ See [Admin-API](Admin-API.md) "healthy": true # Instance state } ``` + +# Pleroma Conversations + +Pleroma Conversations have the same general structure that Mastodon Conversations have. The behavior differs in the following ways when using these endpoints: + +1. Pleroma Conversations never add or remove recipients, unless explicitly changed by the user. +2. Pleroma Conversations statuses can be requested by Conversation id. +3. Pleroma Conversations can be replied to. + +Conversations have the additional field "recipients" under the "pleroma" key. This holds a list of all the accounts that will receive a message in this conversation. + +The status posting endpoint takes an additional parameter, `in_reply_to_conversation_id`, which, when set, will set the visiblity to direct and address only the people who are the recipients of that Conversation. + + +## `GET /api/v1/pleroma/conversations/:id/statuses` +### Timeline for a given conversation +* Method `GET` +* Authentication: required +* Params: Like other timelines +* Response: JSON, statuses (200 - healthy, 503 unhealthy). + + +## `PATCH /api/v1/pleroma/conversations/:id` +### Update a conversation. Used to change the set of recipients. +* Method `PATCH` +* Authentication: required +* Params: + * `recipients`: A list of ids of users that should receive posts to this conversation. +* Response: JSON, statuses (200 - healthy, 503 unhealthy) From d6fe220e32d581220cc33f4f44d6517a401eabbf Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 5 Aug 2019 16:11:23 +0200 Subject: [PATCH 24/94] Linting. --- lib/pleroma/conversation/participation_recipient_ship.ex | 2 +- lib/pleroma/web/pleroma_api/pleroma_api_controller.ex | 6 +++--- test/web/mastodon_api/conversation_view_test.exs | 2 +- test/web/pleroma_api/pleroma_api_controller_test.exs | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/conversation/participation_recipient_ship.ex b/lib/pleroma/conversation/participation_recipient_ship.ex index 27c0c89cd..932cbd04c 100644 --- a/lib/pleroma/conversation/participation_recipient_ship.ex +++ b/lib/pleroma/conversation/participation_recipient_ship.ex @@ -6,8 +6,8 @@ defmodule Pleroma.Conversation.Participation.RecipientShip do use Ecto.Schema alias Pleroma.Conversation.Participation - alias Pleroma.User alias Pleroma.Repo + alias Pleroma.User import Ecto.Changeset diff --git a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex index 759d8aef0..018564452 100644 --- a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex @@ -8,10 +8,10 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do import Pleroma.Web.ControllerHelper, only: [add_link_headers: 7] alias Pleroma.Conversation.Participation - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.MastodonAPI.StatusView - alias Pleroma.Web.MastodonAPI.ConversationView alias Pleroma.Repo + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.MastodonAPI.ConversationView + alias Pleroma.Web.MastodonAPI.StatusView def conversation_statuses( %{assigns: %{user: user}} = conn, diff --git a/test/web/mastodon_api/conversation_view_test.exs b/test/web/mastodon_api/conversation_view_test.exs index 2a4b41fa4..e32cde5a8 100644 --- a/test/web/mastodon_api/conversation_view_test.exs +++ b/test/web/mastodon_api/conversation_view_test.exs @@ -5,8 +5,8 @@ defmodule Pleroma.Web.MastodonAPI.ConversationViewTest do use Pleroma.DataCase - alias Pleroma.Web.CommonAPI alias Pleroma.Conversation.Participation + alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.ConversationView import Pleroma.Factory diff --git a/test/web/pleroma_api/pleroma_api_controller_test.exs b/test/web/pleroma_api/pleroma_api_controller_test.exs index 7989defe0..7c75fb229 100644 --- a/test/web/pleroma_api/pleroma_api_controller_test.exs +++ b/test/web/pleroma_api/pleroma_api_controller_test.exs @@ -6,8 +6,8 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIControllerTest do use Pleroma.Web.ConnCase alias Pleroma.Conversation.Participation - alias Pleroma.Web.CommonAPI alias Pleroma.Repo + alias Pleroma.Web.CommonAPI import Pleroma.Factory From a49c92f6ae2dc68a80345cff4793820a75835eb1 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 6 Aug 2019 14:51:17 +0200 Subject: [PATCH 25/94] Participation: Setting recipients will always add the owner. --- docs/api/pleroma_api.md | 2 +- lib/pleroma/conversation/participation.ex | 6 ++++++ test/conversation/participation_test.exs | 19 +++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md index 4323e59f0..590f2a3fb 100644 --- a/docs/api/pleroma_api.md +++ b/docs/api/pleroma_api.md @@ -346,5 +346,5 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa * Method `PATCH` * Authentication: required * Params: - * `recipients`: A list of ids of users that should receive posts to this conversation. + * `recipients`: A list of ids of users that should receive posts to this conversation. This will replace the current list of recipients, so submit the full list. The owner of owner of the conversation will always be part of the set of recipients, though. * Response: JSON, statuses (200 - healthy, 503 unhealthy) diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index acdee5517..d17b6f7c5 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -101,6 +101,10 @@ def get(id) do end def set_recipients(participation, user_ids) do + user_ids = + [participation.user_id | user_ids] + |> Enum.uniq() + Repo.transaction(fn -> query = from(r in RecipientShip, @@ -118,5 +122,7 @@ def set_recipients(participation, user_ids) do RecipientShip.create(users, participation) :ok end) + + {:ok, Repo.preload(participation, :recipients, force: true)} end end diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs index 4a3c397bd..7958e8e89 100644 --- a/test/conversation/participation_test.exs +++ b/test/conversation/participation_test.exs @@ -132,4 +132,23 @@ test "Doesn't die when the conversation gets empty" do [] = Participation.for_user_with_last_activity_id(user) end + + test "it sets recipients, always keeping the owner of the participation even when not explicitly set" do + user = insert(:user) + other_user = insert(:user) + + {:ok, _activity} = CommonAPI.post(user, %{"status" => ".", "visibility" => "direct"}) + [participation] = Participation.for_user_with_last_activity_id(user) + + participation = Repo.preload(participation, :recipients) + + assert participation.recipients |> length() == 1 + assert user in participation.recipients + + {:ok, participation} = Participation.set_recipients(participation, [other_user.id]) + + assert participation.recipients |> length() == 2 + assert user in participation.recipients + assert other_user in participation.recipients + end end From e4a01d253ef7ab09d028198e5e39b9aba357486c Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 6 Aug 2019 15:06:19 +0200 Subject: [PATCH 26/94] Conversation: Rename function to better express what it does. --- lib/pleroma/conversation.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index fb0dfedca..be5821ad7 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -40,7 +40,7 @@ def get_for_ap_id(ap_id) do Repo.get_by(__MODULE__, ap_id: ap_id) end - def maybe_set_recipients(participation, activity) do + def maybe_create_recipientships(participation, activity) do participation = Repo.preload(participation, :recipients) if participation.recipients |> Enum.empty?() do @@ -70,7 +70,7 @@ def create_or_bump_for(activity, opts \\ []) do {:ok, participation} = Participation.create_for_user_and_conversation(user, conversation, opts) - maybe_set_recipients(participation, activity) + maybe_create_recipientships(participation, activity) participation end) From dfae61c25c7ee2bb8add38b2cbaa8391f03c9550 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Fri, 9 Aug 2019 23:05:28 +0300 Subject: [PATCH 27/94] Fix deactivated user deletion --- lib/mix/tasks/pleroma/user.ex | 2 +- lib/pleroma/user.ex | 34 +++++++++---------- lib/pleroma/web/activity_pub/activity_pub.ex | 10 +++--- .../web/activity_pub/transmogrifier.ex | 2 +- .../web/admin_api/admin_api_controller.ex | 4 +-- .../web/ostatus/handlers/delete_handler.ex | 2 +- test/user_test.exs | 8 +++++ 7 files changed, 36 insertions(+), 26 deletions(-) diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index a3f8bc945..f33d01429 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -176,7 +176,7 @@ def run(["rm", nickname]) do start_pleroma() with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do - User.perform(:delete, user) + User.perform(:delete, user, nil) shell_info("User #{nickname} deleted.") else _ -> diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 7d18f099e..14057a0e4 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1029,13 +1029,26 @@ def update_notification_settings(%User{} = user, settings \\ %{}) do |> update_and_set_cache() end + @spec perform(atom(), User.t()) :: {:ok, User.t()} + def perform(:fetch_initial_posts, %User{} = user) do + pages = Pleroma.Config.get!([:fetch_initial_posts, :pages]) + + Enum.each( + # Insert all the posts in reverse order, so they're in the right order on the timeline + Enum.reverse(Utils.fetch_ordered_collection(user.info.source_data["outbox"], pages)), + &Pleroma.Web.Federator.incoming_ap_doc/1 + ) + + {:ok, user} + end + @spec delete(User.t()) :: :ok - def delete(%User{} = user), - do: PleromaJobQueue.enqueue(:background, __MODULE__, [:delete, user]) + def delete(%User{} = user, actor \\ nil), + do: PleromaJobQueue.enqueue(:background, __MODULE__, [:delete, user, actor]) @spec perform(atom(), User.t()) :: {:ok, User.t()} - def perform(:delete, %User{} = user) do - {:ok, _user} = ActivityPub.delete(user) + def perform(:delete, %User{} = user, actor) do + {:ok, _user} = ActivityPub.delete(user, actor: actor) # Remove all relationships {:ok, followers} = User.get_followers(user) @@ -1057,19 +1070,6 @@ def perform(:delete, %User{} = user) do Repo.delete(user) end - @spec perform(atom(), User.t()) :: {:ok, User.t()} - def perform(:fetch_initial_posts, %User{} = user) do - pages = Pleroma.Config.get!([:fetch_initial_posts, :pages]) - - Enum.each( - # Insert all the posts in reverse order, so they're in the right order on the timeline - Enum.reverse(Utils.fetch_ordered_collection(user.info.source_data["outbox"], pages)), - &Pleroma.Web.Federator.incoming_ap_doc/1 - ) - - {:ok, user} - end - def perform(:deactivate_async, user, status), do: deactivate(user, status) @spec perform(atom(), User.t(), list()) :: list() | {:error, any()} diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 1a279a7df..8f669acb9 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -403,11 +403,13 @@ def unfollow(follower, followed, activity_id \\ nil, local \\ true) do end end - def delete(%User{ap_id: ap_id, follower_address: follower_address} = user) do + def delete(data, opts \\ %{actor: nil, local: true}) + + def delete(%User{ap_id: ap_id, follower_address: follower_address} = user, opts) do with data <- %{ "to" => [follower_address], "type" => "Delete", - "actor" => ap_id, + "actor" => opts[:actor] || ap_id, "object" => %{"type" => "Person", "id" => ap_id} }, {:ok, activity} <- insert(data, true, true), @@ -416,7 +418,7 @@ def delete(%User{ap_id: ap_id, follower_address: follower_address} = user) do end end - def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, local \\ true) do + def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, opts) do user = User.get_cached_by_ap_id(actor) to = (object.data["to"] || []) ++ (object.data["cc"] || []) @@ -428,7 +430,7 @@ def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, local \\ tru "to" => to, "deleted_activity_id" => activity && activity.id }, - {:ok, activity} <- insert(data, local, false), + {:ok, activity} <- insert(data, opts[:local], false), stream_out_participations(object, user), _ <- decrease_replies_count_if_reply(object), # Changing note count prior to enqueuing federation task in order to avoid diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 5403b71d8..b34ef73c0 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -649,7 +649,7 @@ def handle_incoming( {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id), :ok <- Containment.contain_origin(actor.ap_id, object.data), - {:ok, activity} <- ActivityPub.delete(object, false) do + {:ok, activity} <- ActivityPub.delete(object, local: false) do {:ok, activity} else nil -> diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 2d3d0adc4..63c9a7d7f 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -25,9 +25,9 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do action_fallback(:errors) - def user_delete(conn, %{"nickname" => nickname}) do + def user_delete(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do User.get_cached_by_nickname(nickname) - |> User.delete() + |> User.delete(admin.ap_id) conn |> json(nickname) diff --git a/lib/pleroma/web/ostatus/handlers/delete_handler.ex b/lib/pleroma/web/ostatus/handlers/delete_handler.ex index b2f9f3946..ac2dc115c 100644 --- a/lib/pleroma/web/ostatus/handlers/delete_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/delete_handler.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Web.OStatus.DeleteHandler do def handle_delete(entry, _doc \\ nil) do with id <- XML.string_from_xpath("//id", entry), %Object{} = object <- Object.normalize(id), - {:ok, delete} <- ActivityPub.delete(object, false) do + {:ok, delete} <- ActivityPub.delete(object, local: false) do delete end end diff --git a/test/user_test.exs b/test/user_test.exs index 8440d456d..e2da8d84b 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -998,6 +998,14 @@ test ".delete_user_activities deletes all create activities", %{user: user} do refute Activity.get_by_id(activity.id) end + test "it deletes deactivated user" do + admin = insert(:user, %{info: %{is_admin: true}}) + {:ok, user} = insert(:user, info: %{deactivated: true}) |> User.set_cache() + + assert {:ok, _} = User.delete(user, admin.ap_id) + refute User.get_by_id(user.id) + end + test "it deletes a user, all follow relationships and all activities", %{user: user} do follower = insert(:user) {:ok, follower} = User.follow(follower, user) From 23c46f7e72701b773d87b825526450e5f4ec6322 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 12 Aug 2019 12:51:08 +0200 Subject: [PATCH 28/94] Conversations: Use 'recipients' for accounts in conversation view. According to gargron, this is the intended usage. --- .../web/mastodon_api/views/conversation_view.ex | 15 +++------------ test/web/mastodon_api/conversation_view_test.exs | 6 ------ .../pleroma_api/pleroma_api_controller_test.exs | 8 ++++---- 3 files changed, 7 insertions(+), 22 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex index 5adaecdb0..4a81f0248 100644 --- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex +++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex @@ -12,7 +12,7 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do alias Pleroma.Web.MastodonAPI.StatusView def render("participation.json", %{participation: participation, user: user}) do - participation = Repo.preload(participation, conversation: :users, recipients: []) + participation = Repo.preload(participation, conversation: [], recipients: []) last_activity_id = with nil <- participation.last_activity_id do @@ -28,7 +28,7 @@ def render("participation.json", %{participation: participation, user: user}) do # Conversations return all users except the current user. users = - participation.conversation.users + participation.recipients |> Enum.reject(&(&1.id == user.id)) accounts = @@ -37,20 +37,11 @@ def render("participation.json", %{participation: participation, user: user}) do as: :user }) - recipients = - AccountView.render("accounts.json", %{ - users: participation.recipients, - as: :user - }) - %{ id: participation.id |> to_string(), accounts: accounts, unread: !participation.read, - last_status: last_status, - pleroma: %{ - recipients: recipients - } + last_status: last_status } end end diff --git a/test/web/mastodon_api/conversation_view_test.exs b/test/web/mastodon_api/conversation_view_test.exs index e32cde5a8..27f668d9f 100644 --- a/test/web/mastodon_api/conversation_view_test.exs +++ b/test/web/mastodon_api/conversation_view_test.exs @@ -30,11 +30,5 @@ test "represents a Mastodon Conversation entity" do assert [account] = conversation.accounts assert account.id == other_user.id - - assert recipients = conversation.pleroma.recipients - recipient_ids = recipients |> Enum.map(& &1.id) - - assert user.id in recipient_ids - assert other_user.id in recipient_ids end end diff --git a/test/web/pleroma_api/pleroma_api_controller_test.exs b/test/web/pleroma_api/pleroma_api_controller_test.exs index 7c75fb229..56bc1572c 100644 --- a/test/web/pleroma_api/pleroma_api_controller_test.exs +++ b/test/web/pleroma_api/pleroma_api_controller_test.exs @@ -67,10 +67,10 @@ test "PATCH /api/v1/pleroma/conversations/:id", %{conn: conn} do assert result["id"] == participation.id |> to_string - assert recipients = result["pleroma"]["recipients"] - recipient_ids = Enum.map(recipients, & &1["id"]) + [participation] = Participation.for_user(user) + participation = Repo.preload(participation, :recipients) - assert user.id in recipient_ids - assert other_user.id in recipient_ids + assert user in participation.recipients + assert other_user in participation.recipients end end From 60231ec7bd0af993dc19f69a57b261b3b4167636 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 12 Aug 2019 13:58:04 +0200 Subject: [PATCH 29/94] Conversation: Add endpoint to get a conversation by id. --- docs/api/pleroma_api.md | 6 ++++++ .../web/pleroma_api/pleroma_api_controller.ex | 9 +++++++++ lib/pleroma/web/router.ex | 1 + .../pleroma_api_controller_test.exs | 18 ++++++++++++++++++ 4 files changed, 34 insertions(+) diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md index 590f2a3fb..b134b31a8 100644 --- a/docs/api/pleroma_api.md +++ b/docs/api/pleroma_api.md @@ -340,6 +340,12 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa * Params: Like other timelines * Response: JSON, statuses (200 - healthy, 503 unhealthy). +## `GET /api/v1/pleroma/conversations/:id` +### The conversation with the given ID. +* Method `GET` +* Authentication: required +* Params: None +* Response: JSON, statuses (200 - healthy, 503 unhealthy). ## `PATCH /api/v1/pleroma/conversations/:id` ### Update a conversation. Used to change the set of recipients. diff --git a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex index 018564452..3175a99b1 100644 --- a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex @@ -13,6 +13,15 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do alias Pleroma.Web.MastodonAPI.ConversationView alias Pleroma.Web.MastodonAPI.StatusView + def conversation(%{assigns: %{user: user}} = conn, %{"id" => participation_id}) do + with %Participation{} = participation <- Participation.get(participation_id), + true <- user.id == participation.user_id do + conn + |> put_view(ConversationView) + |> render("participation.json", %{participation: participation, user: user}) + end + end + def conversation_statuses( %{assigns: %{user: user}} = conn, %{"id" => participation_id} = params diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index c835f06b4..f0b6a02e9 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -265,6 +265,7 @@ defmodule Pleroma.Web.Router do scope [] do pipe_through(:oauth_write) get("/conversations/:id/statuses", PleromaAPIController, :conversation_statuses) + get("/conversations/:id", PleromaAPIController, :conversation) patch("/conversations/:id", PleromaAPIController, :update_conversation) end end diff --git a/test/web/pleroma_api/pleroma_api_controller_test.exs b/test/web/pleroma_api/pleroma_api_controller_test.exs index 56bc1572c..ed6b79727 100644 --- a/test/web/pleroma_api/pleroma_api_controller_test.exs +++ b/test/web/pleroma_api/pleroma_api_controller_test.exs @@ -11,6 +11,24 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIControllerTest do import Pleroma.Factory + test "/api/v1/pleroma/conversations/:id", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, _activity} = + CommonAPI.post(user, %{"status" => "Hi @#{other_user.nickname}!", "visibility" => "direct"}) + + [participation] = Participation.for_user(other_user) + + result = + conn + |> assign(:user, other_user) + |> get("/api/v1/pleroma/conversations/#{participation.id}") + |> json_response(200) + + assert result["id"] == participation.id |> to_string() + end + test "/api/v1/pleroma/conversations/:id/statuses", %{conn: conn} do user = insert(:user) other_user = insert(:user) From 511ccea5aa36b4b0098e49b409b335b0ce8f042e Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 12 Aug 2019 14:23:06 +0200 Subject: [PATCH 30/94] ConversationView: Align parameter names with other views. --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 4 ++-- lib/pleroma/web/mastodon_api/views/conversation_view.ex | 2 +- lib/pleroma/web/pleroma_api/pleroma_api_controller.ex | 4 ++-- lib/pleroma/web/streamer.ex | 2 +- test/web/mastodon_api/conversation_view_test.exs | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 0deeab2be..eb2351eb7 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1743,7 +1743,7 @@ def conversations(%{assigns: %{user: user}} = conn, params) do conversations = Enum.map(participations, fn participation -> - ConversationView.render("participation.json", %{participation: participation, user: user}) + ConversationView.render("participation.json", %{participation: participation, for: user}) end) conn @@ -1756,7 +1756,7 @@ def conversation_read(%{assigns: %{user: user}} = conn, %{"id" => participation_ Repo.get_by(Participation, id: participation_id, user_id: user.id), {:ok, participation} <- Participation.mark_as_read(participation) do participation_view = - ConversationView.render("participation.json", %{participation: participation, user: user}) + ConversationView.render("participation.json", %{participation: participation, for: user}) conn |> json(participation_view) diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex index 4a81f0248..40acc07b3 100644 --- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex +++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.StatusView - def render("participation.json", %{participation: participation, user: user}) do + def render("participation.json", %{participation: participation, for: user}) do participation = Repo.preload(participation, conversation: [], recipients: []) last_activity_id = diff --git a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex index 3175a99b1..b5c3d2728 100644 --- a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex @@ -18,7 +18,7 @@ def conversation(%{assigns: %{user: user}} = conn, %{"id" => participation_id}) true <- user.id == participation.user_id do conn |> put_view(ConversationView) - |> render("participation.json", %{participation: participation, user: user}) + |> render("participation.json", %{participation: participation, for: user}) end end @@ -69,7 +69,7 @@ def update_conversation( {:ok, _} <- Participation.set_recipients(participation, recipients) do conn |> put_view(ConversationView) - |> render("participation.json", %{participation: participation, user: user}) + |> render("participation.json", %{participation: participation, for: user}) end end end diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 9ee331030..a0bb10895 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -209,7 +209,7 @@ def represent_conversation(%Participation{} = participation) do payload: Pleroma.Web.MastodonAPI.ConversationView.render("participation.json", %{ participation: participation, - user: participation.user + for: participation.user }) |> Jason.encode!() } diff --git a/test/web/mastodon_api/conversation_view_test.exs b/test/web/mastodon_api/conversation_view_test.exs index 27f668d9f..a2a880705 100644 --- a/test/web/mastodon_api/conversation_view_test.exs +++ b/test/web/mastodon_api/conversation_view_test.exs @@ -23,7 +23,7 @@ test "represents a Mastodon Conversation entity" do assert participation conversation = - ConversationView.render("participation.json", %{participation: participation, user: user}) + ConversationView.render("participation.json", %{participation: participation, for: user}) assert conversation.id == participation.id |> to_string() assert conversation.last_status.id == activity.id From 2674db14a2ee29e98265c0c0b1db412835b6bbed Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 12 Aug 2019 14:26:18 +0200 Subject: [PATCH 31/94] Modify Changelog. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 069974e44..f8c90a73b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Report email not being sent to admins when the reporter is a remote user ### Added +- Conversations: Add Pleroma-specific conversation endpoints and status posting extensions. Run the `bump_all_conversations` task again to create the necessary data. - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) - MRF: Support for excluding specific domains from Transparency. - MRF: Support for filtering posts based on who they mention (`Pleroma.Web.ActivityPub.MRF.MentionPolicy`) From e0ac5c7a66664c897e1b3af9a55e0b73f32fa034 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 24 Jul 2019 19:26:35 +0700 Subject: [PATCH 32/94] Add custom profile fields --- config/config.exs | 1 + docs/config.md | 1 + lib/pleroma/user/info.ex | 31 +++++++++++++++ .../mastodon_api/mastodon_api_controller.ex | 11 ++++++ .../web/mastodon_api/views/account_view.ex | 9 ++--- .../update_credentials_test.exs | 39 +++++++++++++++++++ 6 files changed, 87 insertions(+), 5 deletions(-) diff --git a/config/config.exs b/config/config.exs index 758661120..109cf6516 100644 --- a/config/config.exs +++ b/config/config.exs @@ -255,6 +255,7 @@ dynamic_configuration: false, user_bio_length: 5000, user_name_length: 100, + max_account_fields: 4, external_user_synchronization: true config :pleroma, :markup, diff --git a/docs/config.md b/docs/config.md index 20311db54..ca5da7db1 100644 --- a/docs/config.md +++ b/docs/config.md @@ -132,6 +132,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `skip_thread_containment`: Skip filter out broken threads. The default is `false`. * `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. * `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. +* `max_account_fields`: The maximum number of custom fields in the user profile (default: `4`) * `external_user_synchronization`: Enabling following/followers counters synchronization for external users. diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 22eb9a182..fa57052fb 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -49,6 +49,7 @@ defmodule Pleroma.User.Info do field(:mascot, :map, default: nil) field(:emoji, {:array, :map}, default: []) field(:pleroma_settings_store, :map, default: %{}) + field(:fields, {:array, :map}, default: []) field(:notification_settings, :map, default: %{ @@ -286,10 +287,32 @@ def profile_update(info, params) do :background, :show_role, :skip_thread_containment, + :fields, :pleroma_settings_store ]) + |> validate_fields() end + def validate_fields(changeset) do + limit = Pleroma.Config.get([:instance, :max_account_fields], 0) + + changeset + |> validate_length(:fields, max: limit) + |> validate_change(:fields, fn :fields, fields -> + if Enum.all?(fields, &valid_field?/1) do + [] + else + [fields: "invalid"] + end + end) + end + + defp valid_field?(%{"name" => name, "value" => value}) do + is_binary(name) && is_binary(value) + end + + defp valid_field?(_), do: false + @spec confirmation_changeset(Info.t(), keyword()) :: Changeset.t() def confirmation_changeset(info, opts) do need_confirmation? = Keyword.get(opts, :need_confirmation) @@ -384,6 +407,14 @@ def remove_reblog_mute(info, ap_id) do cast(info, params, [:muted_reblogs]) end + def fields(%{source_data: %{"attachment" => attachment}}) do + attachment + |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) + |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) + end + + def fields(%{fields: fields}), do: fields + def follow_information_update(info, params) do info |> cast(params, [ diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 7ce2b5b06..e79a02caa 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -156,6 +156,17 @@ def update_credentials(%{assigns: %{user: user}} = conn, params) do end) end) |> add_if_present(params, "default_scope", :default_scope) + |> add_if_present(params, "fields", :fields, fn fields -> + fields = + Enum.map(fields, fn field -> + %{ + "name" => Formatter.html_escape(field["name"], "text/plain"), + "value" => Formatter.html_escape(field["value"], "text/plain") + } + end) + + {:ok, fields} + end) |> add_if_present(params, "pleroma_settings_store", :pleroma_settings_store, fn value -> {:ok, Map.merge(user.info.pleroma_settings_store, value)} end) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 72c092f25..d2f3986ff 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -93,10 +93,8 @@ defp do_render("account.json", %{user: user} = opts) do } end) - fields = - (user.info.source_data["attachment"] || []) - |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) - |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) + fields = User.Info.fields(user.info) + fields_html = Enum.map(fields, fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end) bio = HTML.filter_tags(user.bio, User.html_filter_policy(opts[:for])) @@ -119,11 +117,12 @@ defp do_render("account.json", %{user: user} = opts) do header: header, header_static: header, emojis: emojis, - fields: fields, + fields: fields_html, bot: bot, source: %{ note: HTML.strip_tags((user.bio || "") |> String.replace("
", "\n")), sensitive: false, + fields: fields, pleroma: %{} }, diff --git a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs index 71d0c8af8..a3eadde16 100644 --- a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs @@ -300,5 +300,44 @@ test "updates profile emojos", %{conn: conn} do assert user["display_name"] == name assert [%{"shortcode" => "blank"}, %{"shortcode" => "firefox"}] = user["emojis"] end + + test "update fields", %{conn: conn} do + user = insert(:user) + + fields = [ + %{"name" => "foo", "value" => "bar"}, + %{"name" => "link", "value" => "cofe.io"} + ] + + account = + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_credentials", %{"fields" => fields}) + |> json_response(200) + + assert account["fields"] == [ + %{"name" => "<b>foo<b>", "value" => "<i>bar</i>"}, + %{"name" => "link", "value" => "cofe.io"} + ] + + assert account["source"]["fields"] == [ + %{"name" => "<b>foo<b>", "value" => "<i>bar</i>"}, + %{"name" => "link", "value" => "cofe.io"} + ] + + Pleroma.Config.put([:instance, :max_account_fields], 1) + + fields = [ + %{"name" => "foo", "value" => "bar"}, + %{"name" => "link", "value" => "cofe.io"} + ] + + conn = + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_credentials", %{"fields" => fields}) + + assert %{"error" => "Invalid request"} == json_response(conn, 403) + end end end From d6094b405d1a744eaa1c17752b3088dfee16c8d2 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 24 Jul 2019 19:48:15 +0700 Subject: [PATCH 33/94] Fix tests --- test/web/mastodon_api/account_view_test.exs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs index a26f514a5..1d8b28339 100644 --- a/test/web/mastodon_api/account_view_test.exs +++ b/test/web/mastodon_api/account_view_test.exs @@ -67,7 +67,8 @@ test "Represent a user account" do source: %{ note: "valid html", sensitive: false, - pleroma: %{} + pleroma: %{}, + fields: [] }, pleroma: %{ background_image: "https://example.com/images/asuka_hospital.png", @@ -134,7 +135,8 @@ test "Represent a Service(bot) account" do source: %{ note: user.bio, sensitive: false, - pleroma: %{} + pleroma: %{}, + fields: [] }, pleroma: %{ background_image: nil, @@ -304,7 +306,8 @@ test "represent an embedded relationship" do source: %{ note: user.bio, sensitive: false, - pleroma: %{} + pleroma: %{}, + fields: [] }, pleroma: %{ background_image: nil, From 069951722f9b674a759e0b8683aff9c03019467b Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 25 Jul 2019 14:34:52 +0700 Subject: [PATCH 34/94] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 358287096..a54b04658 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Add `pleroma.deactivated` to the Account entity - Mastodon API: added `/auth/password` endpoint for password reset with rate limit. - Mastodon API: /api/v1/accounts/:id/statuses now supports nicknames or user id +- Mastodon API: Add support for the user profile custom fields - Admin API: Return users' tags when querying reports - Admin API: Return avatar and display name when querying users - Admin API: Allow querying user by ID From a22f540fc42dd941631e94fe931d1f655b2904a1 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 25 Jul 2019 19:33:18 +0700 Subject: [PATCH 35/94] Add custom fields to TwitterAPI.UserView --- lib/pleroma/user/info.ex | 2 ++ lib/pleroma/web/twitter_api/views/user_view.ex | 7 +------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index fa57052fb..98b894223 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -407,6 +407,8 @@ def remove_reblog_mute(info, ap_id) do cast(info, params, [:muted_reblogs]) end + # ``fields`` is an array of mastodon profile field, containing ``{"name": "…", "value": "…"}``. + # For example: [{"name": "Pronoun", "value": "she/her"}, …] def fields(%{source_data: %{"attachment" => attachment}}) do attachment |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex index 8d8892068..3681773be 100644 --- a/lib/pleroma/web/twitter_api/views/user_view.ex +++ b/lib/pleroma/web/twitter_api/views/user_view.ex @@ -74,12 +74,7 @@ defp do_render("user.json", %{user: user = %User{}} = assigns) do |> HTML.filter_tags(User.html_filter_policy(for_user)) |> Formatter.emojify(emoji) - # ``fields`` is an array of mastodon profile field, containing ``{"name": "…", "value": "…"}``. - # For example: [{"name": "Pronoun", "value": "she/her"}, …] - fields = - (user.info.source_data["attachment"] || []) - |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) - |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) + fields = User.Info.fields(user.info) data = %{ From 88598c9bafcdcf89b0f1fb00d0785c77b583cd65 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 25 Jul 2019 19:35:34 +0700 Subject: [PATCH 36/94] Add profile custom fields to ActivityPub.UserView --- lib/pleroma/web/activity_pub/views/user_view.ex | 6 ++++++ test/web/activity_pub/views/user_view_test.exs | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 06c9e1c71..7b4bc998b 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -80,6 +80,11 @@ def render("user.json", %{user: user}) do |> Transmogrifier.add_emoji_tags() |> Map.get("tag", []) + fields = + user.info + |> User.Info.fields() + |> Enum.map(&Map.put(&1, "type", "PropertyValue")) + %{ "id" => user.ap_id, "type" => "Person", @@ -98,6 +103,7 @@ def render("user.json", %{user: user}) do "publicKeyPem" => public_key }, "endpoints" => endpoints, + "attachment" => fields, "tag" => (user.info.source_data["tag"] || []) ++ user_tags } |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user)) diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index 86254117f..48a522c6c 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -22,6 +22,22 @@ test "Renders a user, including the public key" do assert String.contains?(result["publicKey"]["publicKeyPem"], "BEGIN PUBLIC KEY") end + test "Renders profile fields" do + fields = [ + %{"name" => "foo", "value" => "bar"}, + %{"name" => "website", "value" => "cofe.my"} + ] + + user = insert(:user, info: %{fields: fields}) + + assert %{ + "attachment" => [ + %{"name" => "foo", "type" => "PropertyValue", "value" => "bar"}, + %{"name" => "website", "type" => "PropertyValue", "value" => "cofe.my"} + ] + } = UserView.render("user.json", %{user: user}) + end + test "Does not add an avatar image if the user hasn't set one" do user = insert(:user) {:ok, user} = User.ensure_keys_present(user) From 8ab87ce40b0b8a5be7cd3576eddbacc279d8d3a1 Mon Sep 17 00:00:00 2001 From: Egor Date: Fri, 26 Jul 2019 12:08:46 +0000 Subject: [PATCH 37/94] Apply suggestion to CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a54b04658..483c88073 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,7 +64,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Add `pleroma.deactivated` to the Account entity - Mastodon API: added `/auth/password` endpoint for password reset with rate limit. - Mastodon API: /api/v1/accounts/:id/statuses now supports nicknames or user id -- Mastodon API: Add support for the user profile custom fields +- Mastodon API: Improve support for the user profile custom fields - Admin API: Return users' tags when querying reports - Admin API: Return avatar and display name when querying users - Admin API: Allow querying user by ID From 5178f960c3f5a35e2071bd5463b537cadc9a53af Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 29 Jul 2019 19:01:15 +0700 Subject: [PATCH 38/94] Support user attachment update in Transmogrifier --- .../web/activity_pub/transmogrifier.ex | 2 + test/fixtures/mastodon-update.json | 36 +++++++++--- .../tesla_mock/admin@mastdon.example.org.json | 55 ++++++++++++++++++- test/web/activity_pub/transmogrifier_test.exs | 36 ++++++++++++ 4 files changed, 120 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 0fcc81bf3..225c34875 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -598,11 +598,13 @@ def handle_incoming( banner = new_user_data[:info][:banner] locked = new_user_data[:info][:locked] || false + attachment = get_in(new_user_data, [:info, "source_data", "attachment"]) update_data = new_user_data |> Map.take([:name, :bio, :avatar]) |> Map.put(:info, %{banner: banner, locked: locked}) + |> Map.put(:info, %{"banner" => banner, "locked" => locked, "source_data" => source_data}) actor |> User.upgrade_changeset(update_data) diff --git a/test/fixtures/mastodon-update.json b/test/fixtures/mastodon-update.json index f6713fea5..dbf8b6dff 100644 --- a/test/fixtures/mastodon-update.json +++ b/test/fixtures/mastodon-update.json @@ -1,10 +1,10 @@ -{ - "type": "Update", - "object": { - "url": "http://mastodon.example.org/@gargron", - "type": "Person", - "summary": "

Some bio

", - "publicKey": { +{ + "type": "Update", + "object": { + "url": "http://mastodon.example.org/@gargron", + "type": "Person", + "summary": "

Some bio

", + "publicKey": { "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0gs3VnQf6am3R+CeBV4H\nlfI1HZTNRIBHgvFszRZkCERbRgEWMu+P+I6/7GJC5H5jhVQ60z4MmXcyHOGmYMK/\n5XyuHQz7V2Ssu1AxLfRN5Biq1ayb0+DT/E7QxNXDJPqSTnstZ6C7zKH/uAETqg3l\nBonjCQWyds+IYbQYxf5Sp3yhvQ80lMwHML3DaNCMlXWLoOnrOX5/yK5+dedesg2\n/HIvGk+HEt36vm6hoH7bwPuEkgA++ACqwjXRe5Mta7i3eilHxFaF8XIrJFARV0t\nqOu4GID/jG6oA+swIWndGrtR2QRJIt9QIBFfK3HG5M0koZbY1eTqwNFRHFL3xaD\nUQIDAQAB\n-----END PUBLIC KEY-----\n", "owner": "http://mastodon.example.org/users/gargron", "id": "http://mastodon.example.org/users/gargron#main-key" @@ -20,7 +20,27 @@ "endpoints": { "sharedInbox": "http://mastodon.example.org/inbox" }, - "icon":{"type":"Image","mediaType":"image/jpeg","url":"https://cd.niu.moe/accounts/avatars/000/033/323/original/fd7f8ae0b3ffedc9.jpeg"},"image":{"type":"Image","mediaType":"image/png","url":"https://cd.niu.moe/accounts/headers/000/033/323/original/850b3448fa5fd477.png"} + "attachment": [{ + "type": "PropertyValue", + "name": "foo", + "value": "updated" + }, + { + "type": "PropertyValue", + "name": "foo1", + "value": "updated" + } + ], + "icon": { + "type": "Image", + "mediaType": "image/jpeg", + "url": "https://cd.niu.moe/accounts/avatars/000/033/323/original/fd7f8ae0b3ffedc9.jpeg" + }, + "image": { + "type": "Image", + "mediaType": "image/png", + "url": "https://cd.niu.moe/accounts/headers/000/033/323/original/850b3448fa5fd477.png" + } }, "id": "http://mastodon.example.org/users/gargron#updates/1519563538", "actor": "http://mastodon.example.org/users/gargron", diff --git a/test/fixtures/tesla_mock/admin@mastdon.example.org.json b/test/fixtures/tesla_mock/admin@mastdon.example.org.json index c297e4349..8159dc20a 100644 --- a/test/fixtures/tesla_mock/admin@mastdon.example.org.json +++ b/test/fixtures/tesla_mock/admin@mastdon.example.org.json @@ -1 +1,54 @@ -{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1",{"manuallyApprovesFollowers":"as:manuallyApprovesFollowers","sensitive":"as:sensitive","movedTo":"as:movedTo","Hashtag":"as:Hashtag","ostatus":"http://ostatus.org#","atomUri":"ostatus:atomUri","inReplyToAtomUri":"ostatus:inReplyToAtomUri","conversation":"ostatus:conversation","toot":"http://joinmastodon.org/ns#","Emoji":"toot:Emoji"}],"id":"http://mastodon.example.org/users/admin","type":"Person","following":"http://mastodon.example.org/users/admin/following","followers":"http://mastodon.example.org/users/admin/followers","inbox":"http://mastodon.example.org/users/admin/inbox","outbox":"http://mastodon.example.org/users/admin/outbox","preferredUsername":"admin","name":null,"summary":"\u003cp\u003e\u003c/p\u003e","url":"http://mastodon.example.org/@admin","manuallyApprovesFollowers":false,"publicKey":{"id":"http://mastodon.example.org/users/admin#main-key","owner":"http://mastodon.example.org/users/admin","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtc4Tir+3ADhSNF6VKrtW\nOU32T01w7V0yshmQei38YyiVwVvFu8XOP6ACchkdxbJ+C9mZud8qWaRJKVbFTMUG\nNX4+6Q+FobyuKrwN7CEwhDALZtaN2IPbaPd6uG1B7QhWorrY+yFa8f2TBM3BxnUy\nI4T+bMIZIEYG7KtljCBoQXuTQmGtuffO0UwJksidg2ffCF5Q+K//JfQagJ3UzrR+\nZXbKMJdAw4bCVJYs4Z5EhHYBwQWiXCyMGTd7BGlmMkY6Av7ZqHKC/owp3/0EWDNz\nNqF09Wcpr3y3e8nA10X40MJqp/wR+1xtxp+YGbq/Cj5hZGBG7etFOmIpVBrDOhry\nBwIDAQAB\n-----END PUBLIC KEY-----\n"},"endpoints":{"sharedInbox":"http://mastodon.example.org/inbox"},"icon":{"type":"Image","mediaType":"image/jpeg","url":"https://cdn.niu.moe/accounts/avatars/000/033/323/original/fd7f8ae0b3ffedc9.jpeg"},"image":{"type":"Image","mediaType":"image/png","url":"https://cdn.niu.moe/accounts/headers/000/033/323/original/850b3448fa5fd477.png"}} +{ + "@context": ["https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1", { + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "sensitive": "as:sensitive", + "movedTo": "as:movedTo", + "Hashtag": "as:Hashtag", + "ostatus": "http://ostatus.org#", + "atomUri": "ostatus:atomUri", + "inReplyToAtomUri": "ostatus:inReplyToAtomUri", + "conversation": "ostatus:conversation", + "toot": "http://joinmastodon.org/ns#", + "Emoji": "toot:Emoji" + }], + "id": "http://mastodon.example.org/users/admin", + "type": "Person", + "following": "http://mastodon.example.org/users/admin/following", + "followers": "http://mastodon.example.org/users/admin/followers", + "inbox": "http://mastodon.example.org/users/admin/inbox", + "outbox": "http://mastodon.example.org/users/admin/outbox", + "preferredUsername": "admin", + "name": null, + "summary": "\u003cp\u003e\u003c/p\u003e", + "url": "http://mastodon.example.org/@admin", + "manuallyApprovesFollowers": false, + "publicKey": { + "id": "http://mastodon.example.org/users/admin#main-key", + "owner": "http://mastodon.example.org/users/admin", + "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtc4Tir+3ADhSNF6VKrtW\nOU32T01w7V0yshmQei38YyiVwVvFu8XOP6ACchkdxbJ+C9mZud8qWaRJKVbFTMUG\nNX4+6Q+FobyuKrwN7CEwhDALZtaN2IPbaPd6uG1B7QhWorrY+yFa8f2TBM3BxnUy\nI4T+bMIZIEYG7KtljCBoQXuTQmGtuffO0UwJksidg2ffCF5Q+K//JfQagJ3UzrR+\nZXbKMJdAw4bCVJYs4Z5EhHYBwQWiXCyMGTd7BGlmMkY6Av7ZqHKC/owp3/0EWDNz\nNqF09Wcpr3y3e8nA10X40MJqp/wR+1xtxp+YGbq/Cj5hZGBG7etFOmIpVBrDOhry\nBwIDAQAB\n-----END PUBLIC KEY-----\n" + }, + "attachment": [{ + "type": "PropertyValue", + "name": "foo", + "value": "bar" + }, + { + "type": "PropertyValue", + "name": "foo1", + "value": "bar1" + } + ], + "endpoints": { + "sharedInbox": "http://mastodon.example.org/inbox" + }, + "icon": { + "type": "Image", + "mediaType": "image/jpeg", + "url": "https://cdn.niu.moe/accounts/avatars/000/033/323/original/fd7f8ae0b3ffedc9.jpeg" + }, + "image": { + "type": "Image", + "mediaType": "image/png", + "url": "https://cdn.niu.moe/accounts/headers/000/033/323/original/850b3448fa5fd477.png" + } +} diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 060b91e29..05ec09ec1 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -509,6 +509,42 @@ test "it works for incoming update activities" do assert user.bio == "

Some bio

" end + test "it works with custom profile fields" do + {:ok, activity} = + "test/fixtures/mastodon-post-activity.json" + |> File.read!() + |> Poison.decode!() + |> Transmogrifier.handle_incoming() + + user = User.get_cached_by_ap_id(activity.actor) + + assert user.info.source_data["attachment"] == [ + %{"name" => "foo", "type" => "PropertyValue", "value" => "bar"}, + %{"name" => "foo1", "type" => "PropertyValue", "value" => "bar1"} + ] + + update_data = File.read!("test/fixtures/mastodon-update.json") |> Poison.decode!() + + object = + update_data["object"] + |> Map.put("actor", user.ap_id) + |> Map.put("id", user.ap_id) + + update_data = + update_data + |> Map.put("actor", user.ap_id) + |> Map.put("object", object) + + {:ok, _update_activity} = Transmogrifier.handle_incoming(update_data) + + user = User.get_cached_by_ap_id(user.ap_id) + + assert user.info.source_data["attachment"] == [ + %{"name" => "foo", "type" => "PropertyValue", "value" => "updated"}, + %{"name" => "foo1", "type" => "PropertyValue", "value" => "updated"} + ] + end + test "it works for incoming update activities which lock the account" do data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() From 7d6f8a7fd75e5de4e0c9ce208ac9276dcbe044f5 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 29 Jul 2019 19:17:09 +0700 Subject: [PATCH 39/94] Linkify custom fields values in ActivityPub.UserViewx --- lib/pleroma/web/activity_pub/views/user_view.ex | 1 + test/web/activity_pub/views/user_view_test.exs | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 7b4bc998b..b2a22478d 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -84,6 +84,7 @@ def render("user.json", %{user: user}) do user.info |> User.Info.fields() |> Enum.map(&Map.put(&1, "type", "PropertyValue")) + |> Enum.map(fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end) %{ "id" => user.ap_id, diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index 48a522c6c..a2aa52381 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -33,7 +33,11 @@ test "Renders profile fields" do assert %{ "attachment" => [ %{"name" => "foo", "type" => "PropertyValue", "value" => "bar"}, - %{"name" => "website", "type" => "PropertyValue", "value" => "cofe.my"} + %{ + "name" => "website", + "type" => "PropertyValue", + "value" => "cofe.my" + } ] } = UserView.render("user.json", %{user: user}) end From db3c05f6b4c226733633a409cb1f1a290db4c48b Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 30 Jul 2019 17:22:52 +0700 Subject: [PATCH 40/94] Add configurable account field value length limit --- config/config.exs | 1 + docs/config.md | 1 + lib/pleroma/user/info.ex | 7 ++++- .../update_credentials_test.exs | 31 +++++++++++++++---- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/config/config.exs b/config/config.exs index 109cf6516..21f4861f2 100644 --- a/config/config.exs +++ b/config/config.exs @@ -256,6 +256,7 @@ user_bio_length: 5000, user_name_length: 100, max_account_fields: 4, + account_field_value_length: 255, external_user_synchronization: true config :pleroma, :markup, diff --git a/docs/config.md b/docs/config.md index ca5da7db1..fbb506093 100644 --- a/docs/config.md +++ b/docs/config.md @@ -133,6 +133,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. * `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. * `max_account_fields`: The maximum number of custom fields in the user profile (default: `4`) +* `account_field_value_length`: An account field value maximum length (default: `255`) * `external_user_synchronization`: Enabling following/followers counters synchronization for external users. diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 98b894223..9e4d381f8 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -308,7 +308,12 @@ def validate_fields(changeset) do end defp valid_field?(%{"name" => name, "value" => value}) do - is_binary(name) && is_binary(value) + value_limit = Pleroma.Config.get([:instance, :account_field_value_length], 255) + + is_binary(name) && + is_binary(value) && + String.length(name) <= 255 && + String.length(value) <= value_limit end defp valid_field?(_), do: false diff --git a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs index a3eadde16..992a692f0 100644 --- a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs @@ -325,6 +325,26 @@ test "update fields", %{conn: conn} do %{"name" => "link", "value" => "cofe.io"} ] + value_limit = Pleroma.Config.get([:instance, :account_field_value_length]) + + long_str = Enum.map(0..value_limit, fn _ -> "x" end) |> Enum.join() + + fields = [%{"name" => "foo", "value" => long_str}] + + assert %{"error" => "Invalid request"} == + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_credentials", %{"fields" => fields}) + |> json_response(403) + + fields = [%{"name" => long_str, "value" => "bar"}] + + assert %{"error" => "Invalid request"} == + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_credentials", %{"fields" => fields}) + |> json_response(403) + Pleroma.Config.put([:instance, :max_account_fields], 1) fields = [ @@ -332,12 +352,11 @@ test "update fields", %{conn: conn} do %{"name" => "link", "value" => "cofe.io"} ] - conn = - conn - |> assign(:user, user) - |> patch("/api/v1/accounts/update_credentials", %{"fields" => fields}) - - assert %{"error" => "Invalid request"} == json_response(conn, 403) + assert %{"error" => "Invalid request"} == + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_credentials", %{"fields" => fields}) + |> json_response(403) end end end From 2c35d4b0b04e58368c51f2828536d295f72839a2 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 1 Aug 2019 15:09:15 +0700 Subject: [PATCH 41/94] Add configurable account field name length limit --- config/config.exs | 1 + docs/config.md | 1 + lib/pleroma/user/info.ex | 3 ++- .../mastodon_api_controller/update_credentials_test.exs | 9 ++++++--- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/config/config.exs b/config/config.exs index 21f4861f2..4fd241e90 100644 --- a/config/config.exs +++ b/config/config.exs @@ -256,6 +256,7 @@ user_bio_length: 5000, user_name_length: 100, max_account_fields: 4, + account_field_name_length: 255, account_field_value_length: 255, external_user_synchronization: true diff --git a/docs/config.md b/docs/config.md index fbb506093..6744f5879 100644 --- a/docs/config.md +++ b/docs/config.md @@ -133,6 +133,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. * `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. * `max_account_fields`: The maximum number of custom fields in the user profile (default: `4`) +* `account_field_name_length`: An account field name maximum length (default: `255`) * `account_field_value_length`: An account field value maximum length (default: `255`) * `external_user_synchronization`: Enabling following/followers counters synchronization for external users. diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 9e4d381f8..e54243f06 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -308,11 +308,12 @@ def validate_fields(changeset) do end defp valid_field?(%{"name" => name, "value" => value}) do + name_limit = Pleroma.Config.get([:instance, :account_field_name_length], 255) value_limit = Pleroma.Config.get([:instance, :account_field_value_length], 255) is_binary(name) && is_binary(value) && - String.length(name) <= 255 && + String.length(name) <= name_limit && String.length(value) <= value_limit end diff --git a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs index 992a692f0..e75f25d51 100644 --- a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs @@ -325,11 +325,12 @@ test "update fields", %{conn: conn} do %{"name" => "link", "value" => "cofe.io"} ] + name_limit = Pleroma.Config.get([:instance, :account_field_name_length]) value_limit = Pleroma.Config.get([:instance, :account_field_value_length]) - long_str = Enum.map(0..value_limit, fn _ -> "x" end) |> Enum.join() + long_value = Enum.map(0..value_limit, fn _ -> "x" end) |> Enum.join() - fields = [%{"name" => "foo", "value" => long_str}] + fields = [%{"name" => "foo", "value" => long_value}] assert %{"error" => "Invalid request"} == conn @@ -337,7 +338,9 @@ test "update fields", %{conn: conn} do |> patch("/api/v1/accounts/update_credentials", %{"fields" => fields}) |> json_response(403) - fields = [%{"name" => long_str, "value" => "bar"}] + long_name = Enum.map(0..name_limit, fn _ -> "x" end) |> Enum.join() + + fields = [%{"name" => long_name, "value" => "bar"}] assert %{"error" => "Invalid request"} == conn From f7bbf99caade7f06756e95e3a4e2f0e4d3e76579 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 6 Aug 2019 18:21:25 +0700 Subject: [PATCH 42/94] Use info.fields instead of source_data for remote users --- lib/pleroma/html.ex | 28 +++++++++++++++++++ lib/pleroma/user/info.ex | 6 +++- .../web/activity_pub/transmogrifier.ex | 10 +++++-- .../web/activity_pub/views/user_view.ex | 7 ++++- .../mastodon_api/mastodon_api_controller.ex | 13 ++++----- .../web/mastodon_api/views/account_view.ex | 18 ++++++++---- .../web/twitter_api/views/user_view.ex | 10 ++++++- test/web/activity_pub/transmogrifier_test.exs | 12 ++++---- .../web/activity_pub/views/user_view_test.exs | 17 ++++------- .../update_credentials_test.exs | 9 ++++-- 10 files changed, 91 insertions(+), 39 deletions(-) diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index 2fae7281c..bf2000d90 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -280,3 +280,31 @@ def scrub({tag, attributes, children}), do: {tag, attributes, children} def scrub({_tag, children}), do: children def scrub(text), do: text end + +defmodule Pleroma.HTML.Scrubber.LinksOnly do + @moduledoc """ + An HTML scrubbing policy which limits to links only. + """ + + @valid_schemes Pleroma.Config.get([:uri_schemes, :valid_schemes], []) + + require HtmlSanitizeEx.Scrubber.Meta + alias HtmlSanitizeEx.Scrubber.Meta + + Meta.remove_cdata_sections_before_scrub() + Meta.strip_comments() + + # links + Meta.allow_tag_with_uri_attributes("a", ["href"], @valid_schemes) + + Meta.allow_tag_with_this_attribute_values("a", "rel", [ + "tag", + "nofollow", + "noopener", + "noreferrer", + "me" + ]) + + Meta.allow_tag_with_these_attributes("a", ["name", "title"]) + Meta.strip_everything_not_covered() +end diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index e54243f06..ada9fb689 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -50,6 +50,7 @@ defmodule Pleroma.User.Info do field(:emoji, {:array, :map}, default: []) field(:pleroma_settings_store, :map, default: %{}) field(:fields, {:array, :map}, default: []) + field(:raw_fields, {:array, :map}, default: []) field(:notification_settings, :map, default: %{ @@ -270,8 +271,10 @@ def user_upgrade(info, params) do :follower_count, :following_count, :hide_follows, + :fields, :hide_followers ]) + |> validate_fields() end def profile_update(info, params) do @@ -288,6 +291,7 @@ def profile_update(info, params) do :show_role, :skip_thread_containment, :fields, + :raw_fields, :pleroma_settings_store ]) |> validate_fields() @@ -415,7 +419,7 @@ def remove_reblog_mute(info, ap_id) do # ``fields`` is an array of mastodon profile field, containing ``{"name": "…", "value": "…"}``. # For example: [{"name": "Pronoun", "value": "she/her"}, …] - def fields(%{source_data: %{"attachment" => attachment}}) do + def fields(%{fields: [], source_data: %{"attachment" => attachment}}) do attachment |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 225c34875..2be2e3294 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -598,13 +598,17 @@ def handle_incoming( banner = new_user_data[:info][:banner] locked = new_user_data[:info][:locked] || false - attachment = get_in(new_user_data, [:info, "source_data", "attachment"]) + attachment = get_in(new_user_data, [:info, :source_data, "attachment"]) || [] + + fields = + attachment + |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) + |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) update_data = new_user_data |> Map.take([:name, :bio, :avatar]) - |> Map.put(:info, %{banner: banner, locked: locked}) - |> Map.put(:info, %{"banner" => banner, "locked" => locked, "source_data" => source_data}) + |> Map.put(:info, %{banner: banner, locked: locked, fields: fields}) actor |> User.upgrade_changeset(update_data) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index b2a22478d..7be734b26 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -83,8 +83,13 @@ def render("user.json", %{user: user}) do fields = user.info |> User.Info.fields() + |> Enum.map(fn %{"name" => name, "value" => value} -> + %{ + "name" => Pleroma.HTML.strip_tags(name), + "value" => Pleroma.HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly) + } + end) |> Enum.map(&Map.put(&1, "type", "PropertyValue")) - |> Enum.map(fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end) %{ "id" => user.ap_id, diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index e79a02caa..e8fac8880 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -137,7 +137,9 @@ def update_credentials(%{assigns: %{user: user}} = conn, params) do emojis_text = (user_params["display_name"] || "") <> (user_params["note"] || "") user_info_emojis = - ((user.info.emoji || []) ++ Formatter.get_emoji_map(emojis_text)) + user.info + |> Map.get(:emoji, []) + |> Enum.concat(Formatter.get_emoji_map(emojis_text)) |> Enum.dedup() info_params = @@ -157,16 +159,11 @@ def update_credentials(%{assigns: %{user: user}} = conn, params) do end) |> add_if_present(params, "default_scope", :default_scope) |> add_if_present(params, "fields", :fields, fn fields -> - fields = - Enum.map(fields, fn field -> - %{ - "name" => Formatter.html_escape(field["name"], "text/plain"), - "value" => Formatter.html_escape(field["value"], "text/plain") - } - end) + fields = Enum.map(fields, fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end) {:ok, fields} end) + |> add_if_present(params, "fields", :raw_fields) |> add_if_present(params, "pleroma_settings_store", :pleroma_settings_store, fn value -> {:ok, Map.merge(user.info.pleroma_settings_store, value)} end) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index d2f3986ff..a2297a8e8 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -93,11 +93,19 @@ defp do_render("account.json", %{user: user} = opts) do } end) - fields = User.Info.fields(user.info) - fields_html = Enum.map(fields, fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end) + fields = + user.info + |> User.Info.fields() + |> Enum.map(fn %{"name" => name, "value" => value} -> + %{ + "name" => Pleroma.HTML.strip_tags(name), + "value" => Pleroma.HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly) + } + end) + + raw_fields = Map.get(user.info, :raw_fields, []) bio = HTML.filter_tags(user.bio, User.html_filter_policy(opts[:for])) - relationship = render("relationship.json", %{user: opts[:for], target: user}) %{ @@ -117,12 +125,12 @@ defp do_render("account.json", %{user: user} = opts) do header: header, header_static: header, emojis: emojis, - fields: fields_html, + fields: fields, bot: bot, source: %{ note: HTML.strip_tags((user.bio || "") |> String.replace("
", "\n")), sensitive: false, - fields: fields, + fields: raw_fields, pleroma: %{} }, diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex index 3681773be..8a7d2fc72 100644 --- a/lib/pleroma/web/twitter_api/views/user_view.ex +++ b/lib/pleroma/web/twitter_api/views/user_view.ex @@ -74,7 +74,15 @@ defp do_render("user.json", %{user: user = %User{}} = assigns) do |> HTML.filter_tags(User.html_filter_policy(for_user)) |> Formatter.emojify(emoji) - fields = User.Info.fields(user.info) + fields = + user.info + |> User.Info.fields() + |> Enum.map(fn %{"name" => name, "value" => value} -> + %{ + "name" => Pleroma.HTML.strip_tags(name), + "value" => Pleroma.HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly) + } + end) data = %{ diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 05ec09ec1..7e2c8769d 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -518,9 +518,9 @@ test "it works with custom profile fields" do user = User.get_cached_by_ap_id(activity.actor) - assert user.info.source_data["attachment"] == [ - %{"name" => "foo", "type" => "PropertyValue", "value" => "bar"}, - %{"name" => "foo1", "type" => "PropertyValue", "value" => "bar1"} + assert User.Info.fields(user.info) == [ + %{"name" => "foo", "value" => "bar"}, + %{"name" => "foo1", "value" => "bar1"} ] update_data = File.read!("test/fixtures/mastodon-update.json") |> Poison.decode!() @@ -539,9 +539,9 @@ test "it works with custom profile fields" do user = User.get_cached_by_ap_id(user.ap_id) - assert user.info.source_data["attachment"] == [ - %{"name" => "foo", "type" => "PropertyValue", "value" => "updated"}, - %{"name" => "foo1", "type" => "PropertyValue", "value" => "updated"} + assert User.Info.fields(user.info) == [ + %{"name" => "foo", "value" => "updated"}, + %{"name" => "foo1", "value" => "updated"} ] end diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index a2aa52381..fb7fd9e79 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -24,21 +24,16 @@ test "Renders a user, including the public key" do test "Renders profile fields" do fields = [ - %{"name" => "foo", "value" => "bar"}, - %{"name" => "website", "value" => "cofe.my"} + %{"name" => "foo", "value" => "bar"} ] - user = insert(:user, info: %{fields: fields}) + {:ok, user} = + insert(:user) + |> User.upgrade_changeset(%{info: %{fields: fields}}) + |> User.update_and_set_cache() assert %{ - "attachment" => [ - %{"name" => "foo", "type" => "PropertyValue", "value" => "bar"}, - %{ - "name" => "website", - "type" => "PropertyValue", - "value" => "cofe.my" - } - ] + "attachment" => [%{"name" => "foo", "type" => "PropertyValue", "value" => "bar"}] } = UserView.render("user.json", %{user: user}) end diff --git a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs index e75f25d51..dd443495b 100644 --- a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs @@ -305,7 +305,7 @@ test "update fields", %{conn: conn} do user = insert(:user) fields = [ - %{"name" => "foo", "value" => "bar"}, + %{"name" => "foo", "value" => ""}, %{"name" => "link", "value" => "cofe.io"} ] @@ -316,12 +316,15 @@ test "update fields", %{conn: conn} do |> json_response(200) assert account["fields"] == [ - %{"name" => "<b>foo<b>", "value" => "<i>bar</i>"}, + %{"name" => "foo", "value" => "bar"}, %{"name" => "link", "value" => "cofe.io"} ] assert account["source"]["fields"] == [ - %{"name" => "<b>foo<b>", "value" => "<i>bar</i>"}, + %{ + "name" => "foo", + "value" => "" + }, %{"name" => "link", "value" => "cofe.io"} ] From e457fcc47971df6c76c3da096e6b45c2972e4029 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 7 Aug 2019 18:14:22 +0700 Subject: [PATCH 43/94] Add `:max_remote_account_fields` config option --- config/config.exs | 1 + docs/config.md | 1 + lib/pleroma/user.ex | 4 ++-- lib/pleroma/user/info.ex | 11 +++++++---- lib/pleroma/web/activity_pub/activity_pub.ex | 7 +++++++ lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +- test/web/activity_pub/transmogrifier_test.exs | 18 ++++++++++++++++++ 7 files changed, 37 insertions(+), 7 deletions(-) diff --git a/config/config.exs b/config/config.exs index 4fd241e90..3937f7e70 100644 --- a/config/config.exs +++ b/config/config.exs @@ -256,6 +256,7 @@ user_bio_length: 5000, user_name_length: 100, max_account_fields: 4, + max_remote_account_fields: 10, account_field_name_length: 255, account_field_value_length: 255, external_user_synchronization: true diff --git a/docs/config.md b/docs/config.md index 6744f5879..b4bdbd078 100644 --- a/docs/config.md +++ b/docs/config.md @@ -133,6 +133,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. * `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. * `max_account_fields`: The maximum number of custom fields in the user profile (default: `4`) +* `max_remote_account_fields`: The maximum number of custom fields in the remote user profile (default: `10`) * `account_field_name_length`: An account field name maximum length (default: `255`) * `account_field_value_length`: An account field value maximum length (default: `255`) * `external_user_synchronization`: Enabling following/followers counters synchronization for external users. diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index b67743846..faa1e3d50 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -199,12 +199,12 @@ def update_changeset(struct, params \\ %{}) do |> validate_length(:name, min: 1, max: name_limit) end - def upgrade_changeset(struct, params \\ %{}) do + def upgrade_changeset(struct, params \\ %{}, remote? \\ false) do bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000) name_limit = Pleroma.Config.get([:instance, :user_name_length], 100) params = Map.put(params, :last_refreshed_at, NaiveDateTime.utc_now()) - info_cng = User.Info.user_upgrade(struct.info, params[:info]) + info_cng = User.Info.user_upgrade(struct.info, params[:info], remote?) struct |> cast(params, [ diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index ada9fb689..47e7df911 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -256,11 +256,13 @@ def remote_user_creation(info, params) do :hide_followers, :hide_follows, :follower_count, + :fields, :following_count ]) + |> validate_fields(true) end - def user_upgrade(info, params) do + def user_upgrade(info, params, remote? \\ false) do info |> cast(params, [ :ap_enabled, @@ -274,7 +276,7 @@ def user_upgrade(info, params) do :fields, :hide_followers ]) - |> validate_fields() + |> validate_fields(remote?) end def profile_update(info, params) do @@ -297,8 +299,9 @@ def profile_update(info, params) do |> validate_fields() end - def validate_fields(changeset) do - limit = Pleroma.Config.get([:instance, :max_account_fields], 0) + def validate_fields(changeset, remote? \\ false) do + limit_name = if remote?, do: :max_remote_account_fields, else: :max_account_fields + limit = Pleroma.Config.get([:instance, limit_name], 0) changeset |> validate_length(:fields, max: limit) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index cf55c9520..7bb7740bf 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1016,6 +1016,12 @@ defp object_to_user_data(data) do "url" => [%{"href" => data["image"]["url"]}] } + fields = + data + |> Map.get("attachment", []) + |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) + |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) + locked = data["manuallyApprovesFollowers"] || false data = Transmogrifier.maybe_fix_user_object(data) @@ -1025,6 +1031,7 @@ defp object_to_user_data(data) do ap_enabled: true, source_data: data, banner: banner, + fields: fields, locked: locked }, avatar: avatar, diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 2be2e3294..36340a3a1 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -611,7 +611,7 @@ def handle_incoming( |> Map.put(:info, %{banner: banner, locked: locked, fields: fields}) actor - |> User.upgrade_changeset(update_data) + |> User.upgrade_changeset(update_data, true) |> User.update_and_set_cache() ActivityPub.update(%{ diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 7e2c8769d..d8fbcd628 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -539,6 +539,24 @@ test "it works with custom profile fields" do user = User.get_cached_by_ap_id(user.ap_id) + assert User.Info.fields(user.info) == [ + %{"name" => "foo", "value" => "updated"}, + %{"name" => "foo1", "value" => "updated"} + ] + + Pleroma.Config.put([:instance, :max_remote_account_fields], 2) + + update_data = + put_in(update_data, ["object", "attachment"], [ + %{"name" => "foo", "type" => "PropertyValue", "value" => "bar"}, + %{"name" => "foo11", "type" => "PropertyValue", "value" => "bar11"}, + %{"name" => "foo22", "type" => "PropertyValue", "value" => "bar22"} + ]) + + {:ok, _} = Transmogrifier.handle_incoming(update_data) + + user = User.get_cached_by_ap_id(user.ap_id) + assert User.Info.fields(user.info) == [ %{"name" => "foo", "value" => "updated"}, %{"name" => "foo1", "value" => "updated"} From 672fcbc7b716f18346a17845d05c286b45dca5f3 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 7 Aug 2019 18:48:05 +0700 Subject: [PATCH 44/94] Limit custom fields for old remote users --- lib/pleroma/user/info.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 47e7df911..45a39924b 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -423,9 +423,12 @@ def remove_reblog_mute(info, ap_id) do # ``fields`` is an array of mastodon profile field, containing ``{"name": "…", "value": "…"}``. # For example: [{"name": "Pronoun", "value": "she/her"}, …] def fields(%{fields: [], source_data: %{"attachment" => attachment}}) do + limit = Pleroma.Config.get([:instance, :max_remote_account_fields], 0) + attachment |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) + |> Enum.take(limit) end def fields(%{fields: fields}), do: fields From a2e1db56323d0f306ee42a1f58471eb55c8c1e68 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 8 Aug 2019 13:20:35 +0700 Subject: [PATCH 45/94] Increase max account fields limits --- config/config.exs | 4 ++-- docs/config.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/config.exs b/config/config.exs index 3937f7e70..7279b43b7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -255,8 +255,8 @@ dynamic_configuration: false, user_bio_length: 5000, user_name_length: 100, - max_account_fields: 4, - max_remote_account_fields: 10, + max_account_fields: 10, + max_remote_account_fields: 20, account_field_name_length: 255, account_field_value_length: 255, external_user_synchronization: true diff --git a/docs/config.md b/docs/config.md index b4bdbd078..7e4e96db0 100644 --- a/docs/config.md +++ b/docs/config.md @@ -132,8 +132,8 @@ config :pleroma, Pleroma.Emails.Mailer, * `skip_thread_containment`: Skip filter out broken threads. The default is `false`. * `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. * `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. -* `max_account_fields`: The maximum number of custom fields in the user profile (default: `4`) -* `max_remote_account_fields`: The maximum number of custom fields in the remote user profile (default: `10`) +* `max_account_fields`: The maximum number of custom fields in the user profile (default: `10`) +* `max_remote_account_fields`: The maximum number of custom fields in the remote user profile (default: `20`) * `account_field_name_length`: An account field name maximum length (default: `255`) * `account_field_value_length`: An account field value maximum length (default: `255`) * `external_user_synchronization`: Enabling following/followers counters synchronization for external users. From 4b7f1c6995ca49c782e3e29d14245f18d4d11430 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 14 Aug 2019 20:46:05 +0700 Subject: [PATCH 46/94] Improve digest email template --- lib/mix/tasks/pleroma/digest.ex | 12 +- lib/pleroma/emails/user_email.ex | 17 + .../web/templates/email/digest.html.eex | 581 +++++++++++++++++- lib/pleroma/web/views/email_view.ex | 10 + test/mix/tasks/pleroma.digest_test.exs | 2 +- 5 files changed, 602 insertions(+), 20 deletions(-) diff --git a/lib/mix/tasks/pleroma/digest.ex b/lib/mix/tasks/pleroma/digest.ex index 81c207e10..430116a50 100644 --- a/lib/mix/tasks/pleroma/digest.ex +++ b/lib/mix/tasks/pleroma/digest.ex @@ -27,7 +27,15 @@ def run(["test", nickname | opts]) do patched_user = %{user | last_digest_emailed_at: last_digest_emailed_at} - _user = Pleroma.DigestEmailWorker.perform(patched_user) - Mix.shell().info("Digest email have been sent to #{nickname} (#{user.email})") + with %Swoosh.Email{} = email <- Pleroma.Emails.UserEmail.digest_email(patched_user) do + {:ok, _} = Pleroma.Emails.Mailer.deliver(email) + + Mix.shell().info("Digest email have been sent to #{nickname} (#{user.email})") + else + _ -> + Mix.shell().info( + "Cound't find any mentions for #{nickname} since #{last_digest_emailed_at}" + ) + end end end diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index 49046bb8b..bf6b811b1 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -123,6 +123,11 @@ def digest_email(user) do end) with [_ | _] = mentions <- new_notifications.mentions do + mentions = + Enum.map(mentions, fn mention -> + update_in(mention.object.data["content"], &format_links/1) + end) + html_data = %{ instance: instance_name(), user: user, @@ -131,17 +136,29 @@ def digest_email(user) do unsubscribe_link: unsubscribe_url(user, "digest") } + logo_path = Path.join(:code.priv_dir(:pleroma), "static/static/logo.png") + new() |> to(recipient(user)) |> from(sender()) |> subject("Your digest from #{instance_name()}") + |> put_layout(false) |> render_body("digest.html", html_data) + |> attachment(Swoosh.Attachment.new(logo_path, filename: "logo.png", type: :inline)) else _ -> nil end end + defp format_links(str) do + re = ~r//iU + + String.replace(str, re, fn link -> + String.replace(link, "Hey <%= @user.nickname %>, here is what you've missed! + -

New Mentions:

-
    -<%= for %{data: mention, object: object, from: from} <- @mentions do %> -
  • <%= link from.nickname, to: mention.activity.actor %>: <%= raw object.data["content"] %>
    • -<% end %> -
+ -<%= if @followers != [] do %> -

<%= length(@followers) %> New Followers:

-
    -<%= for %{data: follow, from: from} <- @followers do %> -
  • <%= link from.nickname, to: follow.activity.actor %>
    • -<% end %> -
-<% end %> + + + + + + + + <%= @email.subject %>< + + + + + + + + + + + + + + + + + + + diff --git a/lib/pleroma/web/views/email_view.ex b/lib/pleroma/web/views/email_view.ex index b63eb162c..b506a234b 100644 --- a/lib/pleroma/web/views/email_view.ex +++ b/lib/pleroma/web/views/email_view.ex @@ -2,4 +2,14 @@ defmodule Pleroma.Web.EmailView do use Pleroma.Web, :view import Phoenix.HTML import Phoenix.HTML.Link + + def avatar_url(user) do + Pleroma.User.avatar_url(user) + end + + def format_date(date) when is_binary(date) do + date + |> Timex.parse!("{ISO:Extended:Z}") + |> Timex.format!("{Mshort} {D}, {YYYY} {h24}:{m}") + end end diff --git a/test/mix/tasks/pleroma.digest_test.exs b/test/mix/tasks/pleroma.digest_test.exs index 595f64ed7..4bfa1fb93 100644 --- a/test/mix/tasks/pleroma.digest_test.exs +++ b/test/mix/tasks/pleroma.digest_test.exs @@ -44,7 +44,7 @@ test "Sends digest to the given user" do assert_email_sent( to: {user2.name, user2.email}, - html_body: ~r/new mentions:/i + html_body: ~r/here is what you've missed!/i ) end end From df81abb68c46e36dfb2af7aab77e0e57a1a1eb28 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 14 Aug 2019 15:55:43 +0200 Subject: [PATCH 47/94] Conversations: Use correct oauth paths for extended api. --- lib/pleroma/web/router.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 9759268f9..1eb6f7b9d 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -263,9 +263,13 @@ defmodule Pleroma.Web.Router do pipe_through(:authenticated_api) scope [] do - pipe_through(:oauth_write) + pipe_through(:oauth_read) get("/conversations/:id/statuses", PleromaAPIController, :conversation_statuses) get("/conversations/:id", PleromaAPIController, :conversation) + end + + scope [] do + pipe_through(:oauth_write) patch("/conversations/:id", PleromaAPIController, :update_conversation) end end From f73212b2a36deef631716f3c8a80d7da11cec759 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 14 Aug 2019 15:56:15 +0200 Subject: [PATCH 48/94] Conversation: Render new participation on update. --- lib/pleroma/web/pleroma_api/pleroma_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex index b5c3d2728..6d74d418e 100644 --- a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex @@ -66,7 +66,7 @@ def update_conversation( |> Participation.get() with true <- user.id == participation.user_id, - {:ok, _} <- Participation.set_recipients(participation, recipients) do + {:ok, participation} <- Participation.set_recipients(participation, recipients) do conn |> put_view(ConversationView) |> render("participation.json", %{participation: participation, for: user}) From c9970feee20f25375223e5f4a32bdbcff7b98607 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 14 Aug 2019 21:03:25 +0700 Subject: [PATCH 49/94] Fix compatibility with Elixir 1.8 --- lib/pleroma/emails/user_email.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index bf6b811b1..3b5e64019 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -154,7 +154,7 @@ def digest_email(user) do defp format_links(str) do re = ~r//iU - String.replace(str, re, fn link -> + Regex.replace(re, str, fn link -> String.replace(link, " Date: Wed, 14 Aug 2019 17:01:11 +0200 Subject: [PATCH 50/94] Conversations: Load relations in one query. --- lib/pleroma/conversation/participation.ex | 16 +++++++++++++--- .../web/pleroma_api/pleroma_api_controller.ex | 4 +--- test/conversation/participation_test.exs | 14 ++++++++++++++ 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index d17b6f7c5..ea5b9fe17 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -94,10 +94,20 @@ def for_user_with_last_activity_id(user, params \\ %{}) do |> Enum.filter(& &1.last_activity_id) end - def get(nil), do: nil + def get(_, _ \\ []) + def get(nil, _), do: nil - def get(id) do - Repo.get(__MODULE__, id) + def get(id, params) do + query = + if preload = params[:preload] do + from(p in __MODULE__, + preload: ^preload + ) + else + __MODULE__ + end + + Repo.get(query, id) end def set_recipients(participation, user_ids) do diff --git a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex index 6d74d418e..b6d2bf86b 100644 --- a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do import Pleroma.Web.ControllerHelper, only: [add_link_headers: 7] alias Pleroma.Conversation.Participation - alias Pleroma.Repo alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.MastodonAPI.ConversationView alias Pleroma.Web.MastodonAPI.StatusView @@ -34,8 +33,7 @@ def conversation_statuses( participation = participation_id - |> Participation.get() - |> Repo.preload(:conversation) + |> Participation.get(preload: [:conversation]) if user.id == participation.user_id do activities = diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs index 7958e8e89..a27167d42 100644 --- a/test/conversation/participation_test.exs +++ b/test/conversation/participation_test.exs @@ -8,6 +8,20 @@ defmodule Pleroma.Conversation.ParticipationTest do alias Pleroma.Conversation.Participation alias Pleroma.Web.CommonAPI + test "getting a participation will also preload things" do + user = insert(:user) + other_user = insert(:user) + + {:ok, _activity} = + CommonAPI.post(user, %{"status" => "Hey @#{other_user.nickname}.", "visibility" => "direct"}) + + [participation] = Participation.for_user(user) + + participation = Participation.get(participation.id, preload: [:conversation]) + + assert %Pleroma.Conversation{} = participation.conversation + end + test "for a new conversation, it sets the recipents of the participation" do user = insert(:user) other_user = insert(:user) From 2b94ae3b3911d78ee603163f7c8aa256ed65643f Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Thu, 15 Aug 2019 01:35:29 +0300 Subject: [PATCH 51/94] Do not check if actor is active when deleting a user --- CHANGELOG.md | 1 + lib/mix/tasks/pleroma/user.ex | 2 +- lib/pleroma/user.ex | 34 +++++++++---------- lib/pleroma/web/activity_pub/activity_pub.ex | 20 +++++------ .../web/activity_pub/transmogrifier.ex | 2 +- .../web/admin_api/admin_api_controller.ex | 4 +-- .../web/ostatus/handlers/delete_handler.ex | 2 +- test/user_test.exs | 3 +- 8 files changed, 34 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dccc36965..a7b776beb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Security - OStatus: eliminate the possibility of a protocol downgrade attack. - OStatus: prevent following locked accounts, bypassing the approval process. +– ActivityPub: Do not check if actor is active when deleting a user ### Changed - **Breaking:** Configuration: A setting to explicitly disable the mailer was added, defaulting to true, if you are using a mailer add `config :pleroma, Pleroma.Emails.Mailer, enabled: true` to your config diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index f33d01429..a3f8bc945 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -176,7 +176,7 @@ def run(["rm", nickname]) do start_pleroma() with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do - User.perform(:delete, user, nil) + User.perform(:delete, user) shell_info("User #{nickname} deleted.") else _ -> diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 14057a0e4..7d18f099e 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1029,26 +1029,13 @@ def update_notification_settings(%User{} = user, settings \\ %{}) do |> update_and_set_cache() end - @spec perform(atom(), User.t()) :: {:ok, User.t()} - def perform(:fetch_initial_posts, %User{} = user) do - pages = Pleroma.Config.get!([:fetch_initial_posts, :pages]) - - Enum.each( - # Insert all the posts in reverse order, so they're in the right order on the timeline - Enum.reverse(Utils.fetch_ordered_collection(user.info.source_data["outbox"], pages)), - &Pleroma.Web.Federator.incoming_ap_doc/1 - ) - - {:ok, user} - end - @spec delete(User.t()) :: :ok - def delete(%User{} = user, actor \\ nil), - do: PleromaJobQueue.enqueue(:background, __MODULE__, [:delete, user, actor]) + def delete(%User{} = user), + do: PleromaJobQueue.enqueue(:background, __MODULE__, [:delete, user]) @spec perform(atom(), User.t()) :: {:ok, User.t()} - def perform(:delete, %User{} = user, actor) do - {:ok, _user} = ActivityPub.delete(user, actor: actor) + def perform(:delete, %User{} = user) do + {:ok, _user} = ActivityPub.delete(user) # Remove all relationships {:ok, followers} = User.get_followers(user) @@ -1070,6 +1057,19 @@ def perform(:delete, %User{} = user, actor) do Repo.delete(user) end + @spec perform(atom(), User.t()) :: {:ok, User.t()} + def perform(:fetch_initial_posts, %User{} = user) do + pages = Pleroma.Config.get!([:fetch_initial_posts, :pages]) + + Enum.each( + # Insert all the posts in reverse order, so they're in the right order on the timeline + Enum.reverse(Utils.fetch_ordered_collection(user.info.source_data["outbox"], pages)), + &Pleroma.Web.Federator.incoming_ap_doc/1 + ) + + {:ok, user} + end + def perform(:deactivate_async, user, status), do: deactivate(user, status) @spec perform(atom(), User.t(), list()) :: list() | {:error, any()} diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 8f669acb9..da873b7b0 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -61,7 +61,9 @@ defp get_recipients(data) do {recipients, to, cc} end - defp check_actor_is_active(actor) do + defp check_actor_is_active(true, _), do: :ok + + defp check_actor_is_active(false, actor) do if not is_nil(actor) do with user <- User.get_cached_by_ap_id(actor), false <- user.info.deactivated do @@ -119,10 +121,10 @@ def increase_poll_votes_if_vote(%{ def increase_poll_votes_if_vote(_create_data), do: :noop - def insert(map, local \\ true, fake \\ false) when is_map(map) do + def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when is_map(map) do with nil <- Activity.normalize(map), map <- lazy_put_activity_defaults(map, fake), - :ok <- check_actor_is_active(map["actor"]), + :ok <- check_actor_is_active(bypass_actor_check, map["actor"]), {_, true} <- {:remote_limit_error, check_remote_limit(map)}, {:ok, map} <- MRF.filter(map), {recipients, _, _} = get_recipients(map), @@ -403,22 +405,20 @@ def unfollow(follower, followed, activity_id \\ nil, local \\ true) do end end - def delete(data, opts \\ %{actor: nil, local: true}) - - def delete(%User{ap_id: ap_id, follower_address: follower_address} = user, opts) do + def delete(%User{ap_id: ap_id, follower_address: follower_address} = user) do with data <- %{ "to" => [follower_address], "type" => "Delete", - "actor" => opts[:actor] || ap_id, + "actor" => ap_id, "object" => %{"type" => "Person", "id" => ap_id} }, - {:ok, activity} <- insert(data, true, true), + {:ok, activity} <- insert(data, true, true, true), :ok <- maybe_federate(activity) do {:ok, user} end end - def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, opts) do + def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, local \\ true) do user = User.get_cached_by_ap_id(actor) to = (object.data["to"] || []) ++ (object.data["cc"] || []) @@ -430,7 +430,7 @@ def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, opts) do "to" => to, "deleted_activity_id" => activity && activity.id }, - {:ok, activity} <- insert(data, opts[:local], false), + {:ok, activity} <- insert(data, local, false), stream_out_participations(object, user), _ <- decrease_replies_count_if_reply(object), # Changing note count prior to enqueuing federation task in order to avoid diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index b34ef73c0..5403b71d8 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -649,7 +649,7 @@ def handle_incoming( {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id), :ok <- Containment.contain_origin(actor.ap_id, object.data), - {:ok, activity} <- ActivityPub.delete(object, local: false) do + {:ok, activity} <- ActivityPub.delete(object, false) do {:ok, activity} else nil -> diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 63c9a7d7f..2d3d0adc4 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -25,9 +25,9 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do action_fallback(:errors) - def user_delete(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do + def user_delete(conn, %{"nickname" => nickname}) do User.get_cached_by_nickname(nickname) - |> User.delete(admin.ap_id) + |> User.delete() conn |> json(nickname) diff --git a/lib/pleroma/web/ostatus/handlers/delete_handler.ex b/lib/pleroma/web/ostatus/handlers/delete_handler.ex index ac2dc115c..b2f9f3946 100644 --- a/lib/pleroma/web/ostatus/handlers/delete_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/delete_handler.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Web.OStatus.DeleteHandler do def handle_delete(entry, _doc \\ nil) do with id <- XML.string_from_xpath("//id", entry), %Object{} = object <- Object.normalize(id), - {:ok, delete} <- ActivityPub.delete(object, local: false) do + {:ok, delete} <- ActivityPub.delete(object, false) do delete end end diff --git a/test/user_test.exs b/test/user_test.exs index e2da8d84b..755c6005d 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -999,10 +999,9 @@ test ".delete_user_activities deletes all create activities", %{user: user} do end test "it deletes deactivated user" do - admin = insert(:user, %{info: %{is_admin: true}}) {:ok, user} = insert(:user, info: %{deactivated: true}) |> User.set_cache() - assert {:ok, _} = User.delete(user, admin.ap_id) + assert {:ok, _} = User.delete(user) refute User.get_by_id(user.id) end From b27fafe161241c954b713281bebd6ffe1e990884 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Thu, 15 Aug 2019 01:42:43 +0300 Subject: [PATCH 52/94] Fix CHANGELOG entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7b776beb..161f88176 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Security - OStatus: eliminate the possibility of a protocol downgrade attack. - OStatus: prevent following locked accounts, bypassing the approval process. -– ActivityPub: Do not check if actor is active when deleting a user ### Changed - **Breaking:** Configuration: A setting to explicitly disable the mailer was added, defaulting to true, if you are using a mailer add `config :pleroma, Pleroma.Emails.Mailer, enabled: true` to your config @@ -39,6 +38,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Invalid SemVer version generation, when the current branch does not have commits ahead of tag/checked out on a tag - Pleroma.Upload base_url was not automatically whitelisted by MediaProxy. Now your custom CDN or file hosting will be accessed directly as expected. - Report email not being sent to admins when the reporter is a remote user +- ActivityPub: Deactivated user deletion ### Added - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) From a4a3e3becd5e008dbfa9a23157ae4b16a0652bce Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 15 Aug 2019 17:37:30 +0300 Subject: [PATCH 53/94] Hide muted theads from home/public timelines unless `with_muted` is set --- lib/pleroma/activity.ex | 1 + lib/pleroma/web/activity_pub/activity_pub.ex | 20 +++++++++++------ lib/pleroma/web/streamer.ex | 6 ++--- test/web/activity_pub/activity_pub_test.exs | 23 ++++++++++++++++++++ test/web/streamer_test.exs | 20 +++++++++++++++++ 5 files changed, 60 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index baf1e7722..35612c882 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -96,6 +96,7 @@ def with_set_thread_muted_field(query, %User{} = user) do from([a] in query, left_join: tm in ThreadMute, on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data), + as: :thread_mute, select: %Activity{a | thread_muted?: not is_nil(tm.id)} ) end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index cf55c9520..defccade8 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -790,14 +790,20 @@ defp restrict_reblogs(query, _), do: query defp restrict_muted(query, %{"with_muted" => val}) when val in [true, "true", "1"], do: query - defp restrict_muted(query, %{"muting_user" => %User{info: info}}) do + defp restrict_muted(query, %{"muting_user" => %User{info: info}} = opts) do mutes = info.mutes - from( - activity in query, - where: fragment("not (? = ANY(?))", activity.actor, ^mutes), - where: fragment("not (?->'to' \\?| ?)", activity.data, ^mutes) - ) + query = + from([activity] in query, + where: fragment("not (? = ANY(?))", activity.actor, ^mutes), + where: fragment("not (?->'to' \\?| ?)", activity.data, ^mutes) + ) + + unless opts["skip_preload"] do + from([thread_mute: tm] in query, where: is_nil(tm)) + else + query + end end defp restrict_muted(query, _), do: query @@ -898,7 +904,7 @@ defp maybe_set_thread_muted_field(query, %{"skip_preload" => true}), do: query defp maybe_set_thread_muted_field(query, opts) do query - |> Activity.with_set_thread_muted_field(opts["user"]) + |> Activity.with_set_thread_muted_field(opts["muting_user"] || opts["user"]) end defp maybe_order(query, %{order: :desc}) do diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index bbaddd852..826be2f9a 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -113,8 +113,7 @@ def handle_cast( |> Map.get("#{topic}:#{item.user_id}", []) |> Enum.each(fn socket -> with %User{} = user <- User.get_cached_by_ap_id(socket.assigns[:user].ap_id), - true <- should_send?(user, item), - false <- CommonAPI.thread_muted?(user, item.activity) do + true <- should_send?(user, item) do send( socket.transport_pid, {:text, represent_notification(socket.assigns[:user], item)} @@ -236,7 +235,8 @@ defp should_send?(%User{} = user, %Activity{} = item) do %{host: parent_host} <- URI.parse(parent.data["actor"]), false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, item_host), false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, parent_host), - true <- thread_containment(item, user) do + true <- thread_containment(item, user), + false <- CommonAPI.thread_muted?(user, item) do true else _ -> false diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index d723f331f..0377d29f6 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -538,6 +538,29 @@ test "doesn't return muted activities" do assert Enum.member?(activities, activity_one) end + test "doesn't return thread muted activities" do + user = insert(:user) + activity_one = insert(:note_activity) + note_two = insert(:note, data: %{"context" => "suya.."}) + activity_two = insert(:note_activity, note: note_two) + + {:ok, _activity_two} = CommonAPI.add_mute(user, activity_two) + + assert [activity_one] = ActivityPub.fetch_activities([], %{"muting_user" => user}) + end + + test "returns thread muted activities when with_muted is set" do + user = insert(:user) + activity_one = insert(:note_activity) + note_two = insert(:note, data: %{"context" => "suya.."}) + activity_two = insert(:note_activity, note: note_two) + + {:ok, activity_two} = CommonAPI.add_mute(user, activity_two) + + assert [activity_two, activity_one] = + ActivityPub.fetch_activities([], %{"muting_user" => user, "with_muted" => true}) + end + test "does include announces on request" do activity_three = insert(:note_activity) user = insert(:user) diff --git a/test/web/streamer_test.exs b/test/web/streamer_test.exs index d47b37efb..5b7fe44d4 100644 --- a/test/web/streamer_test.exs +++ b/test/web/streamer_test.exs @@ -414,6 +414,26 @@ test "it doesn't send muted reblogs" do Task.await(task) end + test "it doesn't send posts from muted threads" do + user = insert(:user) + user2 = insert(:user) + {:ok, user2, user, _activity} = CommonAPI.follow(user2, user) + + {:ok, activity} = CommonAPI.post(user, %{"status" => "super hot take"}) + + {:ok, activity} = CommonAPI.add_mute(user2, activity) + + task = Task.async(fn -> refute_receive {:text, _}, 4_000 end) + + Streamer.add_socket( + "user", + %{transport_pid: task.pid, assigns: %{user: user2}} + ) + + Streamer.stream("user", activity) + Task.await(task) + end + describe "direct streams" do setup do GenServer.start(Streamer, %{}, name: Streamer) From 6a3b1a526eb1a3ea49aca7914ed7ba8736c52059 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 15 Aug 2019 15:34:41 -0500 Subject: [PATCH 54/94] max_body_size -> max_body_length, as it should be --- lib/pleroma/reverse_proxy/reverse_proxy.ex | 4 ++-- test/reverse_proxy_test.exs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 1f98f215c..7518e4c3c 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -109,7 +109,7 @@ def call(conn = %{method: method}, url, opts) when method in @methods do end with {:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts), - :ok <- header_length_constraint(headers, Keyword.get(opts, :max_body_length)) do + :ok <- header_length_constraint(headers, Keyword.get(opts, :max_body_length, @max_body_length)) do response(conn, client, url, code, headers, opts) else {:ok, code, headers} -> @@ -200,7 +200,7 @@ defp chunk_reply(conn, client, opts, sent_so_far, duration) do {:ok, data} <- client().stream_body(client), {:ok, duration} <- increase_read_duration(duration), sent_so_far = sent_so_far + byte_size(data), - :ok <- body_size_constraint(sent_so_far, Keyword.get(opts, :max_body_size)), + :ok <- body_size_constraint(sent_so_far, Keyword.get(opts, :max_body_length, @max_body_length)), {:ok, conn} <- chunk(conn, data) do chunk_reply(conn, client, opts, sent_so_far, duration) else diff --git a/test/reverse_proxy_test.exs b/test/reverse_proxy_test.exs index f4b7d6add..3a83c4c48 100644 --- a/test/reverse_proxy_test.exs +++ b/test/reverse_proxy_test.exs @@ -108,11 +108,11 @@ defp stream_mock(invokes, with_close? \\ false) do end end - test "max_body_size returns error if streaming body more than that option", %{conn: conn} do + test "max_body_length returns error if streaming body more than that option", %{conn: conn} do stream_mock(3, true) assert capture_log(fn -> - ReverseProxy.call(conn, "/stream-bytes/50", max_body_size: 30) + ReverseProxy.call(conn, "/stream-bytes/50", max_body_length: 30) end) =~ "[warn] Elixir.Pleroma.ReverseProxy request to /stream-bytes/50 failed while reading/chunking: :body_too_large" end From 158231cd20ceabf805ce49ae2b4d465e09e34d69 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 16 Aug 2019 18:32:25 +0700 Subject: [PATCH 55/94] Add configurable colors and logo for the digest template --- config/config.exs | 11 ++ docs/config.md | 5 + lib/pleroma/emails/user_email.ex | 88 +++++----- .../web/templates/email/digest.html.eex | 163 +++++++++--------- 4 files changed, 143 insertions(+), 124 deletions(-) diff --git a/config/config.exs b/config/config.exs index d2325edbc..4941953e5 100644 --- a/config/config.exs +++ b/config/config.exs @@ -507,6 +507,17 @@ config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Sendmail, enabled: false +config :pleroma, Pleroma.Emails.UserEmail, + logo: nil, + styling: %{ + link_color: "#d8a070", + background_color: "#2C3645", + content_background_color: "#1B2635", + header_color: "#d8a070", + text_color: "#b9b9ba", + text_muted_color: "#b9b9ba" + } + config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, path: "/api/pleroma/app_metrics" config :pleroma, Pleroma.ScheduledActivity, diff --git a/docs/config.md b/docs/config.md index 703ef67dd..8360e640e 100644 --- a/docs/config.md +++ b/docs/config.md @@ -548,6 +548,11 @@ Email notifications settings. - interval: Minimum interval between digest emails to one user - inactivity_threshold: Minimum user inactivity threshold +## Pleroma.Emails.UserEmail + +- `:logo` - a path to a custom logo. Set it to `nil` to use the default Pleroma logo. +- `:styling` - a map with color settings for email templates. + ## OAuth consumer mode OAuth consumer mode allows sign in / sign up via external OAuth providers (e.g. Twitter, Facebook, Google, Microsoft, etc.). diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index 3b5e64019..40b67ff56 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -7,21 +7,21 @@ defmodule Pleroma.Emails.UserEmail do use Phoenix.Swoosh, view: Pleroma.Web.EmailView, layout: {Pleroma.Web.LayoutView, :email} + alias Pleroma.Config + alias Pleroma.User alias Pleroma.Web.Endpoint alias Pleroma.Web.Router - defp instance_config, do: Pleroma.Config.get(:instance) - - defp instance_name, do: instance_config()[:name] + defp instance_name, do: Config.get([:instance, :name]) defp sender do - email = Keyword.get(instance_config(), :notify_email, instance_config()[:email]) + email = Config.get([:instance, :notify_email]) || Config.get([:instance, :email]) {instance_name(), email} end defp recipient(email, nil), do: email defp recipient(email, name), do: {name, email} - defp recipient(%Pleroma.User{} = user), do: recipient(user.email, user.name) + defp recipient(%User{} = user), do: recipient(user.email, user.name) def password_reset_email(user, token) when is_binary(token) do password_reset_url = Router.Helpers.reset_password_url(Endpoint, :reset, token) @@ -93,50 +93,54 @@ def account_confirmation_email(user) do Includes Mentions and New Followers data If there are no mentions (even when new followers exist), the function will return nil """ - @spec digest_email(Pleroma.User.t()) :: Swoosh.Email.t() | nil + @spec digest_email(User.t()) :: Swoosh.Email.t() | nil def digest_email(user) do - new_notifications = - Pleroma.Notification.for_user_since(user, user.last_digest_emailed_at) - |> Enum.reduce(%{followers: [], mentions: []}, fn - %{activity: %{data: %{"type" => "Create"}, actor: actor} = activity} = notification, - acc -> - new_mention = %{ - data: notification, - object: Pleroma.Object.normalize(activity), - from: Pleroma.User.get_by_ap_id(actor) - } + notifications = Pleroma.Notification.for_user_since(user, user.last_digest_emailed_at) - %{acc | mentions: [new_mention | acc.mentions]} + mentions = + notifications + |> Enum.filter(&(&1.activity.data["type"] == "Create")) + |> Enum.map(fn notification -> + object = Pleroma.Object.normalize(notification.activity) + object = update_in(object.data["content"], &format_links/1) - %{activity: %{data: %{"type" => "Follow"}, actor: actor} = activity} = notification, - acc -> - new_follower = %{ - data: notification, - object: Pleroma.Object.normalize(activity), - from: Pleroma.User.get_by_ap_id(actor) - } - - %{acc | followers: [new_follower | acc.followers]} - - _, acc -> - acc + %{ + data: notification, + object: object, + from: User.get_by_ap_id(notification.activity.actor) + } end) - with [_ | _] = mentions <- new_notifications.mentions do - mentions = - Enum.map(mentions, fn mention -> - update_in(mention.object.data["content"], &format_links/1) - end) + followers = + notifications + |> Enum.filter(&(&1.activity.data["type"] == "Follow")) + |> Enum.map(fn notification -> + %{ + data: notification, + object: Pleroma.Object.normalize(notification.activity), + from: User.get_by_ap_id(notification.activity.actor) + } + end) + + unless Enum.empty?(mentions) do + styling = Config.get([__MODULE__, :styling]) + logo = Config.get([__MODULE__, :logo]) html_data = %{ instance: instance_name(), user: user, mentions: mentions, - followers: new_notifications.followers, - unsubscribe_link: unsubscribe_url(user, "digest") + followers: followers, + unsubscribe_link: unsubscribe_url(user, "digest"), + styling: styling } - logo_path = Path.join(:code.priv_dir(:pleroma), "static/static/logo.png") + logo_path = + if is_nil(logo) do + Path.join(:code.priv_dir(:pleroma), "static/static/logo.png") + else + Path.join(Config.get([:instance, :static_dir]), logo) + end new() |> to(recipient(user)) @@ -145,17 +149,15 @@ def digest_email(user) do |> put_layout(false) |> render_body("digest.html", html_data) |> attachment(Swoosh.Attachment.new(logo_path, filename: "logo.png", type: :inline)) - else - _ -> - nil end end defp format_links(str) do re = ~r//iU + %{link_color: color} = Config.get([__MODULE__, :styling]) Regex.replace(re, str, fn link -> - String.replace(link, " user.id, "act" => %{"unsubscribe" => notifications_type}, "exp" => false} |> Pleroma.JWT.generate_and_sign!() |> Base.encode64() - Router.Helpers.subscription_url(Pleroma.Web.Endpoint, :unsubscribe, token) + Router.Helpers.subscription_url(Endpoint, :unsubscribe, token) end end diff --git a/lib/pleroma/web/templates/email/digest.html.eex b/lib/pleroma/web/templates/email/digest.html.eex index 61d57093b..860df5f9c 100644 --- a/lib/pleroma/web/templates/email/digest.html.eex +++ b/lib/pleroma/web/templates/email/digest.html.eex @@ -21,7 +21,8 @@ } a { - color: #d8a070; + + color: <%= @styling.link_color %>; text-decoration: none; } @@ -100,21 +101,21 @@ - + -