Merge branch 'reply-filtering-refactor' into 'develop'

Reply filtering refactor

See merge request pleroma/pleroma!2424
This commit is contained in:
lain 2020-04-24 16:32:13 +00:00
commit c30b3b2dc5
11 changed files with 617 additions and 11 deletions

View file

@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
<summary>API Changes</summary>
- Mastodon API: Support for `include_types` in `/api/v1/notifications`.
- Mastodon API: Added `/api/v1/notifications/:id/dismiss` endpoint.
- Mastodon API: Add support for filtering replies in public and home timelines
- Admin API: endpoints for create/update/delete OAuth Apps.
</details>

View file

@ -279,7 +279,7 @@ defp insert_activity("like", visibility, group, user, friends, non_friends, opts
actor = get_actor(group, user, friends, non_friends)
with activity_id when not is_nil(activity_id) <- get_random_create_activity_id(),
{:ok, _activity, _object} <- CommonAPI.favorite(activity_id, actor) do
{:ok, _activity} <- CommonAPI.favorite(actor, activity_id) do
:ok
else
{:error, _} ->
@ -313,7 +313,7 @@ defp insert_activity("simple_thread", visibility, group, user, friends, non_frie
tasks = get_reply_tasks(visibility, group)
{:ok, activity} =
CommonAPI.post(user, %{"status" => "Simple status", "visibility" => "unlisted"})
CommonAPI.post(user, %{"status" => "Simple status", "visibility" => visibility})
acc = {activity.id, ["@" <> actor.nickname, "reply to status"]}
insert_replies(tasks, visibility, user, friends, non_friends, acc)

View file

@ -41,6 +41,7 @@ defp fetch_timelines(user) do
fetch_notifications(user)
fetch_favourites(user)
fetch_long_thread(user)
fetch_timelines_with_reply_filtering(user)
end
defp render_views(user) do
@ -495,4 +496,58 @@ defp render_long_thread(user) do
formatters: formatters()
)
end
defp fetch_timelines_with_reply_filtering(user) do
public_params = opts_for_public_timeline(user)
Benchee.run(
%{
"Public timeline without reply filtering" => fn ->
ActivityPub.fetch_public_activities(public_params)
end,
"Public timeline with reply filtering - following" => fn ->
public_params
|> Map.put("reply_visibility", "following")
|> Map.put("reply_filtering_user", user)
|> ActivityPub.fetch_public_activities()
end,
"Public timeline with reply filtering - self" => fn ->
public_params
|> Map.put("reply_visibility", "self")
|> Map.put("reply_filtering_user", user)
|> ActivityPub.fetch_public_activities()
end
},
formatters: formatters()
)
private_params = opts_for_home_timeline(user)
recipients = [user.ap_id | User.following(user)]
Benchee.run(
%{
"Home timeline without reply filtering" => fn ->
ActivityPub.fetch_activities(recipients, private_params)
end,
"Home timeline with reply filtering - following" => fn ->
private_params =
private_params
|> Map.put("reply_filtering_user", user)
|> Map.put("reply_visibility", "following")
ActivityPub.fetch_activities(recipients, private_params)
end,
"Home timeline with reply filtering - self" => fn ->
private_params =
private_params
|> Map.put("reply_filtering_user", user)
|> Map.put("reply_visibility", "self")
ActivityPub.fetch_activities(recipients, private_params)
end
},
formatters: formatters()
)
end
end

View file

@ -44,6 +44,7 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
]
def run(args) do
Logger.configure(level: :error)
Mix.Pleroma.start_pleroma()
clean_tables()
{opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)

View file

@ -14,6 +14,7 @@ Some apps operate under the assumption that no more than 4 attachments can be re
Adding the parameter `with_muted=true` to the timeline queries will also return activities by muted (not by blocked!) users.
Adding the parameter `exclude_visibilities` to the timeline queries will exclude the statuses with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`), e.g., `exclude_visibilities[]=direct&exclude_visibilities[]=private`.
Adding the parameter `reply_visibility` to the public and home timelines queries will filter replies. Possible values: without parameter (default) shows all replies, `following` - replies directed to you or users you follow, `self` - replies directed to you.
## Statuses

View file

@ -832,6 +832,7 @@ def set_cache({:error, err}), do: {:error, err}
def set_cache(%User{} = user) do
Cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
Cachex.put(:user_cache, "nickname:#{user.nickname}", user)
Cachex.put(:user_cache, "friends_ap_ids:#{user.nickname}", get_user_friends_ap_ids(user))
{:ok, user}
end
@ -847,9 +848,22 @@ def update_and_set_cache(changeset) do
end
end
def get_user_friends_ap_ids(user) do
from(u in User.get_friends_query(user), select: u.ap_id)
|> Repo.all()
end
@spec get_cached_user_friends_ap_ids(User.t()) :: [String.t()]
def get_cached_user_friends_ap_ids(user) do
Cachex.fetch!(:user_cache, "friends_ap_ids:#{user.ap_id}", fn _ ->
get_user_friends_ap_ids(user)
end)
end
def invalidate_cache(user) do
Cachex.del(:user_cache, "ap_id:#{user.ap_id}")
Cachex.del(:user_cache, "nickname:#{user.nickname}")
Cachex.del(:user_cache, "friends_ap_ids:#{user.ap_id}")
end
@spec get_cached_by_ap_id(String.t()) :: User.t() | nil

View file

@ -54,13 +54,13 @@ defmodule Pleroma.User.Query do
select: term(),
limit: pos_integer()
}
| %{}
| map()
@ilike_criteria [:nickname, :name, :query]
@equal_criteria [:email]
@contains_criteria [:ap_id, :nickname]
@spec build(criteria()) :: Query.t()
@spec build(Query.t(), criteria()) :: Query.t()
def build(query \\ base_query(), criteria) do
prepare_query(query, criteria)
end

View file

@ -1047,6 +1047,41 @@ defp restrict_replies(query, %{"exclude_replies" => val}) when val == "true" or
)
end
defp restrict_replies(query, %{
"reply_filtering_user" => user,
"reply_visibility" => "self"
}) do
from(
[activity, object] in query,
where:
fragment(
"?->>'inReplyTo' is null OR ? = ANY(?)",
object.data,
^user.ap_id,
activity.recipients
)
)
end
defp restrict_replies(query, %{
"reply_filtering_user" => user,
"reply_visibility" => "following"
}) do
from(
[activity, object] in query,
where:
fragment(
"?->>'inReplyTo' is null OR ? && array_remove(?, ?) OR ? = ?",
object.data,
^[user.ap_id | User.get_cached_user_friends_ap_ids(user)],
activity.recipients,
activity.actor,
activity.actor,
^user.ap_id
)
)
end
defp restrict_replies(query, _), do: query
defp restrict_reblogs(query, %{"exclude_reblogs" => val}) when val == "true" or val == "1" do
@ -1261,6 +1296,7 @@ def fetch_activities_query(recipients, opts \\ %{}) do
|> maybe_set_thread_muted_field(opts)
|> maybe_order(opts)
|> restrict_recipients(recipients, opts["user"])
|> restrict_replies(opts)
|> restrict_tag(opts)
|> restrict_tag_reject(opts)
|> restrict_tag_all(opts)
@ -1275,7 +1311,6 @@ def fetch_activities_query(recipients, opts \\ %{}) do
|> restrict_media(opts)
|> restrict_visibility(opts)
|> restrict_thread_visibility(opts, config)
|> restrict_replies(opts)
|> restrict_reblogs(opts)
|> restrict_pinned(opts)
|> restrict_muted_reblogs(restrict_muted_reblogs_opts)

View file

@ -84,14 +84,18 @@ defp attachments(%{params: params} = draft) do
%__MODULE__{draft | attachments: attachments}
end
defp in_reply_to(draft) do
case Map.get(draft.params, "in_reply_to_status_id") do
"" -> draft
nil -> draft
id -> %__MODULE__{draft | in_reply_to: Activity.get_by_id(id)}
end
defp in_reply_to(%{params: %{"in_reply_to_status_id" => ""}} = draft), do: draft
defp in_reply_to(%{params: %{"in_reply_to_status_id" => id}} = draft) when is_binary(id) do
%__MODULE__{draft | in_reply_to: Activity.get_by_id(id)}
end
defp in_reply_to(%{params: %{"in_reply_to_status_id" => %Activity{} = in_reply_to}} = draft) do
%__MODULE__{draft | in_reply_to: in_reply_to}
end
defp in_reply_to(draft), do: draft
defp in_reply_to_conversation(draft) do
in_reply_to_conversation = Participation.get(draft.params["in_reply_to_conversation_id"])
%__MODULE__{draft | in_reply_to_conversation: in_reply_to_conversation}

View file

@ -37,6 +37,7 @@ def home(%{assigns: %{user: user}} = conn, params) do
|> Map.put("type", ["Create", "Announce"])
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("reply_filtering_user", user)
|> Map.put("user", user)
recipients = [user.ap_id | User.following(user)]
@ -100,6 +101,7 @@ def public(%{assigns: %{user: user}} = conn, params) do
|> Map.put("local_only", local_only)
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("reply_filtering_user", user)
|> ActivityPub.fetch_public_activities()
conn

View file

@ -1910,4 +1910,497 @@ test "old user must be in the new user's `also_known_as` list" do
ActivityPub.move(old_user, new_user)
end
end
test "doesn't retrieve replies activities with exclude_replies" do
user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{"status" => "yeah"})
{:ok, _reply} =
CommonAPI.post(user, %{"status" => "yeah", "in_reply_to_status_id" => activity.id})
[result] = ActivityPub.fetch_public_activities(%{"exclude_replies" => "true"})
assert result.id == activity.id
assert length(ActivityPub.fetch_public_activities()) == 2
end
describe "replies filtering with public messages" do
setup :public_messages
test "public timeline", %{users: %{u1: user}} do
activities_ids =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("local_only", false)
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("reply_filtering_user", user)
|> ActivityPub.fetch_public_activities()
|> Enum.map(& &1.id)
assert length(activities_ids) == 16
end
test "public timeline with reply_visibility `following`", %{
users: %{u1: user},
u1: u1,
u2: u2,
u3: u3,
u4: u4,
activities: activities
} do
activities_ids =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("local_only", false)
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("reply_visibility", "following")
|> Map.put("reply_filtering_user", user)
|> ActivityPub.fetch_public_activities()
|> Enum.map(& &1.id)
assert length(activities_ids) == 14
visible_ids =
Map.values(u1) ++ Map.values(u2) ++ Map.values(u4) ++ Map.values(activities) ++ [u3[:r1]]
assert Enum.all?(visible_ids, &(&1 in activities_ids))
end
test "public timeline with reply_visibility `self`", %{
users: %{u1: user},
u1: u1,
u2: u2,
u3: u3,
u4: u4,
activities: activities
} do
activities_ids =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("local_only", false)
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("reply_visibility", "self")
|> Map.put("reply_filtering_user", user)
|> ActivityPub.fetch_public_activities()
|> Enum.map(& &1.id)
assert length(activities_ids) == 10
visible_ids = Map.values(u1) ++ [u2[:r1], u3[:r1], u4[:r1]] ++ Map.values(activities)
assert Enum.all?(visible_ids, &(&1 in activities_ids))
end
test "home timeline", %{
users: %{u1: user},
activities: activities,
u1: u1,
u2: u2,
u3: u3,
u4: u4
} do
params =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("user", user)
|> Map.put("reply_filtering_user", user)
activities_ids =
ActivityPub.fetch_activities([user.ap_id | User.following(user)], params)
|> Enum.map(& &1.id)
assert length(activities_ids) == 13
visible_ids =
Map.values(u1) ++
Map.values(u3) ++
[
activities[:a1],
activities[:a2],
activities[:a4],
u2[:r1],
u2[:r3],
u4[:r1],
u4[:r2]
]
assert Enum.all?(visible_ids, &(&1 in activities_ids))
end
test "home timeline with reply_visibility `following`", %{
users: %{u1: user},
activities: activities,
u1: u1,
u2: u2,
u3: u3,
u4: u4
} do
params =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("user", user)
|> Map.put("reply_visibility", "following")
|> Map.put("reply_filtering_user", user)
activities_ids =
ActivityPub.fetch_activities([user.ap_id | User.following(user)], params)
|> Enum.map(& &1.id)
assert length(activities_ids) == 11
visible_ids =
Map.values(u1) ++
[
activities[:a1],
activities[:a2],
activities[:a4],
u2[:r1],
u2[:r3],
u3[:r1],
u4[:r1],
u4[:r2]
]
assert Enum.all?(visible_ids, &(&1 in activities_ids))
end
test "home timeline with reply_visibility `self`", %{
users: %{u1: user},
activities: activities,
u1: u1,
u2: u2,
u3: u3,
u4: u4
} do
params =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("user", user)
|> Map.put("reply_visibility", "self")
|> Map.put("reply_filtering_user", user)
activities_ids =
ActivityPub.fetch_activities([user.ap_id | User.following(user)], params)
|> Enum.map(& &1.id)
assert length(activities_ids) == 9
visible_ids =
Map.values(u1) ++
[
activities[:a1],
activities[:a2],
activities[:a4],
u2[:r1],
u3[:r1],
u4[:r1]
]
assert Enum.all?(visible_ids, &(&1 in activities_ids))
end
end
describe "replies filtering with private messages" do
setup :private_messages
test "public timeline", %{users: %{u1: user}} do
activities_ids =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("local_only", false)
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("user", user)
|> ActivityPub.fetch_public_activities()
|> Enum.map(& &1.id)
assert activities_ids == []
end
test "public timeline with default reply_visibility `following`", %{users: %{u1: user}} do
activities_ids =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("local_only", false)
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("reply_visibility", "following")
|> Map.put("reply_filtering_user", user)
|> Map.put("user", user)
|> ActivityPub.fetch_public_activities()
|> Enum.map(& &1.id)
assert activities_ids == []
end
test "public timeline with default reply_visibility `self`", %{users: %{u1: user}} do
activities_ids =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("local_only", false)
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("reply_visibility", "self")
|> Map.put("reply_filtering_user", user)
|> Map.put("user", user)
|> ActivityPub.fetch_public_activities()
|> Enum.map(& &1.id)
assert activities_ids == []
end
test "home timeline", %{users: %{u1: user}} do
params =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("user", user)
activities_ids =
ActivityPub.fetch_activities([user.ap_id | User.following(user)], params)
|> Enum.map(& &1.id)
assert length(activities_ids) == 12
end
test "home timeline with default reply_visibility `following`", %{users: %{u1: user}} do
params =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("user", user)
|> Map.put("reply_visibility", "following")
|> Map.put("reply_filtering_user", user)
activities_ids =
ActivityPub.fetch_activities([user.ap_id | User.following(user)], params)
|> Enum.map(& &1.id)
assert length(activities_ids) == 12
end
test "home timeline with default reply_visibility `self`", %{
users: %{u1: user},
activities: activities,
u1: u1,
u2: u2,
u3: u3,
u4: u4
} do
params =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("user", user)
|> Map.put("reply_visibility", "self")
|> Map.put("reply_filtering_user", user)
activities_ids =
ActivityPub.fetch_activities([user.ap_id | User.following(user)], params)
|> Enum.map(& &1.id)
assert length(activities_ids) == 10
visible_ids =
Map.values(u1) ++ Map.values(u4) ++ [u2[:r1], u3[:r1]] ++ Map.values(activities)
assert Enum.all?(visible_ids, &(&1 in activities_ids))
end
end
defp public_messages(_) do
[u1, u2, u3, u4] = insert_list(4, :user)
{:ok, u1} = User.follow(u1, u2)
{:ok, u2} = User.follow(u2, u1)
{:ok, u1} = User.follow(u1, u4)
{:ok, u4} = User.follow(u4, u1)
{:ok, u2} = User.follow(u2, u3)
{:ok, u3} = User.follow(u3, u2)
{:ok, a1} = CommonAPI.post(u1, %{"status" => "Status"})
{:ok, r1_1} =
CommonAPI.post(u2, %{
"status" => "@#{u1.nickname} reply from u2 to u1",
"in_reply_to_status_id" => a1.id
})
{:ok, r1_2} =
CommonAPI.post(u3, %{
"status" => "@#{u1.nickname} reply from u3 to u1",
"in_reply_to_status_id" => a1.id
})
{:ok, r1_3} =
CommonAPI.post(u4, %{
"status" => "@#{u1.nickname} reply from u4 to u1",
"in_reply_to_status_id" => a1.id
})
{:ok, a2} = CommonAPI.post(u2, %{"status" => "Status"})
{:ok, r2_1} =
CommonAPI.post(u1, %{
"status" => "@#{u2.nickname} reply from u1 to u2",
"in_reply_to_status_id" => a2.id
})
{:ok, r2_2} =
CommonAPI.post(u3, %{
"status" => "@#{u2.nickname} reply from u3 to u2",
"in_reply_to_status_id" => a2.id
})
{:ok, r2_3} =
CommonAPI.post(u4, %{
"status" => "@#{u2.nickname} reply from u4 to u2",
"in_reply_to_status_id" => a2.id
})
{:ok, a3} = CommonAPI.post(u3, %{"status" => "Status"})
{:ok, r3_1} =
CommonAPI.post(u1, %{
"status" => "@#{u3.nickname} reply from u1 to u3",
"in_reply_to_status_id" => a3.id
})
{:ok, r3_2} =
CommonAPI.post(u2, %{
"status" => "@#{u3.nickname} reply from u2 to u3",
"in_reply_to_status_id" => a3.id
})
{:ok, r3_3} =
CommonAPI.post(u4, %{
"status" => "@#{u3.nickname} reply from u4 to u3",
"in_reply_to_status_id" => a3.id
})
{:ok, a4} = CommonAPI.post(u4, %{"status" => "Status"})
{:ok, r4_1} =
CommonAPI.post(u1, %{
"status" => "@#{u4.nickname} reply from u1 to u4",
"in_reply_to_status_id" => a4.id
})
{:ok, r4_2} =
CommonAPI.post(u2, %{
"status" => "@#{u4.nickname} reply from u2 to u4",
"in_reply_to_status_id" => a4.id
})
{:ok, r4_3} =
CommonAPI.post(u3, %{
"status" => "@#{u4.nickname} reply from u3 to u4",
"in_reply_to_status_id" => a4.id
})
{:ok,
users: %{u1: u1, u2: u2, u3: u3, u4: u4},
activities: %{a1: a1.id, a2: a2.id, a3: a3.id, a4: a4.id},
u1: %{r1: r1_1.id, r2: r1_2.id, r3: r1_3.id},
u2: %{r1: r2_1.id, r2: r2_2.id, r3: r2_3.id},
u3: %{r1: r3_1.id, r2: r3_2.id, r3: r3_3.id},
u4: %{r1: r4_1.id, r2: r4_2.id, r3: r4_3.id}}
end
defp private_messages(_) do
[u1, u2, u3, u4] = insert_list(4, :user)
{:ok, u1} = User.follow(u1, u2)
{:ok, u2} = User.follow(u2, u1)
{:ok, u1} = User.follow(u1, u3)
{:ok, u3} = User.follow(u3, u1)
{:ok, u1} = User.follow(u1, u4)
{:ok, u4} = User.follow(u4, u1)
{:ok, u2} = User.follow(u2, u3)
{:ok, u3} = User.follow(u3, u2)
{:ok, a1} = CommonAPI.post(u1, %{"status" => "Status", "visibility" => "private"})
{:ok, r1_1} =
CommonAPI.post(u2, %{
"status" => "@#{u1.nickname} reply from u2 to u1",
"in_reply_to_status_id" => a1.id,
"visibility" => "private"
})
{:ok, r1_2} =
CommonAPI.post(u3, %{
"status" => "@#{u1.nickname} reply from u3 to u1",
"in_reply_to_status_id" => a1.id,
"visibility" => "private"
})
{:ok, r1_3} =
CommonAPI.post(u4, %{
"status" => "@#{u1.nickname} reply from u4 to u1",
"in_reply_to_status_id" => a1.id,
"visibility" => "private"
})
{:ok, a2} = CommonAPI.post(u2, %{"status" => "Status", "visibility" => "private"})
{:ok, r2_1} =
CommonAPI.post(u1, %{
"status" => "@#{u2.nickname} reply from u1 to u2",
"in_reply_to_status_id" => a2.id,
"visibility" => "private"
})
{:ok, r2_2} =
CommonAPI.post(u3, %{
"status" => "@#{u2.nickname} reply from u3 to u2",
"in_reply_to_status_id" => a2.id,
"visibility" => "private"
})
{:ok, a3} = CommonAPI.post(u3, %{"status" => "Status", "visibility" => "private"})
{:ok, r3_1} =
CommonAPI.post(u1, %{
"status" => "@#{u3.nickname} reply from u1 to u3",
"in_reply_to_status_id" => a3.id,
"visibility" => "private"
})
{:ok, r3_2} =
CommonAPI.post(u2, %{
"status" => "@#{u3.nickname} reply from u2 to u3",
"in_reply_to_status_id" => a3.id,
"visibility" => "private"
})
{:ok, a4} = CommonAPI.post(u4, %{"status" => "Status", "visibility" => "private"})
{:ok, r4_1} =
CommonAPI.post(u1, %{
"status" => "@#{u4.nickname} reply from u1 to u4",
"in_reply_to_status_id" => a4.id,
"visibility" => "private"
})
{:ok,
users: %{u1: u1, u2: u2, u3: u3, u4: u4},
activities: %{a1: a1.id, a2: a2.id, a3: a3.id, a4: a4.id},
u1: %{r1: r1_1.id, r2: r1_2.id, r3: r1_3.id},
u2: %{r1: r2_1.id, r2: r2_2.id},
u3: %{r1: r3_1.id, r2: r3_2.id},
u4: %{r1: r4_1.id}}
end
end