sam/pleroma
sam/pleroma
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

GTS: cherry-picks and collection usage (#186)

Browse source 
https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3725?commit_id=61254111e59f02118cad15de49d1e0704c07030e

what is this, a yoink of a yoink? good times

Co-authored-by: Hélène <pleroma-dev@helene.moe>
Co-authored-by: FloatingGhost <hannah@coffee-and-dreams.uk>
Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/186
This commit is contained in:
floatingghost 2022-08-27 18:05:48 +00:00
parent f32e288711
commit 772c209914
12 changed files with 155 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
lib/pleroma/collections/fetcher.ex
View file

@ -11,10 +11,7 @@ defmodule Akkoma.Collections.Fetcher do
alias Pleroma.Config
require Logger
def fetch_collection_by_ap_id(ap_id) when is_binary(ap_id) do
fetch_collection(ap_id)
end
@spec fetch_collection(String.t() | map()) :: {:ok, [Pleroma.Object.t()]} | {:error, any()}
def fetch_collection(ap_id) when is_binary(ap_id) do
with {:ok, page} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do
{:ok, objects_from_collection(page)}
@ -26,7 +23,7 @@ def fetch_collection(ap_id) when is_binary(ap_id) do
end
def fetch_collection(%{"type" => type} = page)
when type in ["Collection", "OrderedCollection"] do
when type in ["Collection", "OrderedCollection", "CollectionPage", "OrderedCollectionPage"] do
{:ok, objects_from_collection(page)}
end
@ -38,12 +35,13 @@ defp items_in_page(%{"type" => type, "items" => items})
when is_list(items) and type in ["Collection", "CollectionPage"],
do: items
defp objects_from_collection(%{"type" => "OrderedCollection", "orderedItems" => items})
when is_list(items),
do: items
defp objects_from_collection(%{"type" => type, "orderedItems" => items} = page)
when is_list(items) and type in ["OrderedCollection", "OrderedCollectionPage"],
do: maybe_next_page(page, items)
defp objects_from_collection(%{"type" => "Collection", "items" => items}) when is_list(items),
do: items
defp objects_from_collection(%{"type" => type, "items" => items} = page)
when is_list(items) and type in ["Collection", "CollectionPage"],
do: maybe_next_page(page, items)
defp objects_from_collection(%{"type" => type, "first" => first})
when is_binary(first) and type in ["Collection", "OrderedCollection"] do
@ -55,17 +53,27 @@ defp objects_from_collection(%{"type" => type, "first" => %{"id" => id}})
fetch_page_items(id)
end
defp objects_from_collection(_page), do: []
defp fetch_page_items(id, items \\ []) do
if Enum.count(items) >= Config.get([:activitypub, :max_collection_objects]) do
items
else
{:ok, page} = Fetcher.fetch_and_contain_remote_object_from_id(id)
objects = items_in_page(page)
with {:ok, page} <- Fetcher.fetch_and_contain_remote_object_from_id(id) do
objects = items_in_page(page)
if Enum.count(objects) > 0 do
maybe_next_page(page, items ++ objects)
if Enum.count(objects) > 0 do
maybe_next_page(page, items ++ objects)
else
items
end
else
items
{:error, "Object has been deleted"} ->
items
{:error, error} ->
Logger.error("Could not fetch page #{id} - #{inspect(error)}")
{:error, error}
end
end
end

19
lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex
View file

@ -6,7 +6,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
use Ecto.Schema
alias Pleroma.User
alias Pleroma.EctoType.ActivityPub.ObjectValidators
alias Pleroma.Object.Fetcher
alias Pleroma.Web.CommonAPI.Utils
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
@ -58,19 +57,10 @@ defp fix_tag(%{"tag" => tag} = data) when is_list(tag), do: data
defp fix_tag(%{"tag" => tag} = data) when is_map(tag), do: Map.put(data, "tag", [tag])
defp fix_tag(data), do: Map.drop(data, ["tag"])
defp fix_replies(%{"replies" => %{"first" => %{"items" => replies}}} = data)
when is_list(replies),
do: Map.put(data, "replies", replies)
defp fix_replies(%{"replies" => %{"items" => replies}} = data) when is_list(replies),
do: Map.put(data, "replies", replies)
defp fix_replies(%{"replies" => replies} = data) when is_bitstring(replies),
do: Map.drop(data, ["replies"])
defp fix_replies(%{"replies" => replies} = data) when is_list(replies), do: data
defp fix_replies(%{"replies" => %{"first" => first}} = data) do
with {:ok, %{"orderedItems" => replies}} <-
Fetcher.fetch_and_contain_remote_object_from_id(first) do
with {:ok, replies} <- Akkoma.Collections.Fetcher.fetch_collection(first) do
Map.put(data, "replies", replies)
else
{:error, _} ->
@ -79,7 +69,10 @@ defp fix_replies(%{"replies" => %{"first" => first}} = data) do
end
end
defp fix_replies(data), do: data
defp fix_replies(%{"replies" => %{"items" => replies}} = data) when is_list(replies),
do: Map.put(data, "replies", replies)
defp fix_replies(data), do: Map.delete(data, "replies")
defp remote_mention_resolver(
%{"id" => ap_id, "tag" => tags},

29
lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
View file

@ -7,8 +7,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do
alias Pleroma.Object
alias Pleroma.Object.Containment
alias Pleroma.User
alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.ActivityPub.Utils
require Pleroma.Constants
def cast_and_filter_recipients(message, field, follower_collection, field_fallback \\ []) do
{:ok, data} = ObjectValidators.Recipients.cast(message[field] || field_fallback)
@ -32,7 +32,7 @@ def fix_object_defaults(data) do
|> cast_and_filter_recipients("cc", follower_collection)
|> cast_and_filter_recipients("bto", follower_collection)
|> cast_and_filter_recipients("bcc", follower_collection)
|> Transmogrifier.fix_implicit_addressing(follower_collection)
|> fix_implicit_addressing(follower_collection)
end
def fix_activity_addressing(activity) do
@ -43,7 +43,7 @@ def fix_activity_addressing(activity) do
|> cast_and_filter_recipients("cc", follower_collection)
|> cast_and_filter_recipients("bto", follower_collection)
|> cast_and_filter_recipients("bcc", follower_collection)
|> Transmogrifier.fix_implicit_addressing(follower_collection)
|> fix_implicit_addressing(follower_collection)
end
def fix_actor(data) do
@ -73,4 +73,27 @@ def fix_object_action_recipients(data, %Object{data: %{"actor" => actor}}) do
Map.put(data, "to", to)
end
# if as:Public is addressed, then make sure the followers collection is also addressed
# so that the activities will be delivered to local users.
def fix_implicit_addressing(%{"to" => to, "cc" => cc} = object, followers_collection) do
recipients = to ++ cc
if followers_collection not in recipients do
cond do
Pleroma.Constants.as_public() in cc ->
to = to ++ [followers_collection]
Map.put(object, "to", to)
Pleroma.Constants.as_public() in to ->
cc = cc ++ [followers_collection]
Map.put(object, "cc", cc)
true ->
object
end
else
object
end
end
end

3
lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
View file

@ -13,7 +13,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
alias Pleroma.User
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
alias Pleroma.Web.ActivityPub.Transmogrifier
import Ecto.Changeset
@ -67,7 +66,7 @@ defp fix_addressing(data, object) do
|> CommonFixes.cast_and_filter_recipients("cc", follower_collection, object["cc"])
|> CommonFixes.cast_and_filter_recipients("bto", follower_collection, object["bto"])
|> CommonFixes.cast_and_filter_recipients("bcc", follower_collection, object["bcc"])
|> Transmogrifier.fix_implicit_addressing(follower_collection)
|> CommonFixes.fix_implicit_addressing(follower_collection)
end
def fix(data, meta) do

26
lib/pleroma/web/activity_pub/transmogrifier.ex
View file

@ -19,6 +19,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
alias Pleroma.Web.ActivityPub.Pipeline
alias Pleroma.Web.ActivityPub.Utils
alias Pleroma.Web.ActivityPub.Visibility
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
alias Pleroma.Web.Federator
alias Pleroma.Workers.TransmogrifierWorker
@ -95,29 +96,6 @@ def fix_explicit_addressing(%{"to" => to, "cc" => cc} = object, follower_collect
|> Map.put("cc", final_cc)
end
# if as:Public is addressed, then make sure the followers collection is also addressed
# so that the activities will be delivered to local users.
def fix_implicit_addressing(%{"to" => to, "cc" => cc} = object, followers_collection) do
recipients = to ++ cc
if followers_collection not in recipients do
cond do
Pleroma.Constants.as_public() in cc ->
to = to ++ [followers_collection]
Map.put(object, "to", to)
Pleroma.Constants.as_public() in to ->
cc = cc ++ [followers_collection]
Map.put(object, "cc", cc)
true ->
object
end
else
object
end
end
def fix_addressing(object) do
{:ok, %User{follower_address: follower_collection}} =
object
@ -130,7 +108,7 @@ def fix_addressing(object) do
|> fix_addressing_list("bto")
|> fix_addressing_list("bcc")
|> fix_explicit_addressing(follower_collection)
|> fix_implicit_addressing(follower_collection)
|> CommonFixes.fix_implicit_addressing(follower_collection)
end
def fix_actor(%{"attributedTo" => actor} = object) do

8
lib/pleroma/web/plugs/http_signature_plug.ex
View file

@ -27,11 +27,11 @@ def call(conn, _opts) do
end
end
def route_aliases(%{path_info: ["objects", id]}) do
def route_aliases(%{path_info: ["objects", id], query_string: query_string}) do
ap_id = Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :object, id)
with %Activity{} = activity <- Activity.get_by_object_ap_id_with_object(ap_id) do
["/notice/#{activity.id}"]
["/notice/#{activity.id}", "/notice/#{activity.id}?#{query_string}"]
else
_ -> []
end
@ -64,7 +64,9 @@ defp maybe_assign_valid_signature(conn) do
if has_signature_header?(conn) do
# set (request-target) header to the appropriate value
# we also replace the digest header with the one we computed
possible_paths = route_aliases(conn) ++ [conn.request_path]
possible_paths =
route_aliases(conn) ++ [conn.request_path, conn.request_path <> "?#{conn.query_string}"]
assign_valid_signature_on_route_aliases(conn, possible_paths)
else
Logger.debug("No signature header!")

59
test/pleroma/collections/collections_fetcher_test.exs
View file

@ -30,7 +30,7 @@ test "it should extract items from an embedded array in a Collection" do
}
end)
{:ok, objects} = Fetcher.fetch_collection_by_ap_id(ap_id)
{:ok, objects} = Fetcher.fetch_collection(ap_id)
assert [%{"type" => "Create"}, %{"type" => "Like"}] = objects
end
@ -53,7 +53,7 @@ test "it should extract items from an embedded array in an OrderedCollection" do
}
end)
{:ok, objects} = Fetcher.fetch_collection_by_ap_id(ap_id)
{:ok, objects} = Fetcher.fetch_collection(ap_id)
assert [%{"type" => "Create"}, %{"type" => "Like"}] = objects
end
@ -106,7 +106,7 @@ test "it should extract items from an referenced first page in a Collection" do
}
end)
{:ok, objects} = Fetcher.fetch_collection_by_ap_id(ap_id)
{:ok, objects} = Fetcher.fetch_collection(ap_id)
assert [%{"type" => "Create"}, %{"type" => "Like"}] = objects
end
@ -161,7 +161,58 @@ test "it should stop fetching when we hit :max_collection_objects" do
}
end)
{:ok, objects} = Fetcher.fetch_collection_by_ap_id(ap_id)
{:ok, objects} = Fetcher.fetch_collection(ap_id)
assert [%{"type" => "Create"}] = objects
end
test "it should stop fetching when we hit a 404" do
clear_config([:activitypub, :max_collection_objects], 1)
unordered_collection =
"test/fixtures/collections/unordered_page_reference.json"
|> File.read!()
first_page =
"test/fixtures/collections/unordered_page_first.json"
|> File.read!()
ap_id = "https://example.com/collection/unordered_page_reference"
first_page_id = "https://example.com/collection/unordered_page_reference?page=1"
second_page_id = "https://example.com/collection/unordered_page_reference?page=2"
Tesla.Mock.mock(fn
%{
method: :get,
url: ^ap_id
} ->
%Tesla.Env{
status: 200,
body: unordered_collection,
headers: [{"content-type", "application/activity+json"}]
}
%{
method: :get,
url: ^first_page_id
} ->
%Tesla.Env{
status: 200,
body: first_page,
headers: [{"content-type", "application/activity+json"}]
}
%{
method: :get,
url: ^second_page_id
} ->
%Tesla.Env{
status: 404,
body: nil,
headers: [{"content-type", "application/activity+json"}]
}
end)
{:ok, objects} = Fetcher.fetch_collection(ap_id)
assert [%{"type" => "Create"}] = objects
end
end

11
test/pleroma/web/activity_pub/activity_pub_controller_test.exs
View file

@ -782,6 +782,7 @@ test "mastodon pin/unpin", %{conn: conn} do
|> String.replace("{{status_id}}", status_id)
status_url = "https://example.com/users/lain/statuses/#{status_id}"
replies_url = status_url <> "/replies?only_other_accounts=true&page=true"
user =
File.read!("test/fixtures/users_mock/user.json")
@ -820,6 +821,16 @@ test "mastodon pin/unpin", %{conn: conn} do
|> String.replace("{{nickname}}", "lain"),
headers: [{"content-type", "application/activity+json"}]
}
%{
method: :get,
url: ^replies_url
} ->
%Tesla.Env{
status: 404,
body: "",
headers: [{"content-type", "application/activity+json"}]
}
end)
data = %{

13
test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs
View file

@ -146,4 +146,17 @@ test "a misskey MFM status with a _misskey_content field should work and be link
"<span class=\"h-card\"><a class=\"u-url mention\" data-user=\"#{local_user.id}\" href=\"#{local_user.ap_id}\" rel=\"ugc\">@<span>akkoma_user</span></a></span>"
end
end
test "a Note without replies/first/items validates" do
insert(:user, ap_id: "https://mastodon.social/users/emelie")
note =
"test/fixtures/tesla_mock/status.emelie.json"
|> File.read!()
|> Jason.decode!()
|> pop_in(["replies", "first", "items"])
|> elem(1)
%{valid?: true} = ArticleNotePageValidator.cast_and_validate(note)
end
end

1
test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs
View file

@ -380,7 +380,6 @@ test "schedules background fetching of `replies` items if max thread depth limit
clear_config([:instance, :federation_incoming_replies_max_depth], 10)
{:ok, activity} = Transmogrifier.handle_incoming(data)
object = Object.normalize(activity.data["object"])
assert object.data["replies"] == items

6
test/pleroma/web/plugs/http_signature_plug_test.exs
View file

@ -86,10 +86,12 @@ test "halts the connection when `signature` header is not present", %{conn: conn
test "aliases redirected /object endpoints", _ do
obj = insert(:note)
act = insert(:note_activity, note: obj)
params = %{"actor" => "http://mastodon.example.org/users/admin"}
params = %{"actor" => "someparam"}
path = URI.parse(obj.data["id"]).path
conn = build_conn(:get, path, params)
assert ["/notice/#{act.id}"] == HTTPSignaturePlug.route_aliases(conn)
assert ["/notice/#{act.id}", "/notice/#{act.id}?actor=someparam"] ==
HTTPSignaturePlug.route_aliases(conn)
end
end
end

9
test/support/http_request_mock.ex
View file

@ -407,6 +407,15 @@ def get("http://mastodon.example.org/users/admin", _, _, _) do
}}
end
def get(
"http://mastodon.example.org/users/admin/statuses/99512778738411822/replies?min_id=99512778738411824&page=true",
_,
_,
_
) do
{:ok, %Tesla.Env{status: 404, body: ""}}
end
def get("http://mastodon.example.org/users/relay", _, _, [
{"accept", "application/activity+json"}
]) do