sam/pleroma
sam/pleroma
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'fix/federation-context-issues' into 'develop'

Browse source 
Fix reply context fixing (Pleroma replies to Misskey threads) and removal of context objects

See merge request pleroma/pleroma!3717
This commit is contained in:
tusooa 2022-09-04 18:43:36 +00:00
commit 20347898e2
33 changed files with 585 additions and 120 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
config/config.exs
View file

@ -673,6 +673,8 @@
config :pleroma, :populate_hashtags_table, fault_rate_allowance: 0.01
config :pleroma, :delete_context_objects, fault_rate_allowance: 0.01
config :pleroma, :env, Mix.env()
config :http_signatures,

21
config/description.exs
View file

@ -495,6 +495,27 @@
}
]
},
%{
group: :pleroma,
key: :delete_context_objects,
type: :group,
description: "`delete_context_objects` background migration settings",
children: [
%{
key: :fault_rate_allowance,
type: :float,
description:
"Max accepted rate of objects that failed in the migration. Any value from 0.0 which tolerates no errors to 1.0 which will enable the feature even if context object deletion failed for all records.",
suggestions: [0.01]
},
%{
key: :sleep_interval_ms,
type: :integer,
description:
"Sleep interval between each chunk of processed records in order to decrease the load on the system (defaults to 0 and should be keep default on most instances)."
}
]
},
%{
group: :pleroma,
key: :instance,

3
lib/pleroma/application.ex
View file

@ -249,7 +249,8 @@ defp dont_run_in_test(_) do
defp background_migrators do
[
Pleroma.Migrators.HashtagsTableMigrator
Pleroma.Migrators.HashtagsTableMigrator,
Pleroma.Migrators.ContextObjectsDeletionMigrator
]
end

1
lib/pleroma/data_migration.ex
View file

@ -42,4 +42,5 @@ def get_by_name(name) do
end
def populate_hashtags_table, do: get_by_name("populate_hashtags_table")
def delete_context_objects, do: get_by_name("delete_context_objects")
end

139
lib/pleroma/migrators/context_objects_deletion_migrator.ex Normal file
View file

@ -0,0 +1,139 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Migrators.ContextObjectsDeletionMigrator do
defmodule State do
use Pleroma.Migrators.Support.BaseMigratorState
@impl Pleroma.Migrators.Support.BaseMigratorState
defdelegate data_migration(), to: Pleroma.DataMigration, as: :delete_context_objects
end
use Pleroma.Migrators.Support.BaseMigrator
alias Pleroma.Migrators.Support.BaseMigrator
alias Pleroma.Object
@doc "This migration removes objects created exclusively for contexts, containing only an `id` field."
@impl BaseMigrator
def feature_config_path, do: [:features, :delete_context_objects]
@impl BaseMigrator
def fault_rate_allowance, do: Config.get([:delete_context_objects, :fault_rate_allowance], 0)
@impl BaseMigrator
def perform do
data_migration_id = data_migration_id()
max_processed_id = get_stat(:max_processed_id, 0)
Logger.info("Deleting context objects from `objects` (from oid: #{max_processed_id})...")
query()
|> where([object], object.id > ^max_processed_id)
|> Repo.chunk_stream(100, :batches, timeout: :infinity)
|> Stream.each(fn objects ->
object_ids = Enum.map(objects, & &1.id)
results = Enum.map(object_ids, &delete_context_object(&1))
failed_ids =
results
|> Enum.filter(&(elem(&1, 0) == :error))
|> Enum.map(&elem(&1, 1))
chunk_affected_count =
results
|> Enum.filter(&(elem(&1, 0) == :ok))
|> length()
for failed_id <- failed_ids do
_ =
Repo.query(
"INSERT INTO data_migration_failed_ids(data_migration_id, record_id) " <>
"VALUES ($1, $2) ON CONFLICT DO NOTHING;",
[data_migration_id, failed_id]
)
end
_ =
Repo.query(
"DELETE FROM data_migration_failed_ids " <>
"WHERE data_migration_id = $1 AND record_id = ANY($2)",
[data_migration_id, object_ids -- failed_ids]
)
max_object_id = Enum.at(object_ids, -1)
put_stat(:max_processed_id, max_object_id)
increment_stat(:iteration_processed_count, length(object_ids))
increment_stat(:processed_count, length(object_ids))
increment_stat(:failed_count, length(failed_ids))
increment_stat(:affected_count, chunk_affected_count)
put_stat(:records_per_second, records_per_second())
persist_state()
# A quick and dirty approach to controlling the load this background migration imposes
sleep_interval = Config.get([:delete_context_objects, :sleep_interval_ms], 0)
Process.sleep(sleep_interval)
end)
|> Stream.run()
end
@impl BaseMigrator
def query do
# Context objects have no activity type, and only one field, `id`.
# Only those context objects are without types.
from(
object in Object,
where: fragment("(?)->'type' IS NULL", object.data),
select: %{
id: object.id
}
)
end
@spec delete_context_object(integer()) :: {:ok | :error, integer()}
defp delete_context_object(id) do
result =
%Object{id: id}
|> Repo.delete()
|> elem(0)
{result, id}
end
@impl BaseMigrator
def retry_failed do
data_migration_id = data_migration_id()
failed_objects_query()
|> Repo.chunk_stream(100, :one)
|> Stream.each(fn object ->
with {res, _} when res != :error <- delete_context_object(object.id) do
_ =
Repo.query(
"DELETE FROM data_migration_failed_ids " <>
"WHERE data_migration_id = $1 AND record_id = $2",
[data_migration_id, object.id]
)
end
end)
|> Stream.run()
put_stat(:failed_count, failures_count())
persist_state()
force_continue()
end
defp failed_objects_query do
from(o in Object)
|> join(:inner, [o], dmf in fragment("SELECT * FROM data_migration_failed_ids"),
on: dmf.record_id == o.id
)
|> where([_o, dmf], dmf.data_migration_id == ^data_migration_id())
|> order_by([o], asc: o.id)
end
end

4
lib/pleroma/object.ex
View file

@ -207,10 +207,6 @@ def get_cached_by_ap_id(ap_id) do
end
end
def context_mapping(context) do
Object.change(%Object{}, %{data: %{"id" => context}})
end
def make_tombstone(%Object{data: %{"id" => id, "type" => type}}, deleted \\ DateTime.utc_now()) do
%ObjectTombstone{
id: id,

2
lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex
View file

@ -97,7 +97,7 @@ def changeset(struct, data) do
defp validate_data(data_cng) do
data_cng
|> validate_inclusion(:type, ["Article", "Note", "Page"])
|> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])
|> validate_required([:id, :actor, :attributedTo, :type, :context])
|> CommonValidations.validate_any_presence([:cc, :to])
|> CommonValidations.validate_fields_match([:actor, :attributedTo])
|> CommonValidations.validate_actor_presence()

2
lib/pleroma/web/activity_pub/object_validators/common_fields.ex
View file

@ -52,8 +52,6 @@ defmacro status_object_fields do
field(:summary, :string)
field(:context, :string)
# short identifier for PleromaFE to group statuses by context
field(:context_id, :integer)
field(:sensitive, :boolean, default: false)
field(:replies_count, :integer, default: 0)

7
lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
View file

@ -22,14 +22,15 @@ def cast_and_filter_recipients(message, field, follower_collection, field_fallba
end
def fix_object_defaults(data) do
%{data: %{"id" => context}, id: context_id} =
Utils.create_context(data["context"] || data["conversation"])
context =
Utils.maybe_create_context(
data["context"] || data["conversation"] || data["inReplyTo"] || data["id"]
)
%User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["attributedTo"])
data
|> Map.put("context", context)
|> Map.put("context_id", context_id)
|> cast_and_filter_recipients("to", follower_collection)
|> cast_and_filter_recipients("cc", follower_collection)
|> cast_and_filter_recipients("bto", follower_collection)

2
lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
View file

@ -75,7 +75,7 @@ def fix(data, meta) do
data
|> CommonFixes.fix_actor()
|> Map.put_new("context", object["context"])
|> Map.put("context", object["context"])
|> fix_addressing(object)
end

2
lib/pleroma/web/activity_pub/object_validators/event_validator.ex
View file

@ -62,7 +62,7 @@ def changeset(struct, data) do
defp validate_data(data_cng) do
data_cng
|> validate_inclusion(:type, ["Event"])
|> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])
|> validate_required([:id, :actor, :attributedTo, :type, :context])
|> CommonValidations.validate_any_presence([:cc, :to])
|> CommonValidations.validate_fields_match([:actor, :attributedTo])
|> CommonValidations.validate_actor_presence()

2
lib/pleroma/web/activity_pub/object_validators/question_validator.ex
View file

@ -80,7 +80,7 @@ def changeset(struct, data) do
defp validate_data(data_cng) do
data_cng
|> validate_inclusion(:type, ["Question"])
|> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])
|> validate_required([:id, :actor, :attributedTo, :type, :context])
|> CommonValidations.validate_any_presence([:cc, :to])
|> CommonValidations.validate_fields_match([:actor, :attributedTo])
|> CommonValidations.validate_actor_presence()

23
lib/pleroma/web/activity_pub/utils.ex
View file

@ -154,22 +154,7 @@ def get_notified_from_object(object) do
Notification.get_notified_from_activity(%Activity{data: object}, false)
end
def create_context(context) do
context = context || generate_id("contexts")
# Ecto has problems accessing the constraint inside the jsonb,
# so we explicitly check for the existed object before insert
object = Object.get_cached_by_ap_id(context)
with true <- is_nil(object),
changeset <- Object.context_mapping(context),
{:ok, inserted_object} <- Repo.insert(changeset) do
inserted_object
else
_ ->
object
end
end
def maybe_create_context(context), do: context || generate_id("contexts")
@doc """
Enqueues an activity for federation if it's local
@ -201,18 +186,16 @@ def lazy_put_activity_defaults(map, true) do
|> Map.put_new("id", "pleroma:fakeid")
|> Map.put_new_lazy("published", &make_date/0)
|> Map.put_new("context", "pleroma:fakecontext")
|> Map.put_new("context_id", -1)
|> lazy_put_object_defaults(true)
end
def lazy_put_activity_defaults(map, _fake?) do
%{data: %{"id" => context}, id: context_id} = create_context(map["context"])
context = maybe_create_context(map["context"])
map
|> Map.put_new_lazy("id", &generate_activity_id/0)
|> Map.put_new_lazy("published", &make_date/0)
|> Map.put_new("context", context)
|> Map.put_new("context_id", context_id)
|> lazy_put_object_defaults(false)
end
@ -226,7 +209,6 @@ defp lazy_put_object_defaults(%{"object" => map} = activity, true)
|> Map.put_new("id", "pleroma:fake_object_id")
|> Map.put_new_lazy("published", &make_date/0)
|> Map.put_new("context", activity["context"])
|> Map.put_new("context_id", activity["context_id"])
|> Map.put_new("fake", true)
%{activity | "object" => object}
@ -239,7 +221,6 @@ defp lazy_put_object_defaults(%{"object" => map} = activity, _)
|> Map.put_new_lazy("id", &generate_object_id/0)
|> Map.put_new_lazy("published", &make_date/0)
|> Map.put_new("context", activity["context"])
|> Map.put_new("context_id", activity["context_id"])
%{activity | "object" => object}
end

9
lib/pleroma/web/api_spec/schemas/status.ex
View file

@ -148,9 +148,15 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do
description:
"A map consisting of alternate representations of the `content` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain`"
},
context: %Schema{
type: :string,
description: "The thread identifier the status is associated with"
},
conversation_id: %Schema{
type: :integer,
description: "The ID of the AP context the status is associated with (if any)"
deprecated: true,
description:
"The ID of the AP context the status is associated with (if any); deprecated, please use `context` instead"
},
direct_conversation_id: %Schema{
type: :integer,
@ -325,6 +331,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do
"pinned" => false,
"pleroma" => %{
"content" => %{"text/plain" => "foobar"},
"context" => "http://localhost:4001/objects/8b4c0c80-6a37-4d2a-b1b9-05a19e3875aa",
"conversation_id" => 345_972,
"direct_conversation_id" => nil,
"emoji_reactions" => [],

29
lib/pleroma/web/common_api/utils.ex
View file

@ -453,35 +453,6 @@ def get_report_statuses(%User{ap_id: actor}, %{status_ids: status_ids})
def get_report_statuses(_, _), do: {:ok, nil}
# DEPRECATED mostly, context objects are now created at insertion time.
def context_to_conversation_id(context) do
with %Object{id: id} <- Object.get_cached_by_ap_id(context) do
id
else
_e ->
changeset = Object.context_mapping(context)
case Repo.insert(changeset) do
{:ok, %{id: id}} ->
id
# This should be solved by an upsert, but it seems ecto
# has problems accessing the constraint inside the jsonb.
{:error, _} ->
Object.get_cached_by_ap_id(context).id
end
end
end
def conversation_id_to_context(id) do
with %Object{data: %{"id" => context}} <- Repo.get(Object, id) do
context
else
_e ->
{:error, dgettext("errors", "No such conversation")}
end
end
def validate_character_limit("" = _full_payload, [] = _attachments) do
{:error, dgettext("errors", "Cannot post an empty status without attachments")}
end

17
lib/pleroma/web/mastodon_api/views/status_view.ex
View file

@ -57,11 +57,19 @@ defp get_replied_to_activities(activities) do
end)
end
defp get_context_id(%{data: %{"context_id" => context_id}}) when not is_nil(context_id),
do: context_id
# DEPRECATED This field seems to be a left-over from the StatusNet era.
# If your application uses `pleroma.conversation_id`: this field is deprecated.
# It is currently stubbed instead by doing a CRC32 of the context, and
# clearing the MSB to avoid overflow exceptions with signed integers on the
# different clients using this field (Java/Kotlin code, mostly; see Husky.)
# This should be removed in a future version of Pleroma. Pleroma-FE currently
# depends on this field, as well.
defp get_context_id(%{data: %{"context" => context}}) when is_binary(context) do
use Bitwise
defp get_context_id(%{data: %{"context" => context}}) when is_binary(context),
do: Utils.context_to_conversation_id(context)
:erlang.crc32(context)
|> band(bnot(0x8000_0000))
end
defp get_context_id(_), do: nil
@ -388,6 +396,7 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity}
pleroma: %{
local: activity.local,
conversation_id: get_context_id(activity),
context: object.data["context"],
in_reply_to_account_acct: reply_to_user && reply_to_user.nickname,
content: %{"text/plain" => content_plaintext},
spoiler_text: %{"text/plain" => summary},

18
priv/repo/migrations/20220807125023_data_migration_delete_context_objects.exs Normal file
View file

@ -0,0 +1,18 @@
defmodule Pleroma.Repo.Migrations.DataMigrationDeleteContextObjects do
use Ecto.Migration
require Logger
def up do
dt = NaiveDateTime.utc_now()
execute(
"INSERT INTO data_migrations(name, inserted_at, updated_at) " <>
"VALUES ('delete_context_objects', '#{dt}', '#{dt}') ON CONFLICT DO NOTHING;"
)
end
def down do
execute("DELETE FROM data_migrations WHERE name = 'delete_context_objects';")
end
end

61
test/fixtures/create-pleroma-reply-to-misskey-thread.json vendored Normal file
View file

@ -0,0 +1,61 @@
{
"@context": [
"https://www.w3.org/ns/activitystreams",
"https://p.helene.moe/schemas/litepub-0.1.jsonld",
{
"@language": "und"
}
],
"actor": "https://p.helene.moe/users/helene",
"attachment": [],
"attributedTo": "https://p.helene.moe/users/helene",
"cc": [
"https://p.helene.moe/users/helene/followers"
],
"context": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d",
"conversation": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d",
"directMessage": false,
"id": "https://p.helene.moe/activities/5f80db86-a9bb-4883-9845-fbdbd1478f3a",
"object": {
"actor": "https://p.helene.moe/users/helene",
"attachment": [],
"attributedTo": "https://p.helene.moe/users/helene",
"cc": [
"https://p.helene.moe/users/helene/followers"
],
"content": "<span class=\"h-card\"><a class=\"u-url mention\" data-user=\"AHntpQ4T3J4OSnpgMC\" href=\"https://mk.absturztau.be/@mametsuko\" rel=\"ugc\">@<span>mametsuko</span></a></span> meow",
"context": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d",
"conversation": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d",
"id": "https://p.helene.moe/objects/fd5910ac-d9dc-412e-8d1d-914b203296c4",
"inReplyTo": "https://mk.absturztau.be/notes/93e7nm8wqg",
"published": "2022-08-02T13:46:58.403996Z",
"sensitive": null,
"source": "@mametsuko@mk.absturztau.be meow",
"summary": "",
"tag": [
{
"href": "https://mk.absturztau.be/users/8ozbzjs3o8",
"name": "@mametsuko@mk.absturztau.be",
"type": "Mention"
}
],
"to": [
"https://mk.absturztau.be/users/8ozbzjs3o8",
"https://www.w3.org/ns/activitystreams#Public"
],
"type": "Note"
},
"published": "2022-08-02T13:46:58.403883Z",
"tag": [
{
"href": "https://mk.absturztau.be/users/8ozbzjs3o8",
"name": "@mametsuko@mk.absturztau.be",
"type": "Mention"
}
],
"to": [
"https://mk.absturztau.be/users/8ozbzjs3o8",
"https://www.w3.org/ns/activitystreams#Public"
],
"type": "Create"
}

50
test/fixtures/tesla_mock/helene@p.helene.moe.json vendored Normal file
View file

@ -0,0 +1,50 @@
{
"@context": [
"https://www.w3.org/ns/activitystreams",
"https://p.helene.moe/schemas/litepub-0.1.jsonld",
{
"@language": "und"
}
],
"alsoKnownAs": [],
"attachment": [
{
"name": "Timezone",
"type": "PropertyValue",
"value": "UTC+2 (Paris/Berlin)"
}
],
"capabilities": {
"acceptsChatMessages": true
},
"discoverable": true,
"endpoints": {
"oauthAuthorizationEndpoint": "https://p.helene.moe/oauth/authorize",
"oauthRegistrationEndpoint": "https://p.helene.moe/api/v1/apps",
"oauthTokenEndpoint": "https://p.helene.moe/oauth/token",
"sharedInbox": "https://p.helene.moe/inbox",
"uploadMedia": "https://p.helene.moe/api/ap/upload_media"
},
"featured": "https://p.helene.moe/users/helene/collections/featured",
"followers": "https://p.helene.moe/users/helene/followers",
"following": "https://p.helene.moe/users/helene/following",
"icon": {
"type": "Image",
"url": "https://p.helene.moe/media/9a39209daa5a66b7ebb0547b08bf8360aa9d8d65a4ffba2603c6ffbe6aecb432.jpg"
},
"id": "https://p.helene.moe/users/helene",
"inbox": "https://p.helene.moe/users/helene/inbox",
"manuallyApprovesFollowers": false,
"name": "Hélène",
"outbox": "https://p.helene.moe/users/helene/outbox",
"preferredUsername": "helene",
"publicKey": {
"id": "https://p.helene.moe/users/helene#main-key",
"owner": "https://p.helene.moe/users/helene",
"publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtoSBPU/VS2Kx3f6ap3zv\nZVacJsgUfaoFb3c2ii/FRh9RmRVlarq8sJXcjsQt1e0oxWaWJaIDDwyKZPt6hXae\nrY/AiGGeNu+NA+BtY7l7+9Yu67HUyT62+1qAwYHKBXX3fLOPs/YmQI0Tt0c4wKAG\nKEkiYsRizghgpzUC6jqdKV71DJkUZ8yhckCGb2fLko1ajbWEssdaP51aLsyRMyC2\nuzeWrxtD4O/HG0ea4S6y5X6hnsAHIK4Y3nnyIQ6pn4tOsl3HgqkjXE9MmZSvMCFx\nBq89TfZrVXNa2gSZdZLdbbJstzEScQWNt1p6tA6rM+e4JXYGr+rMdF3G+jV7afI2\nFQIDAQAB\n-----END PUBLIC KEY-----\n\n"
},
"summary": "I can speak: Français, English, Deutsch (nicht sehr gut), 日本語 (not very well)",
"tag": [],
"type": "Person",
"url": "https://p.helene.moe/users/helene"
}

65
test/fixtures/tesla_mock/mametsuko@mk.absturztau.be.json vendored Normal file
View file

@ -0,0 +1,65 @@
{
"@context": [
"https://www.w3.org/ns/activitystreams",
"https://w3id.org/security/v1",
{
"manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
"sensitive": "as:sensitive",
"Hashtag": "as:Hashtag",
"quoteUrl": "as:quoteUrl",
"toot": "http://joinmastodon.org/ns#",
"Emoji": "toot:Emoji",
"featured": "toot:featured",
"discoverable": "toot:discoverable",
"schema": "http://schema.org#",
"PropertyValue": "schema:PropertyValue",
"value": "schema:value",
"misskey": "https://misskey-hub.net/ns#",
"_misskey_content": "misskey:_misskey_content",
"_misskey_quote": "misskey:_misskey_quote",
"_misskey_reaction": "misskey:_misskey_reaction",
"_misskey_votes": "misskey:_misskey_votes",
"_misskey_talk": "misskey:_misskey_talk",
"isCat": "misskey:isCat",
"vcard": "http://www.w3.org/2006/vcard/ns#"
}
],
"type": "Person",
"id": "https://mk.absturztau.be/users/8ozbzjs3o8",
"inbox": "https://mk.absturztau.be/users/8ozbzjs3o8/inbox",
"outbox": "https://mk.absturztau.be/users/8ozbzjs3o8/outbox",
"followers": "https://mk.absturztau.be/users/8ozbzjs3o8/followers",
"following": "https://mk.absturztau.be/users/8ozbzjs3o8/following",
"featured": "https://mk.absturztau.be/users/8ozbzjs3o8/collections/featured",
"sharedInbox": "https://mk.absturztau.be/inbox",
"endpoints": {
"sharedInbox": "https://mk.absturztau.be/inbox"
},
"url": "https://mk.absturztau.be/@mametsuko",
"preferredUsername": "mametsuko",
"name": "mametschko",
"summary": "<p><span>nya, ich bin eine Brotperson</span></p>",
"icon": {
"type": "Image",
"url": "https://mk.absturztau.be/files/webpublic-3b5594f4-fa52-4548-b4e3-c379ae2143ed",
"sensitive": false,
"name": null
},
"image": {
"type": "Image",
"url": "https://mk.absturztau.be/files/webpublic-0d03b03d-b14b-4916-ac3d-8a137118ec84",
"sensitive": false,
"name": null
},
"tag": [],
"manuallyApprovesFollowers": true,
"discoverable": false,
"publicKey": {
"id": "https://mk.absturztau.be/users/8ozbzjs3o8#main-key",
"type": "Key",
"owner": "https://mk.absturztau.be/users/8ozbzjs3o8",
"publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAuN/S1spBGmh8FXI1Bt16\nXB7Cc0QutBp7UPgmDNHjOfsq0zrF4g3L1UBxvrpU0XX77XPMCd9yPvGwAYURH2mv\ntIcYuE+R90VLDmBu5MTVthcG2D874eCZ2rD2YsEYmN5AjTX7QBIqCck+qDhVWkkM\nEZ6S5Ht6IJ5Of74eKffXElQI/C6QB+9uEDOmPk0jCzgI5gw7xvJqFj/DIF4kUUAu\nA89JqaFZzZlkrSrj4cr48bLN/YOmpdaHu0BKHaDSHct4+MqlixqovgdB6RboCEDw\ne4Aeav7+Q0Y9oGIvuggg0Q+nCubnVNnaPyzd817tpPVzyZmTts+DKyDuv90SX3nR\nsPaNa5Ty60eqplUk4b7X1gSvuzBJUFBxTVV84WnjwoeoydaS6rSyjCDPGLBjaByc\nFyWMMEb/zlQyhLZfBlvT7k96wRSsMszh2hDALWmgYIhq/jNwINvALJ1GKLNHHKZ4\nyz2LnxVpRm2rWrZzbvtcnSQOt3LaPSZn8Wgwv4buyHF02iuVuIamZVtKexsE1Ixl\nIi9qa3AKEc5gOzYXhRhvHaruzoCehUbb/UHC5c8Tto8L5G1xYzjLP3qj3PT9w/wM\n+k1Ra/4JhuAnVFROOoOmx9rIELLHH7juY2nhM7plGhyt1M5gysgqEloij8QzyQU2\nZK1YlAERG2XFO6br8omhcmECAwEAAQ==\n-----END PUBLIC KEY-----\n"
},
"isCat": true,
"vcard:Address": "Vienna, Austria"
}

1
test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg-activity.json vendored Normal file
View file

@ -0,0 +1 @@
{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1",{"manuallyApprovesFollowers":"as:manuallyApprovesFollowers","sensitive":"as:sensitive","Hashtag":"as:Hashtag","quoteUrl":"as:quoteUrl","toot":"http://joinmastodon.org/ns#","Emoji":"toot:Emoji","featured":"toot:featured","discoverable":"toot:discoverable","schema":"http://schema.org#","PropertyValue":"schema:PropertyValue","value":"schema:value","misskey":"https://misskey-hub.net/ns#","_misskey_content":"misskey:_misskey_content","_misskey_quote":"misskey:_misskey_quote","_misskey_reaction":"misskey:_misskey_reaction","_misskey_votes":"misskey:_misskey_votes","_misskey_talk":"misskey:_misskey_talk","isCat":"misskey:isCat","vcard":"http://www.w3.org/2006/vcard/ns#"}],"id":"https://mk.absturztau.be/notes/93e7nm8wqg/activity","actor":"https://mk.absturztau.be/users/8ozbzjs3o8","type":"Create","published":"2022-08-01T11:06:49.568Z","object":{"id":"https://mk.absturztau.be/notes/93e7nm8wqg","type":"Note","attributedTo":"https://mk.absturztau.be/users/8ozbzjs3o8","summary":null,"content":"<p><span>meow</span></p>","_misskey_content":"meow","published":"2022-08-01T11:06:49.568Z","to":["https://www.w3.org/ns/activitystreams#Public"],"cc":["https://mk.absturztau.be/users/8ozbzjs3o8/followers"],"inReplyTo":null,"attachment":[],"sensitive":false,"tag":[]},"to":["https://www.w3.org/ns/activitystreams#Public"],"cc":["https://mk.absturztau.be/users/8ozbzjs3o8/followers"]}

44
test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg.json vendored Normal file
View file

@ -0,0 +1,44 @@
{
"@context": [
"https://www.w3.org/ns/activitystreams",
"https://w3id.org/security/v1",
{
"manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
"sensitive": "as:sensitive",
"Hashtag": "as:Hashtag",
"quoteUrl": "as:quoteUrl",
"toot": "http://joinmastodon.org/ns#",
"Emoji": "toot:Emoji",
"featured": "toot:featured",
"discoverable": "toot:discoverable",
"schema": "http://schema.org#",
"PropertyValue": "schema:PropertyValue",
"value": "schema:value",
"misskey": "https://misskey-hub.net/ns#",
"_misskey_content": "misskey:_misskey_content",
"_misskey_quote": "misskey:_misskey_quote",
"_misskey_reaction": "misskey:_misskey_reaction",
"_misskey_votes": "misskey:_misskey_votes",
"_misskey_talk": "misskey:_misskey_talk",
"isCat": "misskey:isCat",
"vcard": "http://www.w3.org/2006/vcard/ns#"
}
],
"id": "https://mk.absturztau.be/notes/93e7nm8wqg",
"type": "Note",
"attributedTo": "https://mk.absturztau.be/users/8ozbzjs3o8",
"summary": null,
"content": "<p><span>meow</span></p>",
"_misskey_content": "meow",
"published": "2022-08-01T11:06:49.568Z",
"to": [
"https://www.w3.org/ns/activitystreams#Public"
],
"cc": [
"https://mk.absturztau.be/users/8ozbzjs3o8/followers"
],
"inReplyTo": null,
"attachment": [],
"sensitive": false,
"tag": []
}

36
test/fixtures/tesla_mock/p.helene.moe-AM7S6vZQmL6pI9TgPY.json vendored Normal file
View file

@ -0,0 +1,36 @@
{
"@context": [
"https://www.w3.org/ns/activitystreams",
"https://p.helene.moe/schemas/litepub-0.1.jsonld",
{
"@language": "und"
}
],
"actor": "https://p.helene.moe/users/helene",
"attachment": [],
"attributedTo": "https://p.helene.moe/users/helene",
"cc": [
"https://p.helene.moe/users/helene/followers"
],
"content": "<span class=\"h-card\"><a class=\"u-url mention\" data-user=\"AHntpQ4T3J4OSnpgMC\" href=\"https://mk.absturztau.be/@mametsuko\" rel=\"ugc\">@<span>mametsuko</span></a></span> meow",
"context": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d",
"conversation": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d",
"id": "https://p.helene.moe/objects/fd5910ac-d9dc-412e-8d1d-914b203296c4",
"inReplyTo": "https://mk.absturztau.be/notes/93e7nm8wqg",
"published": "2022-08-02T13:46:58.403996Z",
"sensitive": null,
"source": "@mametsuko@mk.absturztau.be meow",
"summary": "",
"tag": [
{
"href": "https://mk.absturztau.be/users/8ozbzjs3o8",
"name": "@mametsuko@mk.absturztau.be",
"type": "Mention"
}
],
"to": [
"https://mk.absturztau.be/users/8ozbzjs3o8",
"https://www.w3.org/ns/activitystreams#Public"
],
"type": "Note"
}

5
test/pleroma/web/activity_pub/activity_pub_test.exs
View file

@ -554,7 +554,6 @@ test "inserts a given map into the activity database, giving it an id if it has
assert activity.data["ok"] == data["ok"]
assert activity.data["id"] == given_id
assert activity.data["context"] == "blabla"
assert activity.data["context_id"]
end
test "adds a context when none is there" do
@ -576,8 +575,6 @@ test "adds a context when none is there" do
assert is_binary(activity.data["context"])
assert is_binary(object.data["context"])
assert activity.data["context_id"]
assert object.data["context_id"]
end
test "adds an id to a given object if it lacks one and is a note and inserts it to the object database" do
@ -1612,7 +1609,7 @@ test "it can create a Flag activity",
})
assert Repo.aggregate(Activity, :count, :id) == 1
assert Repo.aggregate(Object, :count, :id) == 2
assert Repo.aggregate(Object, :count, :id) == 1
assert Repo.aggregate(Notification, :count, :id) == 0
end
end

9
test/pleroma/web/activity_pub/object_validators/create_generic_validator_test.exs
View file

@ -23,10 +23,10 @@ test "a Create/Note from Roadhouse validates" do
{:ok, object_data} = ObjectValidator.cast_and_apply(note_activity["object"])
meta = [object_data: ObjectValidator.stringify_keys(object_data)]
%{valid?: true} = CreateGenericValidator.cast_and_validate(note_activity, meta)
assert %{valid?: true} = CreateGenericValidator.cast_and_validate(note_activity, meta)
end
test "a Create/Note with mismatched context is invalid" do
test "a Create/Note with mismatched context uses the Note's context" do
user = insert(:user)
note = %{
@ -54,6 +54,9 @@ test "a Create/Note with mismatched context is invalid" do
{:ok, object_data} = ObjectValidator.cast_and_apply(note_activity["object"])
meta = [object_data: ObjectValidator.stringify_keys(object_data)]
%{valid?: false} = CreateGenericValidator.cast_and_validate(note_activity, meta)
validated = CreateGenericValidator.cast_and_validate(note_activity, meta)
assert validated.valid?
assert {:context, note["context"]} in validated.changes
end
end

38
test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs
View file

@ -707,4 +707,42 @@ test "take_emoji_tags/1" do
}
]
end
test "the standalone note uses its own ID when context is missing" do
insert(:user, ap_id: "https://mk.absturztau.be/users/8ozbzjs3o8")
activity =
"test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg-activity.json"
|> File.read!()
|> Jason.decode!()
{:ok, %Activity{} = modified} = Transmogrifier.handle_incoming(activity)
object = Object.normalize(modified, fetch: false)
assert object.data["context"] == object.data["id"]
assert modified.data["context"] == object.data["id"]
end
test "the reply note uses its parent's ID when context is missing and reply is unreachable" do
insert(:user, ap_id: "https://mk.absturztau.be/users/8ozbzjs3o8")
activity =
"test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg-activity.json"
|> File.read!()
|> Jason.decode!()
object =
activity["object"]
|> Map.put("inReplyTo", "https://404.site/object/went-to-buy-milk")
activity =
activity
|> Map.put("object", object)
{:ok, %Activity{} = modified} = Transmogrifier.handle_incoming(activity)
object = Object.normalize(modified, fetch: false)
assert object.data["context"] == object.data["inReplyTo"]
assert modified.data["context"] == object.data["inReplyTo"]
end
end

3
test/pleroma/web/activity_pub/transmogrifier/question_handling_test.exs
View file

@ -33,8 +33,6 @@ test "Mastodon Question activity" do
assert object.data["context"] ==
"tag:mastodon.sdf.org,2019-05-10:objectId=15095122:objectType=Conversation"
assert object.data["context_id"]
assert object.data["anyOf"] == []
assert Enum.sort(object.data["oneOf"]) ==
@ -68,7 +66,6 @@ test "Mastodon Question activity" do
reply_object = Object.normalize(reply_activity, fetch: false)
assert reply_object.data["context"] == object.data["context"]
assert reply_object.data["context_id"] == object.data["context_id"]
end
test "Mastodon Question activity with HTML tags in plaintext" do

17
test/pleroma/web/activity_pub/transmogrifier_test.exs
View file

@ -108,15 +108,20 @@ test "it accepts Move activities" do
assert activity.data["type"] == "Move"
end
test "a reply with mismatched context is rejected" do
insert(:user, ap_id: "https://macgirvin.com/channel/mike")
test "it fixes both the Create and object contexts in a reply" do
insert(:user, ap_id: "https://mk.absturztau.be/users/8ozbzjs3o8")
insert(:user, ap_id: "https://p.helene.moe/users/helene")
note_activity =
"test/fixtures/roadhouse-create-activity.json"
create_activity =
"test/fixtures/create-pleroma-reply-to-misskey-thread.json"
|> File.read!()
|> Jason.decode!()
assert {:error, _} = Transmogrifier.handle_incoming(note_activity)
assert {:ok, %Activity{} = activity} = Transmogrifier.handle_incoming(create_activity)
object = Object.normalize(activity, fetch: false)
assert activity.data["context"] == object.data["context"]
end
end
@ -227,7 +232,6 @@ test "it strips internal fields" do
assert is_nil(modified["object"]["like_count"])
assert is_nil(modified["object"]["announcements"])
assert is_nil(modified["object"]["announcement_count"])
assert is_nil(modified["object"]["context_id"])
assert is_nil(modified["object"]["generator"])
end
@ -242,7 +246,6 @@ test "it strips internal fields of article" do
assert is_nil(modified["object"]["like_count"])
assert is_nil(modified["object"]["announcements"])
assert is_nil(modified["object"]["announcement_count"])
assert is_nil(modified["object"]["context_id"])
assert is_nil(modified["object"]["likes"])
end

4
test/pleroma/web/activity_pub/utils_test.exs
View file

@ -429,7 +429,6 @@ test "returns map with id and published data" do
object = Object.normalize(note_activity, fetch: false)
res = Utils.lazy_put_activity_defaults(%{"context" => object.data["id"]})
assert res["context"] == object.data["id"]
assert res["context_id"] == object.id
assert res["id"]
assert res["published"]
end
@ -437,7 +436,6 @@ test "returns map with id and published data" do
test "returns map with fake id and published data" do
assert %{
"context" => "pleroma:fakecontext",
"context_id" => -1,
"id" => "pleroma:fakeid",
"published" => _
} = Utils.lazy_put_activity_defaults(%{}, true)
@ -454,13 +452,11 @@ test "returns activity data with object" do
})
assert res["context"] == object.data["id"]
assert res["context_id"] == object.id
assert res["id"]
assert res["published"]
assert res["object"]["id"]
assert res["object"]["published"]
assert res["object"]["context"] == object.data["id"]
assert res["object"]["context_id"] == object.id
end
end

28
test/pleroma/web/common_api/utils_test.exs
View file

@ -4,7 +4,6 @@
defmodule Pleroma.Web.CommonAPI.UtilsTest do
alias Pleroma.Builders.UserBuilder
alias Pleroma.Object
alias Pleroma.Web.CommonAPI
alias Pleroma.Web.CommonAPI.ActivityDraft
alias Pleroma.Web.CommonAPI.Utils
@ -273,22 +272,6 @@ test "delegated renderers" do
end
end
describe "context_to_conversation_id" do
test "creates a mapping object" do
conversation_id = Utils.context_to_conversation_id("random context")
object = Object.get_by_ap_id("random context")
assert conversation_id == object.id
end
test "returns an existing mapping for an existing object" do
{:ok, object} = Object.context_mapping("random context") |> Repo.insert()
conversation_id = Utils.context_to_conversation_id("random context")
assert conversation_id == object.id
end
end
describe "formats date to asctime" do
test "when date is in ISO 8601 format" do
date = DateTime.utc_now() |> DateTime.to_iso8601()
@ -517,17 +500,6 @@ test "returns empty string when date invalid" do
end
end
describe "conversation_id_to_context/1" do
test "returns id" do
object = insert(:note)
assert Utils.conversation_id_to_context(object.id) == object.data["id"]
end
test "returns error if object not found" do
assert Utils.conversation_id_to_context("123") == {:error, "No such conversation"}
end
end
describe "maybe_notify_mentioned_recipients/2" do
test "returns recipients when activity is not `Create`" do
activity = insert(:like_activity)

2
test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
View file

@ -262,6 +262,7 @@ test "posting a fake status", %{conn: conn} do
|> Map.put("url", nil)
|> Map.put("uri", nil)
|> Map.put("created_at", nil)
|> Kernel.put_in(["pleroma", "context"], nil)
|> Kernel.put_in(["pleroma", "conversation_id"], nil)
fake_conn =
@ -285,6 +286,7 @@ test "posting a fake status", %{conn: conn} do
|> Map.put("url", nil)
|> Map.put("uri", nil)
|> Map.put("created_at", nil)
|> Kernel.put_in(["pleroma", "context"], nil)
|> Kernel.put_in(["pleroma", "conversation_id"], nil)
assert real_status == fake_status

6
test/pleroma/web/mastodon_api/views/status_view_test.exs
View file

@ -14,10 +14,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
alias Pleroma.User
alias Pleroma.UserRelationship
alias Pleroma.Web.CommonAPI
alias Pleroma.Web.CommonAPI.Utils
alias Pleroma.Web.MastodonAPI.AccountView
alias Pleroma.Web.MastodonAPI.StatusView
require Bitwise
import Pleroma.Factory
import Tesla.Mock
import OpenApiSpex.TestAssertions
@ -226,7 +227,7 @@ test "a note activity" do
object_data = Object.normalize(note, fetch: false).data
user = User.get_cached_by_ap_id(note.data["actor"])
convo_id = Utils.context_to_conversation_id(object_data["context"])
convo_id = :erlang.crc32(object_data["context"]) |> Bitwise.band(Bitwise.bnot(0x8000_0000))
status = StatusView.render("show.json", %{activity: note})
@ -280,6 +281,7 @@ test "a note activity" do
pleroma: %{
local: true,
conversation_id: convo_id,
context: object_data["context"],
in_reply_to_account_acct: nil,
content: %{"text/plain" => HTML.strip_tags(object_data["content"])},
spoiler_text: %{"text/plain" => HTML.strip_tags(object_data["summary"])},

53
test/support/http_request_mock.ex
View file

@ -1084,6 +1084,14 @@ def get("http://404.site" <> _, _, _, _) do
}}
end
def get("https://404.site" <> _, _, _, _) do
{:ok,
%Tesla.Env{
status: 404,
body: ""
}}
end
def get(
"https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=acct:lain@zetsubou.xn--q9jyb4c",
_,
@ -1401,6 +1409,51 @@ def get("https://gleasonator.com/users/macgirvin/collections/featured", _, _, _)
}}
end
def get("https://mk.absturztau.be/users/8ozbzjs3o8", _, _, _) do
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/mametsuko@mk.absturztau.be.json"),
headers: activitypub_object_headers()
}}
end
def get("https://p.helene.moe/users/helene", _, _, _) do
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/helene@p.helene.moe.json"),
headers: activitypub_object_headers()
}}
end
def get("https://mk.absturztau.be/notes/93e7nm8wqg", _, _, _) do
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg.json"),
headers: activitypub_object_headers()
}}
end
def get("https://mk.absturztau.be/notes/93e7nm8wqg/activity", _, _, _) do
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg-activity.json"),
headers: activitypub_object_headers()
}}
end
def get("https://p.helene.moe/objects/fd5910ac-d9dc-412e-8d1d-914b203296c4", _, _, _) do
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/p.helene.moe-AM7S6vZQmL6pI9TgPY.json"),
headers: activitypub_object_headers()
}}
end
def get(url, query, body, headers) do
{:error,
"Mock response not implemented for GET #{inspect(url)}, #{query}, #{inspect(body)}, #{inspect(headers)}"}