Merge branch 'develop' into 'fix/hide-follows-counters'

# Conflicts:
#   CHANGELOG.md
This commit is contained in:
kaniini 2019-08-10 18:49:04 +00:00
commit 84808e1697
16 changed files with 290 additions and 50 deletions

View file

@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Mastodon API: Handling of search timeouts (`/api/v1/search` and `/api/v2/search`)
- Mastodon API: Embedded relationships not being properly rendered in the Account entity of Status entity
- Mastodon API: follower/following counters not being nullified, when `hide_follows`/`hide_followers` is set
- Mastodon API: `muted` in the Status entity, using author's account to determine if the tread was muted
- Mastodon API: Add `account_id`, `type`, `offset`, and `limit` to search API (`/api/v1/search` and `/api/v2/search`)
- Mastodon API, streaming: Fix filtering of notifications based on blocks/mutes/thread mutes
- ActivityPub C2S: follower/following collection pages being inaccessible even when authentifucated if `hide_followers`/ `hide_follows` was set
@ -73,6 +74,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Admin API: Endpoint for fetching latest user's statuses
- Pleroma API: Add `/api/v1/pleroma/accounts/confirmation_resend?email=<email>` for resending account confirmation.
- Relays: Added a task to list relay subscriptions.
- Mix Tasks: `mix pleroma.database fix_likes_collections`
- Federation: Remove `likes` from objects.
### Changed
- Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text

View file

@ -18,6 +18,7 @@ Note: `strip_exif` has been replaced by `Pleroma.Upload.Filter.Mogrify`.
## Pleroma.Uploaders.S3
* `bucket`: S3 bucket name
* `bucket_namespace`: S3 bucket namespace
* `public_endpoint`: S3 endpoint that the user finally accesses(ex. "https://s3.dualstack.ap-northeast-1.amazonaws.com")
* `truncated_namespace`: If you use S3 compatible service such as Digital Ocean Spaces or CDN, set folder name or "" etc.
For example, when using CDN to S3 virtual host format, set "".

View file

@ -36,6 +36,10 @@ defmodule Mix.Tasks.Pleroma.Database do
## Remove duplicated items from following and update followers count for all users
mix pleroma.database update_users_following_followers_counts
## Fix the pre-existing "likes" collections for all objects
mix pleroma.database fix_likes_collections
"""
def run(["remove_embedded_objects" | args]) do
{options, [], []} =
@ -125,4 +129,36 @@ def run(["prune_objects" | args]) do
)
end
end
def run(["fix_likes_collections"]) do
import Ecto.Query
start_pleroma()
from(object in Object,
where: fragment("(?)->>'likes' is not null", object.data),
select: %{id: object.id, likes: fragment("(?)->>'likes'", object.data)}
)
|> Pleroma.RepoStreamer.chunk_stream(100)
|> Stream.each(fn objects ->
ids =
objects
|> Enum.filter(fn object -> object.likes |> Jason.decode!() |> is_map() end)
|> Enum.map(& &1.id)
Object
|> where([object], object.id in ^ids)
|> update([object],
set: [
data:
fragment(
"jsonb_set(?, '{likes}', '[]'::jsonb, true)",
object.data
)
]
)
|> Repo.update_all([], timeout: :infinity)
end)
|> Stream.run()
end
end

View file

@ -11,7 +11,7 @@ def get_file(_) do
def put_file(upload) do
{local_path, file} =
case Enum.reverse(String.split(upload.path, "/", trim: true)) do
case Enum.reverse(Path.split(upload.path)) do
[file] ->
{upload_path(), file}
@ -23,7 +23,7 @@ def put_file(upload) do
result_file = Path.join(local_path, file)
unless File.exists?(result_file) do
if not File.exists?(result_file) do
File.cp!(upload.tempfile, result_file)
end

View file

@ -3,6 +3,8 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Uploaders.MDII do
@moduledoc "Represents uploader for https://github.com/hakaba-hitoyo/minimal-digital-image-infrastructure"
alias Pleroma.Config
alias Pleroma.HTTP

View file

@ -6,10 +6,12 @@ defmodule Pleroma.Uploaders.S3 do
@behaviour Pleroma.Uploaders.Uploader
require Logger
alias Pleroma.Config
# The file name is re-encoded with S3's constraints here to comply with previous
# links with less strict filenames
def get_file(file) do
config = Pleroma.Config.get([__MODULE__])
config = Config.get([__MODULE__])
bucket = Keyword.fetch!(config, :bucket)
bucket_with_namespace =
@ -34,15 +36,15 @@ def get_file(file) do
end
def put_file(%Pleroma.Upload{} = upload) do
config = Pleroma.Config.get([__MODULE__])
config = Config.get([__MODULE__])
bucket = Keyword.get(config, :bucket)
{:ok, file_data} = File.read(upload.tempfile)
s3_name = strict_encode(upload.path)
op =
ExAws.S3.put_object(bucket, s3_name, file_data, [
upload.tempfile
|> ExAws.S3.Upload.stream_file()
|> ExAws.S3.upload(bucket, s3_name, [
{:acl, :public_read},
{:content_type, upload.content_type}
])

View file

@ -26,6 +26,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
"""
def fix_object(object, options \\ []) do
object
|> strip_internal_fields
|> fix_actor
|> fix_url
|> fix_attachments
@ -34,7 +35,6 @@ def fix_object(object, options \\ []) do
|> fix_emoji
|> fix_tag
|> fix_content_map
|> fix_likes
|> fix_addressing
|> fix_summary
|> fix_type(options)
@ -151,20 +151,6 @@ def fix_actor(%{"attributedTo" => actor} = object) do
|> Map.put("actor", Containment.get_actor(%{"actor" => actor}))
end
# Check for standardisation
# This is what Peertube does
# curl -H 'Accept: application/activity+json' $likes | jq .totalItems
# Prismo returns only an integer (count) as "likes"
def fix_likes(%{"likes" => likes} = object) when not is_map(likes) do
object
|> Map.put("likes", [])
|> Map.put("like_count", 0)
end
def fix_likes(object) do
object
end
def fix_in_reply_to(object, options \\ [])
def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options)
@ -784,7 +770,6 @@ def prepare_object(object) do
|> add_mention_tags
|> add_emoji_tags
|> add_attributed_to
|> add_likes
|> prepare_attachments
|> set_conversation
|> set_reply_to_uri
@ -971,22 +956,6 @@ def add_attributed_to(object) do
|> Map.put("attributedTo", attributed_to)
end
def add_likes(%{"id" => id, "like_count" => likes} = object) do
likes = %{
"id" => "#{id}/likes",
"first" => "#{id}/likes?page=1",
"type" => "OrderedCollection",
"totalItems" => likes
}
object
|> Map.put("likes", likes)
end
def add_likes(object) do
object
end
def prepare_attachments(object) do
attachments =
(object["attachment"] || [])
@ -1002,6 +971,7 @@ def prepare_attachments(object) do
defp strip_internal_fields(object) do
object
|> Map.drop([
"likes",
"like_count",
"announcements",
"announcement_count",

View file

@ -168,7 +168,7 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity
thread_muted? =
case activity.thread_muted? do
thread_muted? when is_boolean(thread_muted?) -> thread_muted?
nil -> CommonAPI.thread_muted?(user, activity)
nil -> (opts[:for] && CommonAPI.thread_muted?(opts[:for], activity)) || false
end
attachment_data = object.data["attachment"] || []

View file

@ -114,8 +114,9 @@ defp deps do
{:tesla, "~> 1.2"},
{:jason, "~> 1.0"},
{:mogrify, "~> 0.6.1"},
{:ex_aws, "~> 2.0"},
{:ex_aws, "~> 2.1"},
{:ex_aws_s3, "~> 2.0"},
{:sweet_xml, "~> 0.6.6"},
{:earmark, "~> 1.3"},
{:bbcode, "~> 0.1.1"},
{:ex_machina, "~> 2.3", only: :test},

View file

@ -80,6 +80,7 @@
"ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"},
"recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]},
"ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"},
"sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"},
"swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"},
"syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]},
"telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"},

View file

@ -3,8 +3,11 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mix.Tasks.Pleroma.DatabaseTest do
alias Pleroma.Object
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.Web.CommonAPI
use Pleroma.DataCase
import Pleroma.Factory
@ -46,4 +49,37 @@ test "following and followers count are updated" do
assert user.info.follower_count == 0
end
end
describe "running fix_likes_collections" do
test "it turns OrderedCollection likes into empty arrays" do
[user, user2] = insert_pair(:user)
{:ok, %{id: id, object: object}} = CommonAPI.post(user, %{"status" => "test"})
{:ok, %{object: object2}} = CommonAPI.post(user, %{"status" => "test test"})
CommonAPI.favorite(id, user2)
likes = %{
"first" =>
"http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes?page=1",
"id" => "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes",
"totalItems" => 3,
"type" => "OrderedCollection"
}
new_data = Map.put(object2.data, "likes", likes)
object2
|> Ecto.Changeset.change(%{data: new_data})
|> Repo.update()
assert length(Object.get_by_id(object.id).data["likes"]) == 1
assert is_map(Object.get_by_id(object2.id).data["likes"])
assert :ok == Mix.Tasks.Pleroma.Database.run(["fix_likes_collections"])
assert length(Object.get_by_id(object.id).data["likes"]) == 1
assert Enum.empty?(Object.get_by_id(object2.id).data["likes"])
end
end
end

View file

@ -0,0 +1,32 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Uploaders.LocalTest do
use Pleroma.DataCase
alias Pleroma.Uploaders.Local
describe "get_file/1" do
test "it returns path to local folder for files" do
assert Local.get_file("") == {:ok, {:static_dir, "test/uploads"}}
end
end
describe "put_file/1" do
test "put file to local folder" do
file_path = "local_upload/files/image.jpg"
file = %Pleroma.Upload{
name: "image.jpg",
content_type: "image/jpg",
path: file_path,
tempfile: Path.absname("test/fixtures/image_tmp.jpg")
}
assert Local.put_file(file) == :ok
assert Path.join([Local.upload_path(), file_path])
|> File.exists?()
end
end
end

View file

@ -0,0 +1,50 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Uploaders.MDIITest do
use Pleroma.DataCase
alias Pleroma.Uploaders.MDII
import Tesla.Mock
describe "get_file/1" do
test "it returns path to local folder for files" do
assert MDII.get_file("") == {:ok, {:static_dir, "test/uploads"}}
end
end
describe "put_file/1" do
setup do
file_upload = %Pleroma.Upload{
name: "mdii-image.jpg",
content_type: "image/jpg",
path: "test_folder/mdii-image.jpg",
tempfile: Path.absname("test/fixtures/image_tmp.jpg")
}
[file_upload: file_upload]
end
test "save file", %{file_upload: file_upload} do
mock(fn
%{method: :post, url: "https://mdii.sakura.ne.jp/mdii-post.cgi?jpg"} ->
%Tesla.Env{status: 200, body: "mdii-image"}
end)
assert MDII.put_file(file_upload) ==
{:ok, {:url, "https://mdii.sakura.ne.jp/mdii-image.jpg"}}
end
test "save file to local if MDII isn`t available", %{file_upload: file_upload} do
mock(fn
%{method: :post, url: "https://mdii.sakura.ne.jp/mdii-post.cgi?jpg"} ->
%Tesla.Env{status: 500}
end)
assert MDII.put_file(file_upload) == :ok
assert Path.join([Pleroma.Uploaders.Local.upload_path(), file_upload.path])
|> File.exists?()
end
end
end

View file

@ -0,0 +1,90 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Uploaders.S3Test do
use Pleroma.DataCase
alias Pleroma.Config
alias Pleroma.Uploaders.S3
import Mock
import ExUnit.CaptureLog
setup do
config = Config.get([Pleroma.Uploaders.S3])
Config.put([Pleroma.Uploaders.S3],
bucket: "test_bucket",
public_endpoint: "https://s3.amazonaws.com"
)
on_exit(fn ->
Config.put([Pleroma.Uploaders.S3], config)
end)
:ok
end
describe "get_file/1" do
test "it returns path to local folder for files" do
assert S3.get_file("test_image.jpg") == {
:ok,
{:url, "https://s3.amazonaws.com/test_bucket/test_image.jpg"}
}
end
test "it returns path without bucket when truncated_namespace set to ''" do
Config.put([Pleroma.Uploaders.S3],
bucket: "test_bucket",
public_endpoint: "https://s3.amazonaws.com",
truncated_namespace: ""
)
assert S3.get_file("test_image.jpg") == {
:ok,
{:url, "https://s3.amazonaws.com/test_image.jpg"}
}
end
test "it returns path with bucket namespace when namespace is set" do
Config.put([Pleroma.Uploaders.S3],
bucket: "test_bucket",
public_endpoint: "https://s3.amazonaws.com",
bucket_namespace: "family"
)
assert S3.get_file("test_image.jpg") == {
:ok,
{:url, "https://s3.amazonaws.com/family:test_bucket/test_image.jpg"}
}
end
end
describe "put_file/1" do
setup do
file_upload = %Pleroma.Upload{
name: "image-tet.jpg",
content_type: "image/jpg",
path: "test_folder/image-tet.jpg",
tempfile: Path.absname("test/fixtures/image_tmp.jpg")
}
[file_upload: file_upload]
end
test "save file", %{file_upload: file_upload} do
with_mock ExAws, request: fn _ -> {:ok, :ok} end do
assert S3.put_file(file_upload) == {:ok, {:file, "test_folder/image-tet.jpg"}}
end
end
test "returns error", %{file_upload: file_upload} do
with_mock ExAws, request: fn _ -> {:error, "S3 Upload failed"} end do
assert capture_log(fn ->
assert S3.put_file(file_upload) == {:error, "S3 Upload failed"}
end) =~ "Elixir.Pleroma.Uploaders.S3: {:error, \"S3 Upload failed\"}"
end
end
end
end

View file

@ -450,6 +450,27 @@ test "it ensures that address fields become lists" do
assert !is_nil(data["cc"])
end
test "it strips internal likes" do
data =
File.read!("test/fixtures/mastodon-post-activity.json")
|> Poison.decode!()
likes = %{
"first" =>
"http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes?page=1",
"id" => "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes",
"totalItems" => 3,
"type" => "OrderedCollection"
}
object = Map.put(data["object"], "likes", likes)
data = Map.put(data, "object", object)
{:ok, %Activity{object: object}} = Transmogrifier.handle_incoming(data)
refute Map.has_key?(object.data, "likes")
end
test "it works for incoming update activities" do
data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!()
@ -1061,14 +1082,7 @@ test "it strips internal fields of article" do
assert is_nil(modified["object"]["announcements"])
assert is_nil(modified["object"]["announcement_count"])
assert is_nil(modified["object"]["context_id"])
end
test "it adds like collection to object" do
activity = insert(:note_activity)
{:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
assert modified["object"]["likes"]["type"] == "OrderedCollection"
assert modified["object"]["likes"]["totalItems"] == 0
assert is_nil(modified["object"]["likes"])
end
test "the directMessage flag is present" do

View file

@ -2901,8 +2901,10 @@ test "bookmarks" do
describe "conversation muting" do
setup do
post_user = insert(:user)
user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{"status" => "HIE"})
{:ok, activity} = CommonAPI.post(post_user, %{"status" => "HIE"})
[user: user, activity: activity]
end