From e7edfd9fec88af24869c3805a404f2b0a20914de Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 29 May 2019 12:20:18 -0500 Subject: [PATCH 01/45] Permit fetching statuses from API with nickname or id --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 2110027c3..bc75ab35a 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -330,7 +330,7 @@ def public_timeline(%{assigns: %{user: user}} = conn, params) do end def user_statuses(%{assigns: %{user: reading_user}} = conn, params) do - with %User{} = user <- User.get_cached_by_id(params["id"]) do + with %User{} = user <- User.get_cached_by_nickname_or_id(params["id"]) do activities = ActivityPub.fetch_user_activities(user, reading_user, params) conn From e912f81c828cc7e1d2c0dff8daed3ad52f407a61 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 29 May 2019 12:22:14 -0500 Subject: [PATCH 02/45] Update docs to reflect we accept nickname for id for both of these endpoints --- docs/api/differences_in_mastoapi_responses.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index 36b47608e..f952696f7 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -32,7 +32,10 @@ Has these additional fields under the `pleroma` object: ## Accounts -- `/api/v1/accounts/:id`: The `id` parameter can also be the `nickname` of the user. This only works in this endpoint, not the deeper nested ones for following etc. +The `id` parameter can also be the `nickname` of the user. This only works in these endpoints, not the deeper nested ones for following etc. + +- `/api/v1/accounts/:id` +- `/api/v1/accounts/:id/statuses` Has these additional fields under the `pleroma` object: From 46c7c53fbbfd9919d429ba15988af0da9e22d444 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 15 Jul 2019 17:10:33 -0500 Subject: [PATCH 03/45] Add changelog entry for Mastodon API change --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86c90da0b..599ea0a72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API, extension: Ability to reset avatar, profile banner, and background - Mastodon API: Add support for categories for custom emojis by reusing the group feature. - Mastodon API: Add support for muting/unmuting notifications +- Mastodon API: /api/v1/accounts/:id/statuses now supports nicknames or user id - Admin API: Return users' tags when querying reports - Admin API: Return avatar and display name when querying users - Admin API: Allow querying user by ID From 18234cc44e6bc989e3e3cf15714c54b4fa05b9dd Mon Sep 17 00:00:00 2001 From: Sachin Joshi Date: Tue, 16 Jul 2019 22:37:36 +0545 Subject: [PATCH 04/45] add the rich media ttl based on image exp time --- CHANGELOG.md | 1 + config/config.exs | 3 +- ..._set_richmedia_cache_ttl_based_on_image.md | 32 +++++++++++ lib/pleroma/web/rich_media/parser.ex | 41 ++++++++++++++ .../rich_media/parsers/ttl/aws_signed_url.ex | 54 +++++++++++++++++++ test/fixtures/rich_media/amz.html | 5 ++ test/web/rich_media/aws_signed_url_test.exs | 37 +++++++++++++ 7 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 docs/config/howto_set_richmedia_cache_ttl_based_on_image.md create mode 100644 lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex create mode 100644 test/fixtures/rich_media/amz.html create mode 100644 test/web/rich_media/aws_signed_url_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index f3630a1c5..4e58b0a9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text - Admin API: changed json structure for saving config settings. - RichMedia: parsers and their order are configured in `rich_media` config. +- RichMedia: add the rich media ttl based on image expiration time. ## [1.0.1] - 2019-07-14 ### Security diff --git a/config/config.exs b/config/config.exs index 7d539f994..aa5bd0da9 100644 --- a/config/config.exs +++ b/config/config.exs @@ -344,7 +344,8 @@ Pleroma.Web.RichMedia.Parsers.TwitterCard, Pleroma.Web.RichMedia.Parsers.OGP, Pleroma.Web.RichMedia.Parsers.OEmbed - ] + ], + ttl_setters: [Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl] config :pleroma, :media_proxy, enabled: false, diff --git a/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md b/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md new file mode 100644 index 000000000..489f9ece8 --- /dev/null +++ b/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md @@ -0,0 +1,32 @@ +# How to set rich media cache ttl based on image ttl +## Explanation + +Richmedia are cached without the ttl but the rich media may have image which can expire, like aws signed url. +In such cases the old image url (expired) is returned from the media cache. + +So to avoid such situation we can define a moddule that will set ttl based no image. + +The module must have a `run` function and it should be registered in the config. + +### Example + +```exs +defmodule MyModule do + def run(data, url) do + image_url = Map.get(data, :image) + # do some parsing in the url and get the ttl of the image + # ttl is unix time + ttl = parse_ttl_from_url(image_url) + Cachex.expire_at(:rich_media_cache, url, ttl * 1000) + end +end +``` + +And update the config + +```exs +config :pleroma, :rich_media, + ttl_setters: [Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl, MyModule] +``` + +> For reference there is a parser for AWS signed URL `Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl`, it's enabled by default. diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index 0d2523338..ba8dc6f2a 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -24,6 +24,7 @@ def parse(url) do Cachex.fetch!(:rich_media_cache, url, fn _ -> {:commit, parse_url(url)} end) + |> set_ttl_based_on_image(url) rescue e -> {:error, "Cachex error: #{inspect(e)}"} @@ -31,6 +32,46 @@ def parse(url) do end end + @doc """ + Set the rich media cache based on the expiration time of image. + + Define a module that has `run` function + + ## Example + + defmodule MyModule do + def run(data, url) do + image_url = Map.get(data, :image) + # do some parsing in the url and get the ttl of the image + # ttl is unix time + ttl = parse_ttl_from_url(image_url) + Cachex.expire_at(:rich_media_cache, url, ttl * 1000) + end + end + + Define the module in the config + + config :pleroma, :rich_media, + ttl_setters: [MyModule] + """ + def set_ttl_based_on_image({:ok, data}, url) do + case Cachex.ttl(:rich_media_cache, url) do + {:ok, nil} -> + modules = Pleroma.Config.get([:rich_media, :ttl_setters]) + + if Enum.count(modules) > 0 do + Enum.each(modules, & &1.run(data, url)) + end + + {:ok, data} + + _ -> + {:ok, data} + end + end + + def set_ttl_based_on_image(data, _url), do: data + defp parse_url(url) do try do {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options) diff --git a/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex b/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex new file mode 100644 index 000000000..d57107939 --- /dev/null +++ b/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex @@ -0,0 +1,54 @@ +defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do + def run(data, url) do + image = Map.get(data, :image) + + if is_aws_signed_url(image) do + image + |> parse_query_params() + |> format_query_params() + |> get_expiration_timestamp() + |> set_ttl(url) + end + end + + defp is_aws_signed_url(""), do: nil + defp is_aws_signed_url(nil), do: nil + + defp is_aws_signed_url(image) when is_binary(image) do + %URI{host: host, query: query} = URI.parse(image) + + if String.contains?(host, "amazonaws.com") and + String.contains?(query, "X-Amz-Expires") do + image + else + nil + end + end + + defp is_aws_signed_url(_), do: nil + + defp parse_query_params(image) do + %URI{query: query} = URI.parse(image) + query + end + + defp format_query_params(query) do + query + |> String.split(~r/&|=/) + |> Enum.chunk_every(2) + |> Map.new(fn [k, v] -> {k, v} end) + end + + defp get_expiration_timestamp(params) when is_map(params) do + {:ok, date} = + params + |> Map.get("X-Amz-Date") + |> Timex.parse("{ISO:Basic:Z}") + + Timex.to_unix(date) + String.to_integer(Map.get(params, "X-Amz-Expires")) + end + + defp set_ttl(ttl, url) do + Cachex.expire_at(:rich_media_cache, url, ttl * 1000) + end +end diff --git a/test/fixtures/rich_media/amz.html b/test/fixtures/rich_media/amz.html new file mode 100644 index 000000000..d4f8bd1a3 --- /dev/null +++ b/test/fixtures/rich_media/amz.html @@ -0,0 +1,5 @@ + + + + + diff --git a/test/web/rich_media/aws_signed_url_test.exs b/test/web/rich_media/aws_signed_url_test.exs new file mode 100644 index 000000000..75bf6c6df --- /dev/null +++ b/test/web/rich_media/aws_signed_url_test.exs @@ -0,0 +1,37 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.RichMedia.TTL.AwsSignedUrlTest do + use ExUnit.Case, async: true + + test "amazon signed url is parsed and correct ttl is set for rich media" do + url = "https://pleroma.social/amz" + + {:ok, timestamp} = + Timex.now() + |> DateTime.truncate(:second) + |> Timex.format("{ISO:Basic:Z}") + + # in seconds + valid_till = 30 + + data = %{ + image: + "https://pleroma.s3.ap-southeast-1.amazonaws.com/sachin%20%281%29%20_a%20-%25%2Aasdasd%20BNN%20bnnn%20.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIBLWWK6RGDQXDLJQ%2F20190716%2Fap-southeast-1%2Fs3%2Faws4_request&X-Amz-Date=#{ + timestamp + }&X-Amz-Expires=#{valid_till}&X-Amz-Signature=04ffd6b98634f4b1bbabc62e0fac4879093cd54a6eed24fe8eb38e8369526bbf&X-Amz-SignedHeaders=host", + locale: "en_US", + site_name: "Pleroma", + title: "PLeroma", + url: url + } + + Cachex.put(:rich_media_cache, url, data) + assert {:ok, _} = Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl.run(data, url) + {:ok, cache_ttl} = Cachex.ttl(:rich_media_cache, url) + + # as there is delay in setting and pulling the data from cache we ignore 1 second + assert_in_delta(valid_till * 1000, cache_ttl, 1000) + end +end From 88d064d80e4a3272a2a7101089b5f924fd175866 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 18 Jul 2019 15:06:58 +0000 Subject: [PATCH 05/45] http signature plug: remove redundant checks handled by HTTPSignatures library the redundant checks assumed a POST request, which will not work for signed GETs. this check was originally needed because the HTTPSignatures adapter assumed that the requests were also POST requests. but now, the adapter has been corrected. --- lib/pleroma/plugs/http_signature.ex | 51 ++++++++++--------------- test/plugs/http_signature_plug_test.exs | 18 --------- 2 files changed, 21 insertions(+), 48 deletions(-) diff --git a/lib/pleroma/plugs/http_signature.ex b/lib/pleroma/plugs/http_signature.ex index e2874c469..d87fa52fa 100644 --- a/lib/pleroma/plugs/http_signature.ex +++ b/lib/pleroma/plugs/http_signature.ex @@ -3,7 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do - alias Pleroma.Web.ActivityPub.Utils import Plug.Conn require Logger @@ -16,38 +15,30 @@ def call(%{assigns: %{valid_signature: true}} = conn, _opts) do end def call(conn, _opts) do - user = Utils.get_ap_id(conn.params["actor"]) - Logger.debug("Checking sig for #{user}") [signature | _] = get_req_header(conn, "signature") - cond do - signature && String.contains?(signature, user) -> - # set (request-target) header to the appropriate value - # we also replace the digest header with the one we computed - conn = - conn - |> put_req_header( - "(request-target)", - String.downcase("#{conn.method}") <> " #{conn.request_path}" - ) - - conn = - if conn.assigns[:digest] do - conn - |> put_req_header("digest", conn.assigns[:digest]) - else - conn - end - - assign(conn, :valid_signature, HTTPSignatures.validate_conn(conn)) - - signature -> - Logger.debug("Signature not from actor") - assign(conn, :valid_signature, false) - - true -> - Logger.debug("No signature header!") + if signature do + # set (request-target) header to the appropriate value + # we also replace the digest header with the one we computed + conn = conn + |> put_req_header( + "(request-target)", + String.downcase("#{conn.method}") <> " #{conn.request_path}" + ) + + conn = + if conn.assigns[:digest] do + conn + |> put_req_header("digest", conn.assigns[:digest]) + else + conn + end + + assign(conn, :valid_signature, HTTPSignatures.validate_conn(conn)) + else + Logger.debug("No signature header!") + conn end end end diff --git a/test/plugs/http_signature_plug_test.exs b/test/plugs/http_signature_plug_test.exs index efd811df7..d6fd9ea81 100644 --- a/test/plugs/http_signature_plug_test.exs +++ b/test/plugs/http_signature_plug_test.exs @@ -26,22 +26,4 @@ test "it call HTTPSignatures to check validity if the actor sighed it" do assert called(HTTPSignatures.validate_conn(:_)) end end - - test "bails out early if the signature isn't by the activity actor" do - params = %{"actor" => "https://mst3k.interlinked.me/users/luciferMysticus"} - conn = build_conn(:get, "/doesntmattter", params) - - with_mock HTTPSignatures, validate_conn: fn _ -> false end do - conn = - conn - |> put_req_header( - "signature", - "keyId=\"http://mastodon.example.org/users/admin#main-key" - ) - |> HTTPSignaturePlug.call(%{}) - - assert conn.assigns.valid_signature == false - refute called(HTTPSignatures.validate_conn(:_)) - end - end end From 18d8d12d53567b7c0c246bb793ee724d0d2e4c77 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 18 Jul 2019 15:35:42 +0000 Subject: [PATCH 06/45] signature: make key_id_to_actor_id() public --- lib/pleroma/signature.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex index a45c70a9d..2a0823ecf 100644 --- a/lib/pleroma/signature.ex +++ b/lib/pleroma/signature.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Signature do alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub - defp key_id_to_actor_id(key_id) do + def key_id_to_actor_id(key_id) do URI.parse(key_id) |> Map.put(:fragment, nil) |> URI.to_string() From 184fa61fb3a1bc8c5d5515bb7748c12816b11ebf Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 18 Jul 2019 15:38:45 +0000 Subject: [PATCH 07/45] plugs: add MappedSignatureToIdentityPlug --- .../mapped_signature_to_identity_plug.ex | 64 +++++++++++++++++++ lib/pleroma/web/router.ex | 1 + 2 files changed, 65 insertions(+) create mode 100644 lib/pleroma/plugs/mapped_signature_to_identity_plug.ex diff --git a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex new file mode 100644 index 000000000..ae9339595 --- /dev/null +++ b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex @@ -0,0 +1,64 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlug do + alias Pleroma.Signature + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Utils + + import Plug.Conn + require Logger + + def init(options), do: options + + defp key_id_from_conn(conn) do + with %{"keyId" => key_id} <- HTTPSignatures.signature_for_conn(conn) do + Signature.key_id_to_actor_id(key_id) + else + _ -> + nil + end + end + + defp user_from_key_id(conn) do + with key_actor_id when is_binary(key_actor_id) <- key_id_from_conn(conn), + %User{} = user <- User.get_or_fetch_by_ap_id(key_actor_id) do + user + else + _ -> + nil + end + end + + def call(%{assigns: %{mapped_identity: _}} = conn, _opts), do: conn + + # if this has payload make sure it is signed by the same actor that made it + def call(%{assigns: %{valid_signature: true}, params: %{"actor" => actor}} = conn, _opts) do + with actor_id <- Utils.get_ap_id(actor), + %User{} = user <- user_from_key_id(conn), + true <- user.ap_id == actor_id do + assign(conn, :mapped_identity, user) + else + _ -> + Logger.debug("Failed to map identity from signature (payload actor mismatch?)") + Logger.debug("key_id=#{key_id_from_conn(conn)}, actor=#{actor}") + conn + end + end + + # no payload, probably a signed fetch + def call(%{assigns: %{valid_signature: true}} = conn, _opts) do + with %User{} = user <- user_from_key_id(conn) do + assign(conn, :mapped_identity, user) + else + _ -> + Logger.debug("Failed to map identity from signature (no payload actor mismatch)") + Logger.debug("key_id=#{key_id_from_conn(conn)}") + conn + end + end + + # no signature at all + def call(conn, _opts), do: conn +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 8095ac4b1..518720d38 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -617,6 +617,7 @@ defmodule Pleroma.Web.Router do pipeline :activitypub do plug(:accepts, ["activity+json", "json"]) plug(Pleroma.Web.Plugs.HTTPSignaturePlug) + plug(Pleroma.Web.Plugs.MappedSignatureToIdentityPlug) end scope "/", Pleroma.Web.ActivityPub do From cdf0038d0f5d56cf78f640fb149e9141e74800fc Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 18 Jul 2019 15:51:58 +0000 Subject: [PATCH 08/45] mix: update http signatures dependency --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index e4ea19bb6..c12b0a500 100644 --- a/mix.exs +++ b/mix.exs @@ -138,7 +138,7 @@ defp deps do ref: "95e8188490e97505c56636c1379ffdf036c1fdde"}, {:http_signatures, git: "https://git.pleroma.social/pleroma/http_signatures.git", - ref: "a2a5982fa167fb1352fbd518ce6b606ba233a989"}, + ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"}, {:pleroma_job_queue, "~> 0.2.0"}, {:telemetry, "~> 0.3"}, {:prometheus_ex, "~> 3.0"}, diff --git a/mix.lock b/mix.lock index 6477c1ed5..45142ba8f 100644 --- a/mix.lock +++ b/mix.lock @@ -38,7 +38,7 @@ "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, "html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, - "http_signatures": {:git, "https://git.pleroma.social/pleroma/http_signatures.git", "a2a5982fa167fb1352fbd518ce6b606ba233a989", [ref: "a2a5982fa167fb1352fbd518ce6b606ba233a989"]}, + "http_signatures": {:git, "https://git.pleroma.social/pleroma/http_signatures.git", "293d77bb6f4a67ac8bde1428735c3b42f22cbb30", [ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"]}, "httpoison": {:hex, :httpoison, "1.2.0", "2702ed3da5fd7a8130fc34b11965c8cfa21ade2f232c00b42d96d4967c39a3a3", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"}, "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"}, From 5ea0cd69f7457086fc486f13e072f13d2c1ef547 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 18 Jul 2019 16:01:21 +0000 Subject: [PATCH 09/45] mapped signature plug: don't invalidate in cases where a signature is actually not present (testsuite) --- .../plugs/mapped_signature_to_identity_plug.ex | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex index ae9339595..2a8ed4470 100644 --- a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex +++ b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex @@ -36,12 +36,18 @@ def call(%{assigns: %{mapped_identity: _}} = conn, _opts), do: conn # if this has payload make sure it is signed by the same actor that made it def call(%{assigns: %{valid_signature: true}, params: %{"actor" => actor}} = conn, _opts) do with actor_id <- Utils.get_ap_id(actor), - %User{} = user <- user_from_key_id(conn), - true <- user.ap_id == actor_id do + {:user, %User{} = user} <- {:user, user_from_key_id(conn)}, + {:user_match, true} <- {:user_match, user.ap_id == actor_id} do assign(conn, :mapped_identity, user) else - _ -> - Logger.debug("Failed to map identity from signature (payload actor mismatch?)") + {:user_match, false} -> + Logger.debug("Failed to map identity from signature (payload actor mismatch)") + Logger.debug("key_id=#{key_id_from_conn(conn)}, actor=#{actor}") + assign(conn, :valid_signature, false) + + # remove me once testsuite uses mapped capabilities instead of what we do now + {:user, nil} -> + Logger.debug("Failed to map identity from signature (lookup failure)") Logger.debug("key_id=#{key_id_from_conn(conn)}, actor=#{actor}") conn end @@ -55,7 +61,7 @@ def call(%{assigns: %{valid_signature: true}} = conn, _opts) do _ -> Logger.debug("Failed to map identity from signature (no payload actor mismatch)") Logger.debug("key_id=#{key_id_from_conn(conn)}") - conn + assign(conn, :valid_signature, false) end end From a8af0ac053713102204418fe7a28d322f81eb3ea Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 18 Jul 2019 16:27:50 +0000 Subject: [PATCH 10/45] mapped signature plug: fix user lookup --- lib/pleroma/plugs/mapped_signature_to_identity_plug.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex index 2a8ed4470..1e7da4f50 100644 --- a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex +++ b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex @@ -23,7 +23,7 @@ defp key_id_from_conn(conn) do defp user_from_key_id(conn) do with key_actor_id when is_binary(key_actor_id) <- key_id_from_conn(conn), - %User{} = user <- User.get_or_fetch_by_ap_id(key_actor_id) do + {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(key_actor_id) do user else _ -> From 621cacf667e7d5519a2731fadf86c429f70f4489 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 18 Jul 2019 16:28:36 +0000 Subject: [PATCH 11/45] tests: add tests for mapped signature plug --- ...mapped_identity_to_signature_plug_test.exs | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 test/plugs/mapped_identity_to_signature_plug_test.exs diff --git a/test/plugs/mapped_identity_to_signature_plug_test.exs b/test/plugs/mapped_identity_to_signature_plug_test.exs new file mode 100644 index 000000000..9aca534e1 --- /dev/null +++ b/test/plugs/mapped_identity_to_signature_plug_test.exs @@ -0,0 +1,59 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlugTest do + use Pleroma.Web.ConnCase + alias Pleroma.Web.Plugs.MappedSignatureToIdentityPlug + + import Tesla.Mock + import Plug.Conn + + setup do + mock(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + defp set_signature(conn, key_id) do + conn + |> put_req_header("signature", "keyId=\"#{key_id}\"") + |> assign(:valid_signature, true) + end + + test "it successfully maps a valid identity with a valid signature" do + conn = + build_conn(:get, "/doesntmattter") + |> set_signature("http://mastodon.example.org/users/admin") + |> MappedSignatureToIdentityPlug.call(%{}) + + refute is_nil(conn.assigns.mapped_identity) + end + + test "it successfully maps a valid identity with a valid signature with payload" do + conn = + build_conn(:post, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"}) + |> set_signature("http://mastodon.example.org/users/admin") + |> MappedSignatureToIdentityPlug.call(%{}) + + refute is_nil(conn.assigns.mapped_identity) + end + + test "it considers a mapped identity to be invalid when it mismatches a payload" do + conn = + build_conn(:post, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"}) + |> set_signature("https://niu.moe/users/rye") + |> MappedSignatureToIdentityPlug.call(%{}) + + assert %{valid_signature: false} == conn.assigns + end + + @tag skip: "known breakage; the testsuite presently depends on it" + test "it considers a mapped identity to be invalid when the identity cannot be found" do + conn = + build_conn(:post, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"}) + |> set_signature("http://niu.moe/users/rye") + |> MappedSignatureToIdentityPlug.call(%{}) + + assert %{valid_signature: false} == conn.assigns + end +end From c947cfec5ab49f90fb2de83f61bda77568298d6f Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 18 Jul 2019 20:31:25 +0000 Subject: [PATCH 12/45] mapped signature plug: use `user` assign like authentication plug --- lib/pleroma/plugs/mapped_signature_to_identity_plug.ex | 6 +++--- test/plugs/mapped_identity_to_signature_plug_test.exs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex index 1e7da4f50..ce8494b9d 100644 --- a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex +++ b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex @@ -31,14 +31,14 @@ defp user_from_key_id(conn) do end end - def call(%{assigns: %{mapped_identity: _}} = conn, _opts), do: conn + def call(%{assigns: %{user: _}} = conn, _opts), do: conn # if this has payload make sure it is signed by the same actor that made it def call(%{assigns: %{valid_signature: true}, params: %{"actor" => actor}} = conn, _opts) do with actor_id <- Utils.get_ap_id(actor), {:user, %User{} = user} <- {:user, user_from_key_id(conn)}, {:user_match, true} <- {:user_match, user.ap_id == actor_id} do - assign(conn, :mapped_identity, user) + assign(conn, :user, user) else {:user_match, false} -> Logger.debug("Failed to map identity from signature (payload actor mismatch)") @@ -56,7 +56,7 @@ def call(%{assigns: %{valid_signature: true}, params: %{"actor" => actor}} = con # no payload, probably a signed fetch def call(%{assigns: %{valid_signature: true}} = conn, _opts) do with %User{} = user <- user_from_key_id(conn) do - assign(conn, :mapped_identity, user) + assign(conn, :user, user) else _ -> Logger.debug("Failed to map identity from signature (no payload actor mismatch)") diff --git a/test/plugs/mapped_identity_to_signature_plug_test.exs b/test/plugs/mapped_identity_to_signature_plug_test.exs index 9aca534e1..bb45d9edf 100644 --- a/test/plugs/mapped_identity_to_signature_plug_test.exs +++ b/test/plugs/mapped_identity_to_signature_plug_test.exs @@ -26,7 +26,7 @@ test "it successfully maps a valid identity with a valid signature" do |> set_signature("http://mastodon.example.org/users/admin") |> MappedSignatureToIdentityPlug.call(%{}) - refute is_nil(conn.assigns.mapped_identity) + refute is_nil(conn.assigns.user) end test "it successfully maps a valid identity with a valid signature with payload" do @@ -35,7 +35,7 @@ test "it successfully maps a valid identity with a valid signature with payload" |> set_signature("http://mastodon.example.org/users/admin") |> MappedSignatureToIdentityPlug.call(%{}) - refute is_nil(conn.assigns.mapped_identity) + refute is_nil(conn.assigns.user) end test "it considers a mapped identity to be invalid when it mismatches a payload" do From de9906ad56bd25d6c8c38bef1307192df2e95445 Mon Sep 17 00:00:00 2001 From: Sachin Joshi Date: Fri, 19 Jul 2019 11:43:42 +0545 Subject: [PATCH 13/45] change the structure of image ttl parsar --- ..._set_richmedia_cache_ttl_based_on_image.md | 2 +- lib/pleroma/web/rich_media/parser.ex | 36 +++++----- .../rich_media/parsers/ttl/aws_signed_url.ex | 10 ++- lib/pleroma/web/rich_media/parsers/ttl/ttl.ex | 3 + test/web/rich_media/aws_signed_url_test.exs | 70 +++++++++++++++---- 5 files changed, 85 insertions(+), 36 deletions(-) create mode 100644 lib/pleroma/web/rich_media/parsers/ttl/ttl.ex diff --git a/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md b/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md index 489f9ece8..5846b6ab0 100644 --- a/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md +++ b/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md @@ -4,7 +4,7 @@ Richmedia are cached without the ttl but the rich media may have image which can expire, like aws signed url. In such cases the old image url (expired) is returned from the media cache. -So to avoid such situation we can define a moddule that will set ttl based no image. +So to avoid such situation we can define a moddule that will set ttl based on image. The module must have a `run` function and it should be registered in the config. diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index ba8dc6f2a..b69b2be61 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -35,17 +35,17 @@ def parse(url) do @doc """ Set the rich media cache based on the expiration time of image. - Define a module that has `run` function + Adopt behaviour `Pleroma.Web.RichMedia.Parser.TTL` ## Example defmodule MyModule do - def run(data, url) do + @behaviour Pleroma.Web.RichMedia.Parser.TTL + def ttl(data, url) do image_url = Map.get(data, :image) # do some parsing in the url and get the ttl of the image - # ttl is unix time - ttl = parse_ttl_from_url(image_url) - Cachex.expire_at(:rich_media_cache, url, ttl * 1000) + # and return ttl is unix time + parse_ttl_from_url(image_url) end end @@ -55,22 +55,26 @@ def run(data, url) do ttl_setters: [MyModule] """ def set_ttl_based_on_image({:ok, data}, url) do - case Cachex.ttl(:rich_media_cache, url) do - {:ok, nil} -> - modules = Pleroma.Config.get([:rich_media, :ttl_setters]) - - if Enum.count(modules) > 0 do - Enum.each(modules, & &1.run(data, url)) - end - - {:ok, data} - + with {:ok, nil} <- Cachex.ttl(:rich_media_cache, url) do + ttl = get_ttl_from_image(data, url) + Cachex.expire_at(:rich_media_cache, url, ttl * 1000) + {:ok, data} + else _ -> {:ok, data} end end - def set_ttl_based_on_image(data, _url), do: data + defp get_ttl_from_image(data, url) do + Pleroma.Config.get([:rich_media, :ttl_setters]) + |> Enum.reduce({:ok, nil}, fn + module, {:ok, _ttl} -> + module.ttl(data, url) + + _, error -> + error + end) + end defp parse_url(url) do try do diff --git a/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex b/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex index d57107939..014c0935f 100644 --- a/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex +++ b/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex @@ -1,5 +1,8 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do - def run(data, url) do + @behaviour Pleroma.Web.RichMedia.Parser.TTL + + @impl Pleroma.Web.RichMedia.Parser.TTL + def ttl(data, _url) do image = Map.get(data, :image) if is_aws_signed_url(image) do @@ -7,7 +10,6 @@ def run(data, url) do |> parse_query_params() |> format_query_params() |> get_expiration_timestamp() - |> set_ttl(url) end end @@ -47,8 +49,4 @@ defp get_expiration_timestamp(params) when is_map(params) do Timex.to_unix(date) + String.to_integer(Map.get(params, "X-Amz-Expires")) end - - defp set_ttl(ttl, url) do - Cachex.expire_at(:rich_media_cache, url, ttl * 1000) - end end diff --git a/lib/pleroma/web/rich_media/parsers/ttl/ttl.ex b/lib/pleroma/web/rich_media/parsers/ttl/ttl.ex new file mode 100644 index 000000000..6b3ec6d30 --- /dev/null +++ b/lib/pleroma/web/rich_media/parsers/ttl/ttl.ex @@ -0,0 +1,3 @@ +defmodule Pleroma.Web.RichMedia.Parser.TTL do + @callback ttl(Map.t(), String.t()) :: {:ok, Integer.t()} | {:error, String.t()} +end diff --git a/test/web/rich_media/aws_signed_url_test.exs b/test/web/rich_media/aws_signed_url_test.exs index 75bf6c6df..122787bc2 100644 --- a/test/web/rich_media/aws_signed_url_test.exs +++ b/test/web/rich_media/aws_signed_url_test.exs @@ -5,7 +5,7 @@ defmodule Pleroma.Web.RichMedia.TTL.AwsSignedUrlTest do use ExUnit.Case, async: true - test "amazon signed url is parsed and correct ttl is set for rich media" do + test "s3 signed url is parsed correct for expiration time" do url = "https://pleroma.social/amz" {:ok, timestamp} = @@ -16,22 +16,66 @@ test "amazon signed url is parsed and correct ttl is set for rich media" do # in seconds valid_till = 30 - data = %{ - image: - "https://pleroma.s3.ap-southeast-1.amazonaws.com/sachin%20%281%29%20_a%20-%25%2Aasdasd%20BNN%20bnnn%20.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIBLWWK6RGDQXDLJQ%2F20190716%2Fap-southeast-1%2Fs3%2Faws4_request&X-Amz-Date=#{ - timestamp - }&X-Amz-Expires=#{valid_till}&X-Amz-Signature=04ffd6b98634f4b1bbabc62e0fac4879093cd54a6eed24fe8eb38e8369526bbf&X-Amz-SignedHeaders=host", - locale: "en_US", - site_name: "Pleroma", - title: "PLeroma", - url: url - } + metadata = construct_metadata(timestamp, valid_till, url) + + expire_time = + Timex.parse!(timestamp, "{ISO:Basic:Z}") |> Timex.to_unix() |> Kernel.+(valid_till) + + assert expire_time == Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl.ttl(metadata, url) + end + + test "s3 signed url is parsed and correct ttl is set for rich media" do + url = "https://pleroma.social/amz" + + {:ok, timestamp} = + Timex.now() + |> DateTime.truncate(:second) + |> Timex.format("{ISO:Basic:Z}") + + # in seconds + valid_till = 30 + + metadata = construct_metadata(timestamp, valid_till, url) + + body = """ + + + + + + """ + + Tesla.Mock.mock(fn + %{ + method: :get, + url: "https://pleroma.social/amz" + } -> + %Tesla.Env{status: 200, body: body} + end) + + Cachex.put(:rich_media_cache, url, metadata) + + Pleroma.Web.RichMedia.Parser.set_ttl_based_on_image({:ok, metadata}, url) - Cachex.put(:rich_media_cache, url, data) - assert {:ok, _} = Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl.run(data, url) {:ok, cache_ttl} = Cachex.ttl(:rich_media_cache, url) # as there is delay in setting and pulling the data from cache we ignore 1 second assert_in_delta(valid_till * 1000, cache_ttl, 1000) end + + defp construct_s3_url(timestamp, valid_till) do + "https://pleroma.s3.ap-southeast-1.amazonaws.com/sachin%20%281%29%20_a%20-%25%2Aasdasd%20BNN%20bnnn%20.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIBLWWK6RGDQXDLJQ%2F20190716%2Fap-southeast-1%2Fs3%2Faws4_request&X-Amz-Date=#{ + timestamp + }&X-Amz-Expires=#{valid_till}&X-Amz-Signature=04ffd6b98634f4b1bbabc62e0fac4879093cd54a6eed24fe8eb38e8369526bbf&X-Amz-SignedHeaders=host" + end + + defp construct_metadata(timestamp, valid_till, url) do + %{ + image: construct_s3_url(timestamp, valid_till), + site: "Pleroma", + title: "Pleroma", + description: "Pleroma", + url: url + } + end end From 581756ccc50cc08823957a2f24f506bf23c7cd22 Mon Sep 17 00:00:00 2001 From: Sachin Joshi Date: Fri, 19 Jul 2019 11:50:47 +0545 Subject: [PATCH 14/45] update the docs --- ...owto_set_richmedia_cache_ttl_based_on_image.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md b/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md index 5846b6ab0..bfee5a9e6 100644 --- a/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md +++ b/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md @@ -4,20 +4,21 @@ Richmedia are cached without the ttl but the rich media may have image which can expire, like aws signed url. In such cases the old image url (expired) is returned from the media cache. -So to avoid such situation we can define a moddule that will set ttl based on image. - -The module must have a `run` function and it should be registered in the config. +So to avoid such situation we can define a module that will set ttl based on image. +The module must adopt behaviour `Pleroma.Web.RichMedia.Parser.TTL` ### Example ```exs defmodule MyModule do - def run(data, url) do + @behaviour Pleroma.Web.RichMedia.Parser.TTL + + @impl Pleroma.Web.RichMedia.Parser.TTL + def ttl(data, url) do image_url = Map.get(data, :image) # do some parsing in the url and get the ttl of the image - # ttl is unix time - ttl = parse_ttl_from_url(image_url) - Cachex.expire_at(:rich_media_cache, url, ttl * 1000) + # return ttl is unix time + parse_ttl_from_url(image_url) end end ``` From b0b9eca37d43c8cc5e9e0e01fa4b1325a71ce4d2 Mon Sep 17 00:00:00 2001 From: tom79 Date: Fri, 19 Jul 2019 08:39:22 +0000 Subject: [PATCH 15/45] Update clients.md for Fedilab - Change owner (@fedilab@framapiaf.org), Source code: Framagit, Add some other supported features --- docs/clients.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/clients.md b/docs/clients.md index 91096970e..dc4ab37b1 100644 --- a/docs/clients.md +++ b/docs/clients.md @@ -32,10 +32,10 @@ Feel free to contact us to be added to this list! ### Fedilab - Homepage: -- Source Code: -- Contact: [@tom79@mastodon.social](https://mastodon.social/users/tom79) +- Source Code: +- Contact: [@fedilab@mastodon.social](https://framapiaf.org/users/fedilab) - Platforms: Android -- Features: Streaming Ready +- Features: Streaming Ready, Moderation, Text Formatting ### Nekonium - Homepage: [F-Droid Repository](https://repo.gdgd.jp.net/), [Google Play](https://play.google.com/store/apps/details?id=com.apps.nekonium), [Amazon](https://www.amazon.co.jp/dp/B076FXPRBC/) From 5c4a555b1da1d2cb6649a9384e386a48bbcf905f Mon Sep 17 00:00:00 2001 From: tom79 Date: Fri, 19 Jul 2019 08:40:47 +0000 Subject: [PATCH 16/45] Fix domain for the contact clients.md --- docs/clients.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/clients.md b/docs/clients.md index dc4ab37b1..9029361f8 100644 --- a/docs/clients.md +++ b/docs/clients.md @@ -33,7 +33,7 @@ Feel free to contact us to be added to this list! ### Fedilab - Homepage: - Source Code: -- Contact: [@fedilab@mastodon.social](https://framapiaf.org/users/fedilab) +- Contact: [@fedilab@framapiaf.org](https://framapiaf.org/users/fedilab) - Platforms: Android - Features: Streaming Ready, Moderation, Text Formatting From c2e2aadc4254fe931ea519a9813854ccdac456b8 Mon Sep 17 00:00:00 2001 From: Maksim Date: Fri, 19 Jul 2019 16:20:23 +0000 Subject: [PATCH 17/45] #1110 fixed /api/pleroma/healthcheck --- .../controllers/util_controller.ex | 32 ++++++---- test/web/twitter_api/util_controller_test.exs | 64 ++++++++++++++++++- 2 files changed, 79 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index c10c66ff2..9e4da7dca 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -8,7 +8,9 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do require Logger alias Pleroma.Activity + alias Pleroma.Config alias Pleroma.Emoji + alias Pleroma.Healthcheck alias Pleroma.Notification alias Pleroma.Plugs.AuthenticationPlug alias Pleroma.User @@ -23,7 +25,8 @@ def help_test(conn, _params) do end def remote_subscribe(conn, %{"nickname" => nick, "profile" => _}) do - with %User{} = user <- User.get_cached_by_nickname(nick), avatar = User.avatar_url(user) do + with %User{} = user <- User.get_cached_by_nickname(nick), + avatar = User.avatar_url(user) do conn |> render("subscribe.html", %{nickname: nick, avatar: avatar, error: false}) else @@ -338,20 +341,21 @@ def captcha(conn, _params) do end def healthcheck(conn, _params) do - info = - if Pleroma.Config.get([:instance, :healthcheck]) do - Pleroma.Healthcheck.system_info() - else - %{} - end + with true <- Config.get([:instance, :healthcheck]), + %{healthy: true} = info <- Healthcheck.system_info() do + json(conn, info) + else + %{healthy: false} = info -> + service_unavailable(conn, info) - conn = - if info[:healthy] do - conn - else - Plug.Conn.put_status(conn, :service_unavailable) - end + _ -> + service_unavailable(conn, %{}) + end + end - json(conn, info) + defp service_unavailable(conn, info) do + conn + |> put_status(:service_unavailable) + |> json(info) end end diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index 21324399f..3d699e1df 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -10,6 +10,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do alias Pleroma.User alias Pleroma.Web.CommonAPI import Pleroma.Factory + import Mock setup do Tesla.Mock.mock(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -231,10 +232,67 @@ test "show follow account page if the `acct` is a account link", %{conn: conn} d end end - test "GET /api/pleroma/healthcheck", %{conn: conn} do - conn = get(conn, "/api/pleroma/healthcheck") + describe "GET /api/pleroma/healthcheck" do + setup do + config_healthcheck = Pleroma.Config.get([:instance, :healthcheck]) - assert conn.status in [200, 503] + on_exit(fn -> + Pleroma.Config.put([:instance, :healthcheck], config_healthcheck) + end) + + :ok + end + + test "returns 503 when healthcheck disabled", %{conn: conn} do + Pleroma.Config.put([:instance, :healthcheck], false) + + response = + conn + |> get("/api/pleroma/healthcheck") + |> json_response(503) + + assert response == %{} + end + + test "returns 200 when healthcheck enabled and all ok", %{conn: conn} do + Pleroma.Config.put([:instance, :healthcheck], true) + + with_mock Pleroma.Healthcheck, + system_info: fn -> %Pleroma.Healthcheck{healthy: true} end do + response = + conn + |> get("/api/pleroma/healthcheck") + |> json_response(200) + + assert %{ + "active" => _, + "healthy" => true, + "idle" => _, + "memory_used" => _, + "pool_size" => _ + } = response + end + end + + test "returns 503 when healthcheck enabled and health is false", %{conn: conn} do + Pleroma.Config.put([:instance, :healthcheck], true) + + with_mock Pleroma.Healthcheck, + system_info: fn -> %Pleroma.Healthcheck{healthy: false} end do + response = + conn + |> get("/api/pleroma/healthcheck") + |> json_response(503) + + assert %{ + "active" => _, + "healthy" => false, + "idle" => _, + "memory_used" => _, + "pool_size" => _ + } = response + end + end end describe "POST /api/pleroma/disable_account" do From 9a8eb2c94d2243fadd69786eb74d94cc6116468f Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 19 Jul 2019 19:11:04 +0000 Subject: [PATCH 18/45] mix: add pleroma.user unsubscribe_all_from_instance --- lib/mix/tasks/pleroma/user.ex | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index 8a78b4fe6..c9b84b8f9 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -62,6 +62,10 @@ defmodule Mix.Tasks.Pleroma.User do mix pleroma.user unsubscribe NICKNAME + ## Unsubscribe local users from an entire instance and deactivate all accounts + + mix pleroma.user unsubscribe_all_from_instance INSTANCE + ## Create a password reset link. mix pleroma.user reset_password NICKNAME @@ -246,6 +250,20 @@ def run(["unsubscribe", nickname]) do end end + def run(["unsubscribe_all_from_instance", instance]) do + start_pleroma() + + Pleroma.User.Query.build(%{nickname: "@#{instance}"}) + |> Pleroma.RepoStreamer.chunk_stream(500) + |> Stream.each(fn users -> + users + |> Enum.each(fn user -> + run(["unsubscribe", user.nickname]) + end) + end) + |> Stream.run() + end + def run(["set", nickname | rest]) do start_pleroma() From d0198fe215a7542ce506e40e1e4860a27ee2d01e Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 20 Jul 2019 13:03:34 +0300 Subject: [PATCH 19/45] [#1112] Preserving `id` on user insert conflict on order not to violate conversation_partipations_user_id_fkey constraint. --- CHANGELOG.md | 1 + lib/pleroma/user.ex | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e79b5420..f60f3ed97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Embedded relationships not being properly rendered in the Account entity of Status entity - Mastodon API: Add `account_id`, `type`, `offset`, and `limit` to search API (`/api/v1/search` and `/api/v2/search`) - ActivityPub C2S: follower/following collection pages being inaccessible even when authentifucated if `hide_followers`/ `hide_follows` was set +- Existing user id not being preserved on insert conflict ### Added - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index c91fbb68a..5ea2b518b 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1211,7 +1211,7 @@ def insert_or_update_user(data) do data |> Map.put(:name, blank?(data[:name]) || data[:nickname]) |> remote_user_creation() - |> Repo.insert(on_conflict: :replace_all, conflict_target: :nickname) + |> Repo.insert(on_conflict: :replace_all_except_primary_key, conflict_target: :nickname) |> set_cache() end From 43a7cd27feb6ef52e4b130a974af3f819a9b266d Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Sat, 20 Jul 2019 13:07:51 +0000 Subject: [PATCH 20/45] [tests] Mock :crypt.crypt/2 function in AuthenticationPlugTest --- test/plugs/authentication_plug_test.exs | 12 ++++-- test/signature_test.exs | 13 +++++-- test/upload_test.exs | 16 ++++++-- .../mastodon_api/search_controller_test.exs | 30 ++++++++------ test/web/ostatus/ostatus_controller_test.exs | 39 ++++++++++++------- 5 files changed, 73 insertions(+), 37 deletions(-) diff --git a/test/plugs/authentication_plug_test.exs b/test/plugs/authentication_plug_test.exs index b55e746f8..7ca045616 100644 --- a/test/plugs/authentication_plug_test.exs +++ b/test/plugs/authentication_plug_test.exs @@ -8,6 +8,9 @@ defmodule Pleroma.Plugs.AuthenticationPlugTest do alias Pleroma.Plugs.AuthenticationPlug alias Pleroma.User + import ExUnit.CaptureLog + import Mock + setup %{conn: conn} do user = %User{ id: 1, @@ -68,15 +71,18 @@ test "check sha512-crypt hash" do hash = "$6$9psBWV8gxkGOZWBz$PmfCycChoxeJ3GgGzwvhlgacb9mUoZ.KUXNCssekER4SJ7bOK53uXrHNb2e4i8yPFgSKyzaW9CcmrDXWIEMtD1" - assert AuthenticationPlug.checkpw("password", hash) - refute AuthenticationPlug.checkpw("password1", hash) + with_mock :crypt, crypt: fn _password, password_hash -> password_hash end do + assert AuthenticationPlug.checkpw("password", hash) + end end test "it returns false when hash invalid" do hash = "psBWV8gxkGOZWBz$PmfCycChoxeJ3GgGzwvhlgacb9mUoZ.KUXNCssekER4SJ7bOK53uXrHNb2e4i8yPFgSKyzaW9CcmrDXWIEMtD1" - refute Pleroma.Plugs.AuthenticationPlug.checkpw("password", hash) + assert capture_log(fn -> + refute Pleroma.Plugs.AuthenticationPlug.checkpw("password", hash) + end) =~ "[error] Password hash not recognized" end end end diff --git a/test/signature_test.exs b/test/signature_test.exs index 840987cd6..7400cae9a 100644 --- a/test/signature_test.exs +++ b/test/signature_test.exs @@ -5,6 +5,7 @@ defmodule Pleroma.SignatureTest do use Pleroma.DataCase + import ExUnit.CaptureLog import Pleroma.Factory import Tesla.Mock @@ -46,8 +47,10 @@ test "it returns key" do end test "it returns error when not found user" do - assert Signature.fetch_public_key(make_fake_conn("test-ap_id")) == - {:error, :error} + assert capture_log(fn -> + assert Signature.fetch_public_key(make_fake_conn("test-ap_id")) == + {:error, :error} + end) =~ "[error] Could not decode user" end test "it returns error if public key is empty" do @@ -67,8 +70,10 @@ test "it returns key" do end test "it returns error when not found user" do - assert Signature.refetch_public_key(make_fake_conn("test-ap_id")) == - {:error, {:error, :ok}} + assert capture_log(fn -> + assert Signature.refetch_public_key(make_fake_conn("test-ap_id")) == + {:error, {:error, :ok}} + end) =~ "[error] Could not decode user" end end diff --git a/test/upload_test.exs b/test/upload_test.exs index f7b1893ad..32c6977d1 100644 --- a/test/upload_test.exs +++ b/test/upload_test.exs @@ -5,6 +5,8 @@ defmodule Pleroma.UploadTest do use Pleroma.DataCase + import ExUnit.CaptureLog + alias Pleroma.Upload alias Pleroma.Uploaders.Uploader @@ -77,8 +79,12 @@ def put_file(upload), do: TestUploaderBase.put_file(upload, __MODULE__) test "it returns error" do File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") - assert Upload.store(@upload_file) == {:error, "Errors"} - Task.await(Agent.get(TestUploaderError, fn task_pid -> task_pid end)) + + assert capture_log(fn -> + assert Upload.store(@upload_file) == {:error, "Errors"} + Task.await(Agent.get(TestUploaderError, fn task_pid -> task_pid end)) + end) =~ + "[error] Elixir.Pleroma.Upload store (using Pleroma.UploadTest.TestUploaderError) failed: \"Errors\"" end end @@ -89,7 +95,11 @@ test "it returns error" do test "it returns error" do File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") - assert Upload.store(@upload_file) == {:error, "Uploader callback timeout"} + + assert capture_log(fn -> + assert Upload.store(@upload_file) == {:error, "Uploader callback timeout"} + end) =~ + "[error] Elixir.Pleroma.Upload store (using Pleroma.UploadTest.TestUploader) failed: \"Uploader callback timeout\"" end end diff --git a/test/web/mastodon_api/search_controller_test.exs b/test/web/mastodon_api/search_controller_test.exs index 9f50c09f4..043b96c14 100644 --- a/test/web/mastodon_api/search_controller_test.exs +++ b/test/web/mastodon_api/search_controller_test.exs @@ -24,12 +24,16 @@ test "it returns empty result if user or status search return undefined error", {Pleroma.User, [], [search: fn _q, _o -> raise "Oops" end]}, {Pleroma.Activity, [], [search: fn _u, _q, _o -> raise "Oops" end]} ] do - conn = get(conn, "/api/v2/search", %{"q" => "2hu"}) + capture_log(fn -> + results = + conn + |> get("/api/v2/search", %{"q" => "2hu"}) + |> json_response(200) - assert results = json_response(conn, 200) - - assert results["accounts"] == [] - assert results["statuses"] == [] + assert results["accounts"] == [] + assert results["statuses"] == [] + end) =~ + "[error] Elixir.Pleroma.Web.MastodonAPI.SearchController search error: %RuntimeError{message: \"Oops\"}" end end @@ -99,14 +103,16 @@ test "it returns empty result if user or status search return undefined error", {Pleroma.User, [], [search: fn _q, _o -> raise "Oops" end]}, {Pleroma.Activity, [], [search: fn _u, _q, _o -> raise "Oops" end]} ] do - conn = - conn - |> get("/api/v1/search", %{"q" => "2hu"}) + capture_log(fn -> + results = + conn + |> get("/api/v1/search", %{"q" => "2hu"}) + |> json_response(200) - assert results = json_response(conn, 200) - - assert results["accounts"] == [] - assert results["statuses"] == [] + assert results["accounts"] == [] + assert results["statuses"] == [] + end) =~ + "[error] Elixir.Pleroma.Web.MastodonAPI.SearchController search error: %RuntimeError{message: \"Oops\"}" end end diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index 3dd8c6491..bb7648bdd 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -4,7 +4,10 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do use Pleroma.Web.ConnCase + + import ExUnit.CaptureLog import Pleroma.Factory + alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.CommonAPI @@ -27,24 +30,28 @@ test "decodes a salmon", %{conn: conn} do user = insert(:user) salmon = File.read!("test/fixtures/salmon.xml") - conn = - conn - |> put_req_header("content-type", "application/atom+xml") - |> post("/users/#{user.nickname}/salmon", salmon) + assert capture_log(fn -> + conn = + conn + |> put_req_header("content-type", "application/atom+xml") + |> post("/users/#{user.nickname}/salmon", salmon) - assert response(conn, 200) + assert response(conn, 200) + end) =~ "[error]" end test "decodes a salmon with a changed magic key", %{conn: conn} do user = insert(:user) salmon = File.read!("test/fixtures/salmon.xml") - conn = - conn - |> put_req_header("content-type", "application/atom+xml") - |> post("/users/#{user.nickname}/salmon", salmon) + assert capture_log(fn -> + conn = + conn + |> put_req_header("content-type", "application/atom+xml") + |> post("/users/#{user.nickname}/salmon", salmon) - assert response(conn, 200) + assert response(conn, 200) + end) =~ "[error]" # Set a wrong magic-key for a user so it has to refetch salmon_user = User.get_cached_by_ap_id("http://gs.example.org:4040/index.php/user/1") @@ -61,12 +68,14 @@ test "decodes a salmon with a changed magic key", %{conn: conn} do |> Ecto.Changeset.put_embed(:info, info_cng) |> User.update_and_set_cache() - conn = - build_conn() - |> put_req_header("content-type", "application/atom+xml") - |> post("/users/#{user.nickname}/salmon", salmon) + assert capture_log(fn -> + conn = + build_conn() + |> put_req_header("content-type", "application/atom+xml") + |> post("/users/#{user.nickname}/salmon", salmon) - assert response(conn, 200) + assert response(conn, 200) + end) =~ "[error]" end end From bc6c5c513ae69e7a868c63f878a009dce8dd8c63 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sun, 21 Jul 2019 03:52:06 +0000 Subject: [PATCH 21/45] router: ensure the AP sharedinbox path is registered first --- lib/pleroma/web/router.ex | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 518720d38..a33b5ddd7 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -663,6 +663,12 @@ defmodule Pleroma.Web.Router do end end + scope "/", Pleroma.Web.ActivityPub do + pipe_through(:activitypub) + post("/inbox", ActivityPubController, :inbox) + post("/users/:nickname/inbox", ActivityPubController, :inbox) + end + scope "/relay", Pleroma.Web.ActivityPub do pipe_through(:ap_service_actor) @@ -677,12 +683,6 @@ defmodule Pleroma.Web.Router do post("/inbox", ActivityPubController, :inbox) end - scope "/", Pleroma.Web.ActivityPub do - pipe_through(:activitypub) - post("/inbox", ActivityPubController, :inbox) - post("/users/:nickname/inbox", ActivityPubController, :inbox) - end - scope "/.well-known", Pleroma.Web do pipe_through(:well_known) From 33681747857eec90ff56ea0342d2ea179c4f856e Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 21 Jul 2019 18:22:22 +0300 Subject: [PATCH 22/45] Fix rich media parser failing when no TTL can be found by image TTL setters --- CHANGELOG.md | 1 + lib/pleroma/web/rich_media/parser.ex | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f60f3ed97..5c7214f98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Add `account_id`, `type`, `offset`, and `limit` to search API (`/api/v1/search` and `/api/v2/search`) - ActivityPub C2S: follower/following collection pages being inaccessible even when authentifucated if `hide_followers`/ `hide_follows` was set - Existing user id not being preserved on insert conflict +- Rich Media: Parser failing when no TTL can be found by image TTL setters ### Added - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index b69b2be61..185156375 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -55,8 +55,8 @@ def ttl(data, url) do ttl_setters: [MyModule] """ def set_ttl_based_on_image({:ok, data}, url) do - with {:ok, nil} <- Cachex.ttl(:rich_media_cache, url) do - ttl = get_ttl_from_image(data, url) + with {:ok, nil} <- Cachex.ttl(:rich_media_cache, url), + ttl when is_number(ttl) <- get_ttl_from_image(data, url) do Cachex.expire_at(:rich_media_cache, url, ttl * 1000) {:ok, data} else From 56019d53a8fa0a37de4c342c74cc8c70bf1786e9 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Mon, 22 Jul 2019 02:18:45 +0000 Subject: [PATCH 23/45] activitypub: publisher: align sharedinbox usage with AP specification rules While debugging the follow breakage, I observed that our sharedInbox usage did not match the rules in the specification. Accordingly, I have better aligned our usage of sharedInbox with the rules outlined in the ActivityPub specification. --- CHANGELOG.md | 1 + lib/pleroma/web/activity_pub/publisher.ex | 43 ++++++++- test/web/activity_pub/publisher_test.exs | 112 ++++++++++++++++++++++ 3 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 test/web/activity_pub/publisher_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c7214f98..6d1c9ed31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - ActivityPub C2S: follower/following collection pages being inaccessible even when authentifucated if `hide_followers`/ `hide_follows` was set - Existing user id not being preserved on insert conflict - Rich Media: Parser failing when no TTL can be found by image TTL setters +- ActivityPub S2S: sharedInbox usage has been mostly aligned with the rules in the AP specification. ### Added - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index c505223f7..f8a4a4420 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -112,6 +112,45 @@ defp get_cc_ap_ids(ap_id, recipients) do |> Enum.map(& &1.ap_id) end + @as_public "https://www.w3.org/ns/activitystreams#Public" + + defp maybe_use_sharedinbox(%User{info: %{source_data: data}}), + do: (is_map(data["endpoints"]) && Map.get(data["endpoints"], "sharedInbox")) || data["inbox"] + + @doc """ + Determine a user inbox to use based on heuristics. These heuristics + are based on an approximation of the ``sharedInbox`` rules in the + [ActivityPub specification][ap-sharedinbox]. + + Please do not edit this function (or its children) without reading + the spec, as editing the code is likely to introduce some breakage + without some familiarity. + + [ap-sharedinbox]: https://www.w3.org/TR/activitypub/#shared-inbox-delivery + """ + def determine_inbox( + %Activity{data: activity_data}, + %User{info: %{source_data: data}} = user + ) do + to = activity_data["to"] || [] + cc = activity_data["cc"] || [] + type = activity_data["type"] + + cond do + type == "Delete" -> + maybe_use_sharedinbox(user) + + @as_public in to || @as_public in cc -> + maybe_use_sharedinbox(user) + + length(to) + length(cc) > 1 -> + maybe_use_sharedinbox(user) + + true -> + data["inbox"] + end + end + @doc """ Publishes an activity with BCC to all relevant peers. """ @@ -166,8 +205,8 @@ def publish(%User{} = actor, %Activity{} = activity) do recipients(actor, activity) |> Enum.filter(fn user -> User.ap_enabled?(user) end) - |> Enum.map(fn %{info: %{source_data: data}} -> - (is_map(data["endpoints"]) && Map.get(data["endpoints"], "sharedInbox")) || data["inbox"] + |> Enum.map(fn %User{} = user -> + determine_inbox(activity, user) end) |> Enum.uniq() |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) diff --git a/test/web/activity_pub/publisher_test.exs b/test/web/activity_pub/publisher_test.exs new file mode 100644 index 000000000..2b15b3edc --- /dev/null +++ b/test/web/activity_pub/publisher_test.exs @@ -0,0 +1,112 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.PublisherTest do + use Pleroma.DataCase + + import Pleroma.Factory + + alias Pleroma.Activity + alias Pleroma.Web.ActivityPub.Publisher + + @as_public "https://www.w3.org/ns/activitystreams#Public" + + describe "determine_inbox/2" do + test "it returns sharedInbox for messages involving as:Public in to" do + user = + insert(:user, %{ + info: %{source_data: %{"endpoints" => %{"sharedInbox" => "http://example.com/inbox"}}} + }) + + activity = %Activity{ + data: %{"to" => [@as_public], "cc" => [user.follower_address]} + } + + assert Publisher.determine_inbox(activity, user) == "http://example.com/inbox" + end + + test "it returns sharedInbox for messages involving as:Public in cc" do + user = + insert(:user, %{ + info: %{source_data: %{"endpoints" => %{"sharedInbox" => "http://example.com/inbox"}}} + }) + + activity = %Activity{ + data: %{"cc" => [@as_public], "to" => [user.follower_address]} + } + + assert Publisher.determine_inbox(activity, user) == "http://example.com/inbox" + end + + test "it returns sharedInbox for messages involving multiple recipients in to" do + user = + insert(:user, %{ + info: %{source_data: %{"endpoints" => %{"sharedInbox" => "http://example.com/inbox"}}} + }) + + user_two = insert(:user) + user_three = insert(:user) + + activity = %Activity{ + data: %{"cc" => [], "to" => [user.ap_id, user_two.ap_id, user_three.ap_id]} + } + + assert Publisher.determine_inbox(activity, user) == "http://example.com/inbox" + end + + test "it returns sharedInbox for messages involving multiple recipients in cc" do + user = + insert(:user, %{ + info: %{source_data: %{"endpoints" => %{"sharedInbox" => "http://example.com/inbox"}}} + }) + + user_two = insert(:user) + user_three = insert(:user) + + activity = %Activity{ + data: %{"to" => [], "cc" => [user.ap_id, user_two.ap_id, user_three.ap_id]} + } + + assert Publisher.determine_inbox(activity, user) == "http://example.com/inbox" + end + + test "it returns sharedInbox for messages involving multiple recipients in total" do + user = + insert(:user, %{ + info: %{ + source_data: %{ + "inbox" => "http://example.com/personal-inbox", + "endpoints" => %{"sharedInbox" => "http://example.com/inbox"} + } + } + }) + + user_two = insert(:user) + + activity = %Activity{ + data: %{"to" => [user_two.ap_id], "cc" => [user.ap_id]} + } + + assert Publisher.determine_inbox(activity, user) == "http://example.com/inbox" + end + + test "it returns inbox for messages involving single recipients in total" do + user = + insert(:user, %{ + info: %{ + source_data: %{ + "inbox" => "http://example.com/personal-inbox", + "endpoints" => %{"sharedInbox" => "http://example.com/inbox"} + } + } + }) + + activity = %Activity{ + data: %{"to" => [user.ap_id], "cc" => []} + } + + assert Publisher.determine_inbox(activity, user) == "http://example.com/personal-inbox" + end + end +end From a5d6287ba861b9b30edb2ac52584369b9c4665bc Mon Sep 17 00:00:00 2001 From: Sadposter Date: Mon, 22 Jul 2019 02:42:29 +0000 Subject: [PATCH 24/45] Hide blocked users from interactions --- .../mastodon_api/mastodon_api_controller.ex | 10 ++++-- .../mastodon_api_controller_test.exs | 36 +++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index e8b43e475..d660f3f05 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -883,7 +883,10 @@ def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do with %Activity{data: %{"object" => object}} <- Activity.get_by_id(id), %Object{data: %{"likes" => likes}} <- Object.normalize(object) do q = from(u in User, where: u.ap_id in ^likes) - users = Repo.all(q) + + users = + Repo.all(q) + |> Enum.filter(&(not User.blocks?(user, &1))) conn |> put_view(AccountView) @@ -897,7 +900,10 @@ def reblogged_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do with %Activity{data: %{"object" => object}} <- Activity.get_by_id(id), %Object{data: %{"announcements" => announces}} <- Object.normalize(object) do q = from(u in User, where: u.ap_id in ^announces) - users = Repo.all(q) + + users = + Repo.all(q) + |> Enum.filter(&(not User.blocks?(user, &1))) conn |> put_view(AccountView) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index b4b1dd785..a3e4c4136 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -3768,6 +3768,24 @@ test "returns empty array when status has not been favorited yet", %{ assert Enum.empty?(response) end + + test "does not return users who have favorited the status but are blocked", %{ + conn: %{assigns: %{user: user}} = conn, + activity: activity + } do + other_user = insert(:user) + {:ok, user} = User.block(user, other_user) + + {:ok, _, _} = CommonAPI.favorite(activity.id, other_user) + + response = + conn + |> assign(:user, user) + |> get("/api/v1/statuses/#{activity.id}/favourited_by") + |> json_response(:ok) + + assert Enum.empty?(response) + end end describe "GET /api/v1/statuses/:id/reblogged_by" do @@ -3807,6 +3825,24 @@ test "returns empty array when status has not been reblogged yet", %{ assert Enum.empty?(response) end + + test "does not return users who have reblogged the status but are blocked", %{ + conn: %{assigns: %{user: user}} = conn, + activity: activity + } do + other_user = insert(:user) + {:ok, user} = User.block(user, other_user) + + {:ok, _, _} = CommonAPI.repeat(activity.id, other_user) + + response = + conn + |> assign(:user, user) + |> get("/api/v1/statuses/#{activity.id}/reblogged_by") + |> json_response(:ok) + + assert Enum.empty?(response) + end end describe "POST /auth/password, with valid parameters" do From 05b5af8075621bfefb207ee84b54608f652fe757 Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Mon, 22 Jul 2019 02:43:15 +0000 Subject: [PATCH 25/45] Add tests for users tasks and PleromaAuthenticator --- lib/pleroma/user_invite_token.ex | 2 +- .../web/admin_api/admin_api_controller.ex | 12 +-- test/tasks/user_test.exs | 84 +++++++++++++++++++ .../admin_api/admin_api_controller_test.exs | 11 +++ test/web/oauth/oauth_controller_test.exs | 74 ++++++++-------- 5 files changed, 139 insertions(+), 44 deletions(-) diff --git a/lib/pleroma/user_invite_token.ex b/lib/pleroma/user_invite_token.ex index fadc89891..b9e80acdd 100644 --- a/lib/pleroma/user_invite_token.ex +++ b/lib/pleroma/user_invite_token.ex @@ -74,7 +74,7 @@ def find_by_token!(token), do: Repo.get_by!(UserInviteToken, token: token) @spec find_by_token(token()) :: {:ok, UserInviteToken.t()} | nil def find_by_token(token) do - with invite <- Repo.get_by(UserInviteToken, token: token) do + with %UserInviteToken{} = invite <- Repo.get_by(UserInviteToken, token: token) do {:ok, invite} end end diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 4a0bf4823..811be1eff 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -272,11 +272,13 @@ def invites(conn, _params) do @doc "Revokes invite by token" def revoke_invite(conn, %{"token" => token}) do - invite = UserInviteToken.find_by_token!(token) - {:ok, updated_invite} = UserInviteToken.update_invite(invite, %{used: true}) - - conn - |> json(AccountView.render("invite.json", %{invite: updated_invite})) + with {:ok, invite} <- UserInviteToken.find_by_token(token), + {:ok, updated_invite} = UserInviteToken.update_invite(invite, %{used: true}) do + conn + |> json(AccountView.render("invite.json", %{invite: updated_invite})) + else + nil -> {:error, :not_found} + end end @doc "Get a password reset token (base64 string) for given nickname" diff --git a/test/tasks/user_test.exs b/test/tasks/user_test.exs index 3d4b08fba..2b9453042 100644 --- a/test/tasks/user_test.exs +++ b/test/tasks/user_test.exs @@ -5,6 +5,9 @@ defmodule Mix.Tasks.Pleroma.UserTest do alias Pleroma.Repo alias Pleroma.User + alias Pleroma.Web.OAuth.Authorization + alias Pleroma.Web.OAuth.Token + use Pleroma.DataCase import Pleroma.Factory @@ -327,6 +330,13 @@ test "invite is revoked" do assert_received {:mix_shell, :info, [message]} assert message =~ "Invite for token #{invite.token} was revoked." end + + test "it prints an error message when invite is not exist" do + Mix.Tasks.Pleroma.User.run(["revoke_invite", "foo"]) + + assert_received {:mix_shell, :error, [message]} + assert message =~ "No invite found" + end end describe "running delete_activities" do @@ -337,6 +347,13 @@ test "activities are deleted" do assert_received {:mix_shell, :info, [message]} assert message == "User #{nickname} statuses deleted." end + + test "it prints an error message when user is not exist" do + Mix.Tasks.Pleroma.User.run(["delete_activities", "foo"]) + + assert_received {:mix_shell, :error, [message]} + assert message =~ "No local user" + end end describe "running toggle_confirmed" do @@ -364,6 +381,13 @@ test "user is not confirmed" do refute user.info.confirmation_pending refute user.info.confirmation_token end + + test "it prints an error message when user is not exist" do + Mix.Tasks.Pleroma.User.run(["toggle_confirmed", "foo"]) + + assert_received {:mix_shell, :error, [message]} + assert message =~ "No local user" + end end describe "search" do @@ -386,4 +410,64 @@ test "it returns users matching" do User.Search.search("moon fediverse", for_user: user) |> Enum.map(& &1.id) end end + + describe "signing out" do + test "it deletes all user's tokens and authorizations" do + user = insert(:user) + insert(:oauth_token, user: user) + insert(:oauth_authorization, user: user) + + assert Repo.get_by(Token, user_id: user.id) + assert Repo.get_by(Authorization, user_id: user.id) + + :ok = Mix.Tasks.Pleroma.User.run(["sign_out", user.nickname]) + + refute Repo.get_by(Token, user_id: user.id) + refute Repo.get_by(Authorization, user_id: user.id) + end + + test "it prints an error message when user is not exist" do + Mix.Tasks.Pleroma.User.run(["sign_out", "foo"]) + + assert_received {:mix_shell, :error, [message]} + assert message =~ "No local user" + end + end + + describe "tagging" do + test "it add tags to a user" do + user = insert(:user) + + :ok = Mix.Tasks.Pleroma.User.run(["tag", user.nickname, "pleroma"]) + + user = User.get_cached_by_nickname(user.nickname) + assert "pleroma" in user.tags + end + + test "it prints an error message when user is not exist" do + Mix.Tasks.Pleroma.User.run(["tag", "foo"]) + + assert_received {:mix_shell, :error, [message]} + assert message =~ "Could not change user tags" + end + end + + describe "untagging" do + test "it deletes tags from a user" do + user = insert(:user, tags: ["pleroma"]) + assert "pleroma" in user.tags + + :ok = Mix.Tasks.Pleroma.User.run(["untag", user.nickname, "pleroma"]) + + user = User.get_cached_by_nickname(user.nickname) + assert Enum.empty?(user.tags) + end + + test "it prints an error message when user is not exist" do + Mix.Tasks.Pleroma.User.run(["untag", "foo"]) + + assert_received {:mix_shell, :error, [message]} + assert message =~ "Could not change user tags" + end + end end diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index ee48b752c..03aa46cae 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -1010,6 +1010,17 @@ test "with token" do "uses" => 0 } end + + test "with invalid token" do + admin = insert(:user, info: %{is_admin: true}) + + conn = + build_conn() + |> assign(:user, admin) + |> post("/api/pleroma/admin/users/revoke_invite", %{"token" => "foo"}) + + assert json_response(conn, :not_found) == "Not found" + end end describe "GET /api/pleroma/admin/reports/:id" do diff --git a/test/web/oauth/oauth_controller_test.exs b/test/web/oauth/oauth_controller_test.exs index aae34804d..92e156347 100644 --- a/test/web/oauth/oauth_controller_test.exs +++ b/test/web/oauth/oauth_controller_test.exs @@ -5,9 +5,7 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do use Pleroma.Web.ConnCase import Pleroma.Factory - import Mock - alias Pleroma.Registration alias Pleroma.Repo alias Pleroma.Web.OAuth.Authorization alias Pleroma.Web.OAuth.OAuthController @@ -108,28 +106,26 @@ test "with user-bound registration, GET /oauth//callback redirects to "state" => "" } - with_mock Pleroma.Web.Auth.Authenticator, - get_registration: fn _ -> {:ok, registration} end do - conn = - get( - conn, - "/oauth/twitter/callback", - %{ - "oauth_token" => "G-5a3AAAAAAAwMH9AAABaektfSM", - "oauth_verifier" => "QZl8vUqNvXMTKpdmUnGejJxuHG75WWWs", - "provider" => "twitter", - "state" => Poison.encode!(state_params) - } - ) + conn = + conn + |> assign(:ueberauth_auth, %{provider: registration.provider, uid: registration.uid}) + |> get( + "/oauth/twitter/callback", + %{ + "oauth_token" => "G-5a3AAAAAAAwMH9AAABaektfSM", + "oauth_verifier" => "QZl8vUqNvXMTKpdmUnGejJxuHG75WWWs", + "provider" => "twitter", + "state" => Poison.encode!(state_params) + } + ) - assert response = html_response(conn, 302) - assert redirected_to(conn) =~ ~r/#{redirect_uri}\?code=.+/ - end + assert response = html_response(conn, 302) + assert redirected_to(conn) =~ ~r/#{redirect_uri}\?code=.+/ end test "with user-unbound registration, GET /oauth//callback renders registration_details page", %{app: app, conn: conn} do - registration = insert(:registration, user: nil) + user = insert(:user) state_params = %{ "scope" => "read write", @@ -138,26 +134,28 @@ test "with user-unbound registration, GET /oauth//callback renders reg "state" => "a_state" } - with_mock Pleroma.Web.Auth.Authenticator, - get_registration: fn _ -> {:ok, registration} end do - conn = - get( - conn, - "/oauth/twitter/callback", - %{ - "oauth_token" => "G-5a3AAAAAAAwMH9AAABaektfSM", - "oauth_verifier" => "QZl8vUqNvXMTKpdmUnGejJxuHG75WWWs", - "provider" => "twitter", - "state" => Poison.encode!(state_params) - } - ) + conn = + conn + |> assign(:ueberauth_auth, %{ + provider: "twitter", + uid: "171799000", + info: %{nickname: user.nickname, email: user.email, name: user.name, description: nil} + }) + |> get( + "/oauth/twitter/callback", + %{ + "oauth_token" => "G-5a3AAAAAAAwMH9AAABaektfSM", + "oauth_verifier" => "QZl8vUqNvXMTKpdmUnGejJxuHG75WWWs", + "provider" => "twitter", + "state" => Poison.encode!(state_params) + } + ) - assert response = html_response(conn, 200) - assert response =~ ~r/name="op" type="submit" value="register"/ - assert response =~ ~r/name="op" type="submit" value="connect"/ - assert response =~ Registration.email(registration) - assert response =~ Registration.nickname(registration) - end + assert response = html_response(conn, 200) + assert response =~ ~r/name="op" type="submit" value="register"/ + assert response =~ ~r/name="op" type="submit" value="connect"/ + assert response =~ user.email + assert response =~ user.nickname end test "on authentication error, GET /oauth//callback redirects to `redirect_uri`", %{ From f712ee879ab771b5cb9591ae402f52e26a8bebf3 Mon Sep 17 00:00:00 2001 From: Sadposter Date: Mon, 22 Jul 2019 02:43:55 +0000 Subject: [PATCH 26/45] Bugfix: muted/blocked user notification streaming --- CHANGELOG.md | 1 + lib/pleroma/web/streamer.ex | 40 +++++++++++++++++++++++++++---------- test/web/streamer_test.exs | 38 +++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c7214f98..bcdd0615f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Handling of search timeouts (`/api/v1/search` and `/api/v2/search`) - Mastodon API: Embedded relationships not being properly rendered in the Account entity of Status entity - Mastodon API: Add `account_id`, `type`, `offset`, and `limit` to search API (`/api/v1/search` and `/api/v2/search`) +- Mastodon API, streaming: Fix filtering of notifications based on blocks/mutes/thread mutes - ActivityPub C2S: follower/following collection pages being inaccessible even when authentifucated if `hide_followers`/ `hide_follows` was set - Existing user id not being preserved on insert conflict - Rich Media: Parser failing when no TTL can be found by image TTL setters diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 4f325113a..86e2dc4dd 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -13,6 +13,7 @@ defmodule Pleroma.Web.Streamer do alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.NotificationView @keepalive_interval :timer.seconds(30) @@ -118,10 +119,14 @@ def handle_cast( topics |> Map.get("#{topic}:#{item.user_id}", []) |> Enum.each(fn socket -> - send( - socket.transport_pid, - {:text, represent_notification(socket.assigns[:user], item)} - ) + with %User{} = user <- User.get_cached_by_ap_id(socket.assigns[:user].ap_id), + true <- should_send?(user, item), + false <- CommonAPI.thread_muted?(user, item.activity) do + send( + socket.transport_pid, + {:text, represent_notification(socket.assigns[:user], item)} + ) + end end) {:noreply, topics} @@ -225,19 +230,32 @@ defp represent_notification(%User{} = user, %Notification{} = notify) do |> Jason.encode!() end + defp should_send?(%User{} = user, %Activity{} = item) do + blocks = user.info.blocks || [] + mutes = user.info.mutes || [] + reblog_mutes = user.info.muted_reblogs || [] + + with parent when not is_nil(parent) <- Object.normalize(item), + true <- Enum.all?([blocks, mutes, reblog_mutes], &(item.actor not in &1)), + true <- Enum.all?([blocks, mutes], &(parent.data["actor"] not in &1)), + true <- thread_containment(item, user) do + true + else + _ -> false + end + end + + defp should_send?(%User{} = user, %Notification{activity: activity}) do + should_send?(user, activity) + end + def push_to_socket(topics, topic, %Activity{data: %{"type" => "Announce"}} = item) do Enum.each(topics[topic] || [], fn socket -> # Get the current user so we have up-to-date blocks etc. if socket.assigns[:user] do user = User.get_cached_by_ap_id(socket.assigns[:user].ap_id) - blocks = user.info.blocks || [] - mutes = user.info.mutes || [] - reblog_mutes = user.info.muted_reblogs || [] - with parent when not is_nil(parent) <- Object.normalize(item), - true <- Enum.all?([blocks, mutes, reblog_mutes], &(item.actor not in &1)), - true <- Enum.all?([blocks, mutes], &(parent.data["actor"] not in &1)), - true <- thread_containment(item, user) do + if should_send?(user, item) do send(socket.transport_pid, {:text, represent_update(item, user)}) end else diff --git a/test/web/streamer_test.exs b/test/web/streamer_test.exs index 4633d7765..8f56e7486 100644 --- a/test/web/streamer_test.exs +++ b/test/web/streamer_test.exs @@ -65,6 +65,44 @@ test "it sends notify to in the 'user:notification' stream", %{user: user, notif Streamer.stream("user:notification", notify) Task.await(task) end + + test "it doesn't send notify to the 'user:notification' stream when a user is blocked", %{ + user: user + } do + blocked = insert(:user) + {:ok, user} = User.block(user, blocked) + + task = Task.async(fn -> refute_receive {:text, _}, 4_000 end) + + Streamer.add_socket( + "user:notification", + %{transport_pid: task.pid, assigns: %{user: user}} + ) + + {:ok, activity} = CommonAPI.post(user, %{"status" => ":("}) + {:ok, notif, _} = CommonAPI.favorite(activity.id, blocked) + + Streamer.stream("user:notification", notif) + Task.await(task) + end + + test "it doesn't send notify to the 'user:notification' stream when a thread is muted", %{ + user: user + } do + user2 = insert(:user) + task = Task.async(fn -> refute_receive {:text, _}, 4_000 end) + + Streamer.add_socket( + "user:notification", + %{transport_pid: task.pid, assigns: %{user: user}} + ) + + {:ok, activity} = CommonAPI.post(user, %{"status" => "super hot take"}) + {:ok, activity} = CommonAPI.add_mute(user, activity) + {:ok, notif, _} = CommonAPI.favorite(activity.id, user2) + Streamer.stream("user:notification", notif) + Task.await(task) + end end test "it sends to public" do From b70e659304ba35f7afc598c3d3d1b96fa16f6cdf Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 22 Jul 2019 14:33:58 +0000 Subject: [PATCH 27/45] Feature/1087 wildcard option for blocks --- lib/pleroma/user.ex | 7 +- lib/pleroma/web/activity_pub/mrf.ex | 10 ++ .../web/activity_pub/mrf/simple_policy.ex | 55 ++++++++--- lib/pleroma/web/activity_pub/publisher.ex | 9 +- test/user_test.exs | 42 +++++++++ test/web/activity_pub/mrf/mrf_test.exs | 46 +++++++++ .../activity_pub/mrf/simple_policy_test.exs | 93 +++++++++++++++++++ 7 files changed, 244 insertions(+), 18 deletions(-) create mode 100644 test/web/activity_pub/mrf/mrf_test.exs diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 5ea2b518b..a3f6add28 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -873,10 +873,13 @@ def muted_notifications?(user, %{ap_id: ap_id}), def blocks?(%User{info: info} = _user, %{ap_id: ap_id}) do blocks = info.blocks - domain_blocks = info.domain_blocks + + domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(info.domain_blocks) + %{host: host} = URI.parse(ap_id) - Enum.member?(blocks, ap_id) || Enum.any?(domain_blocks, &(&1 == host)) + Enum.member?(blocks, ap_id) || + Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, host) end def subscribed_to?(user, %{ap_id: ap_id}) do diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 10ceef715..dd204b21c 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -25,4 +25,14 @@ def get_policies do defp get_policies(policy) when is_atom(policy), do: [policy] defp get_policies(policies) when is_list(policies), do: policies defp get_policies(_), do: [] + + @spec subdomains_regex([String.t()]) :: [Regex.t()] + def subdomains_regex(domains) when is_list(domains) do + for domain <- domains, do: ~r(^#{String.replace(domain, "*.", "(.*\\.)*")}$) + end + + @spec subdomain_match?([Regex.t()], String.t()) :: boolean() + def subdomain_match?(domains, host) do + Enum.any?(domains, fn domain -> Regex.match?(domain, host) end) + end end diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 433d23c5f..2cf63d3db 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -4,22 +4,29 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do alias Pleroma.User + alias Pleroma.Web.ActivityPub.MRF @moduledoc "Filter activities depending on their origin instance" - @behaviour Pleroma.Web.ActivityPub.MRF + @behaviour MRF defp check_accept(%{host: actor_host} = _actor_info, object) do - accepts = Pleroma.Config.get([:mrf_simple, :accept]) + accepts = + Pleroma.Config.get([:mrf_simple, :accept]) + |> MRF.subdomains_regex() cond do accepts == [] -> {:ok, object} actor_host == Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host]) -> {:ok, object} - Enum.member?(accepts, actor_host) -> {:ok, object} + MRF.subdomain_match?(accepts, actor_host) -> {:ok, object} true -> {:reject, nil} end end defp check_reject(%{host: actor_host} = _actor_info, object) do - if Enum.member?(Pleroma.Config.get([:mrf_simple, :reject]), actor_host) do + rejects = + Pleroma.Config.get([:mrf_simple, :reject]) + |> MRF.subdomains_regex() + + if MRF.subdomain_match?(rejects, actor_host) do {:reject, nil} else {:ok, object} @@ -31,8 +38,12 @@ defp check_media_removal( %{"type" => "Create", "object" => %{"attachment" => child_attachment}} = object ) when length(child_attachment) > 0 do + media_removal = + Pleroma.Config.get([:mrf_simple, :media_removal]) + |> MRF.subdomains_regex() + object = - if Enum.member?(Pleroma.Config.get([:mrf_simple, :media_removal]), actor_host) do + if MRF.subdomain_match?(media_removal, actor_host) do child_object = Map.delete(object["object"], "attachment") Map.put(object, "object", child_object) else @@ -51,8 +62,12 @@ defp check_media_nsfw( "object" => child_object } = object ) do + media_nsfw = + Pleroma.Config.get([:mrf_simple, :media_nsfw]) + |> MRF.subdomains_regex() + object = - if Enum.member?(Pleroma.Config.get([:mrf_simple, :media_nsfw]), actor_host) do + if MRF.subdomain_match?(media_nsfw, actor_host) do tags = (child_object["tag"] || []) ++ ["nsfw"] child_object = Map.put(child_object, "tag", tags) child_object = Map.put(child_object, "sensitive", true) @@ -67,12 +82,12 @@ defp check_media_nsfw( defp check_media_nsfw(_actor_info, object), do: {:ok, object} defp check_ftl_removal(%{host: actor_host} = _actor_info, object) do + timeline_removal = + Pleroma.Config.get([:mrf_simple, :federated_timeline_removal]) + |> MRF.subdomains_regex() + object = - with true <- - Enum.member?( - Pleroma.Config.get([:mrf_simple, :federated_timeline_removal]), - actor_host - ), + with true <- MRF.subdomain_match?(timeline_removal, actor_host), user <- User.get_cached_by_ap_id(object["actor"]), true <- "https://www.w3.org/ns/activitystreams#Public" in object["to"] do to = @@ -94,7 +109,11 @@ defp check_ftl_removal(%{host: actor_host} = _actor_info, object) do end defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = object) do - if actor_host in Pleroma.Config.get([:mrf_simple, :report_removal]) do + report_removal = + Pleroma.Config.get([:mrf_simple, :report_removal]) + |> MRF.subdomains_regex() + + if MRF.subdomain_match?(report_removal, actor_host) do {:reject, nil} else {:ok, object} @@ -104,7 +123,11 @@ defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} defp check_report_removal(_actor_info, object), do: {:ok, object} defp check_avatar_removal(%{host: actor_host} = _actor_info, %{"icon" => _icon} = object) do - if actor_host in Pleroma.Config.get([:mrf_simple, :avatar_removal]) do + avatar_removal = + Pleroma.Config.get([:mrf_simple, :avatar_removal]) + |> MRF.subdomains_regex() + + if MRF.subdomain_match?(avatar_removal, actor_host) do {:ok, Map.delete(object, "icon")} else {:ok, object} @@ -114,7 +137,11 @@ defp check_avatar_removal(%{host: actor_host} = _actor_info, %{"icon" => _icon} defp check_avatar_removal(_actor_info, object), do: {:ok, object} defp check_banner_removal(%{host: actor_host} = _actor_info, %{"image" => _image} = object) do - if actor_host in Pleroma.Config.get([:mrf_simple, :banner_removal]) do + banner_removal = + Pleroma.Config.get([:mrf_simple, :banner_removal]) + |> MRF.subdomains_regex() + + if MRF.subdomain_match?(banner_removal, actor_host) do {:ok, Map.delete(object, "image")} else {:ok, object} diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index f8a4a4420..0bbe6ee80 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -87,8 +87,13 @@ defp should_federate?(inbox, public) do if public do true else - inbox_info = URI.parse(inbox) - !Enum.member?(Config.get([:instance, :quarantined_instances], []), inbox_info.host) + %{host: host} = URI.parse(inbox) + + quarantined_instances = + Config.get([:instance, :quarantined_instances], []) + |> Pleroma.Web.ActivityPub.MRF.subdomains_regex() + + !Pleroma.Web.ActivityPub.MRF.subdomain_match?(quarantined_instances, host) end end diff --git a/test/user_test.exs b/test/user_test.exs index 908f72a0e..8a7b7537f 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -824,6 +824,48 @@ test "blocks domains" do assert User.blocks?(user, collateral_user) end + test "does not block domain with same end" do + user = insert(:user) + + collateral_user = + insert(:user, %{ap_id: "https://another-awful-and-rude-instance.com/user/bully"}) + + {:ok, user} = User.block_domain(user, "awful-and-rude-instance.com") + + refute User.blocks?(user, collateral_user) + end + + test "does not block domain with same end if wildcard added" do + user = insert(:user) + + collateral_user = + insert(:user, %{ap_id: "https://another-awful-and-rude-instance.com/user/bully"}) + + {:ok, user} = User.block_domain(user, "*.awful-and-rude-instance.com") + + refute User.blocks?(user, collateral_user) + end + + test "blocks domain with wildcard for subdomain" do + user = insert(:user) + + user_from_subdomain = + insert(:user, %{ap_id: "https://subdomain.awful-and-rude-instance.com/user/bully"}) + + user_with_two_subdomains = + insert(:user, %{ + ap_id: "https://subdomain.second_subdomain.awful-and-rude-instance.com/user/bully" + }) + + user_domain = insert(:user, %{ap_id: "https://awful-and-rude-instance.com/user/bully"}) + + {:ok, user} = User.block_domain(user, "*.awful-and-rude-instance.com") + + assert User.blocks?(user, user_from_subdomain) + assert User.blocks?(user, user_with_two_subdomains) + assert User.blocks?(user, user_domain) + end + test "unblocks domains" do user = insert(:user) collateral_user = insert(:user, %{ap_id: "https://awful-and-rude-instance.com/user/bully"}) diff --git a/test/web/activity_pub/mrf/mrf_test.exs b/test/web/activity_pub/mrf/mrf_test.exs new file mode 100644 index 000000000..a9cdf5317 --- /dev/null +++ b/test/web/activity_pub/mrf/mrf_test.exs @@ -0,0 +1,46 @@ +defmodule Pleroma.Web.ActivityPub.MRFTest do + use ExUnit.Case, async: true + alias Pleroma.Web.ActivityPub.MRF + + test "subdomains_regex/1" do + assert MRF.subdomains_regex(["unsafe.tld", "*.unsafe.tld"]) == [ + ~r/^unsafe.tld$/, + ~r/^(.*\.)*unsafe.tld$/ + ] + end + + describe "subdomain_match/2" do + test "common domains" do + regexes = MRF.subdomains_regex(["unsafe.tld", "unsafe2.tld"]) + + assert regexes == [~r/^unsafe.tld$/, ~r/^unsafe2.tld$/] + + assert MRF.subdomain_match?(regexes, "unsafe.tld") + assert MRF.subdomain_match?(regexes, "unsafe2.tld") + + refute MRF.subdomain_match?(regexes, "example.com") + end + + test "wildcard domains with one subdomain" do + regexes = MRF.subdomains_regex(["*.unsafe.tld"]) + + assert regexes == [~r/^(.*\.)*unsafe.tld$/] + + assert MRF.subdomain_match?(regexes, "unsafe.tld") + assert MRF.subdomain_match?(regexes, "sub.unsafe.tld") + refute MRF.subdomain_match?(regexes, "anotherunsafe.tld") + refute MRF.subdomain_match?(regexes, "unsafe.tldanother") + end + + test "wildcard domains with two subdomains" do + regexes = MRF.subdomains_regex(["*.unsafe.tld"]) + + assert regexes == [~r/^(.*\.)*unsafe.tld$/] + + assert MRF.subdomain_match?(regexes, "unsafe.tld") + assert MRF.subdomain_match?(regexes, "sub.sub.unsafe.tld") + refute MRF.subdomain_match?(regexes, "sub.anotherunsafe.tld") + refute MRF.subdomain_match?(regexes, "sub.unsafe.tldanother") + end + end +end diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs index 0fd68e103..8e86d2219 100644 --- a/test/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/web/activity_pub/mrf/simple_policy_test.exs @@ -49,6 +49,19 @@ test "has a matching host" do assert SimplePolicy.filter(local_message) == {:ok, local_message} end + + test "match with wildcard domain" do + Config.put([:mrf_simple, :media_removal], ["*.remote.instance"]) + media_message = build_media_message() + local_message = build_local_message() + + assert SimplePolicy.filter(media_message) == + {:ok, + media_message + |> Map.put("object", Map.delete(media_message["object"], "attachment"))} + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end end describe "when :media_nsfw" do @@ -74,6 +87,20 @@ test "has a matching host" do assert SimplePolicy.filter(local_message) == {:ok, local_message} end + + test "match with wildcard domain" do + Config.put([:mrf_simple, :media_nsfw], ["*.remote.instance"]) + media_message = build_media_message() + local_message = build_local_message() + + assert SimplePolicy.filter(media_message) == + {:ok, + media_message + |> put_in(["object", "tag"], ["foo", "nsfw"]) + |> put_in(["object", "sensitive"], true)} + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end end defp build_media_message do @@ -106,6 +133,15 @@ test "has a matching host" do assert SimplePolicy.filter(report_message) == {:reject, nil} assert SimplePolicy.filter(local_message) == {:ok, local_message} end + + test "match with wildcard domain" do + Config.put([:mrf_simple, :report_removal], ["*.remote.instance"]) + report_message = build_report_message() + local_message = build_local_message() + + assert SimplePolicy.filter(report_message) == {:reject, nil} + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end end defp build_report_message do @@ -146,6 +182,27 @@ test "has a matching host" do assert SimplePolicy.filter(local_message) == {:ok, local_message} end + test "match with wildcard domain" do + {actor, ftl_message} = build_ftl_actor_and_message() + + ftl_message_actor_host = + ftl_message + |> Map.fetch!("actor") + |> URI.parse() + |> Map.fetch!(:host) + + Config.put([:mrf_simple, :federated_timeline_removal], ["*." <> ftl_message_actor_host]) + local_message = build_local_message() + + assert {:ok, ftl_message} = SimplePolicy.filter(ftl_message) + assert actor.follower_address in ftl_message["to"] + refute actor.follower_address in ftl_message["cc"] + refute "https://www.w3.org/ns/activitystreams#Public" in ftl_message["to"] + assert "https://www.w3.org/ns/activitystreams#Public" in ftl_message["cc"] + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end + test "has a matching host but only as:Public in to" do {_actor, ftl_message} = build_ftl_actor_and_message() @@ -192,6 +249,14 @@ test "has a matching host" do assert SimplePolicy.filter(remote_message) == {:reject, nil} end + + test "match with wildcard domain" do + Config.put([:mrf_simple, :reject], ["*.remote.instance"]) + + remote_message = build_remote_message() + + assert SimplePolicy.filter(remote_message) == {:reject, nil} + end end describe "when :accept" do @@ -224,6 +289,16 @@ test "has a matching host" do assert SimplePolicy.filter(local_message) == {:ok, local_message} assert SimplePolicy.filter(remote_message) == {:ok, remote_message} end + + test "match with wildcard domain" do + Config.put([:mrf_simple, :accept], ["*.remote.instance"]) + + local_message = build_local_message() + remote_message = build_remote_message() + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + assert SimplePolicy.filter(remote_message) == {:ok, remote_message} + end end describe "when :avatar_removal" do @@ -251,6 +326,15 @@ test "has a matching host" do refute filtered["icon"] end + + test "match with wildcard domain" do + Config.put([:mrf_simple, :avatar_removal], ["*.remote.instance"]) + + remote_user = build_remote_user() + {:ok, filtered} = SimplePolicy.filter(remote_user) + + refute filtered["icon"] + end end describe "when :banner_removal" do @@ -278,6 +362,15 @@ test "has a matching host" do refute filtered["image"] end + + test "match with wildcard domain" do + Config.put([:mrf_simple, :banner_removal], ["*.remote.instance"]) + + remote_user = build_remote_user() + {:ok, filtered} = SimplePolicy.filter(remote_user) + + refute filtered["image"] + end end defp build_local_message do From 2442b5c18e49f622a9cd5cad371fdb9c2844ad7a Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 22 Jul 2019 19:53:53 +0000 Subject: [PATCH 28/45] changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fcc8e390..4043b1edf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) - MRF: Support for excluding specific domains from Transparency. - MRF: Support for filtering posts based on who they mention (`Pleroma.Web.ActivityPub.MRF.MentionPolicy`) +- MRF (Simple Policy): Support for wildcard domains. +- Support for wildcard domains in user domain blocks setting. +- Configuration: `quarantined_instances` support wildcard domains. - Configuration: `federation_incoming_replies_max_depth` option - Mastodon API: Support for the [`tagged` filter](https://github.com/tootsuite/mastodon/pull/9755) in [`GET /api/v1/accounts/:id/statuses`](https://docs.joinmastodon.org/api/rest/accounts/#get-api-v1-accounts-id-statuses) - Mastodon API, streaming: Add support for passing the token in the `Sec-WebSocket-Protocol` header From 9340896c9ee43dff26e6f11c417e4f6dccdd13de Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Mon, 22 Jul 2019 19:54:22 +0000 Subject: [PATCH 29/45] Exclude tests that use :crypt.crypt/2 on macOS --- test/plugs/authentication_plug_test.exs | 6 ++-- .../plugs/legacy_authentication_plug_test.exs | 34 +++++++------------ test/test_helper.exs | 3 +- 3 files changed, 16 insertions(+), 27 deletions(-) diff --git a/test/plugs/authentication_plug_test.exs b/test/plugs/authentication_plug_test.exs index 7ca045616..f7f8fd9f3 100644 --- a/test/plugs/authentication_plug_test.exs +++ b/test/plugs/authentication_plug_test.exs @@ -9,7 +9,6 @@ defmodule Pleroma.Plugs.AuthenticationPlugTest do alias Pleroma.User import ExUnit.CaptureLog - import Mock setup %{conn: conn} do user = %User{ @@ -67,13 +66,12 @@ test "check pbkdf2 hash" do refute AuthenticationPlug.checkpw("test-password1", hash) end + @tag :skip_on_mac test "check sha512-crypt hash" do hash = "$6$9psBWV8gxkGOZWBz$PmfCycChoxeJ3GgGzwvhlgacb9mUoZ.KUXNCssekER4SJ7bOK53uXrHNb2e4i8yPFgSKyzaW9CcmrDXWIEMtD1" - with_mock :crypt, crypt: fn _password, password_hash -> password_hash end do - assert AuthenticationPlug.checkpw("password", hash) - end + assert AuthenticationPlug.checkpw("password", hash) end test "it returns false when hash invalid" do diff --git a/test/plugs/legacy_authentication_plug_test.exs b/test/plugs/legacy_authentication_plug_test.exs index 02f530058..9804e073b 100644 --- a/test/plugs/legacy_authentication_plug_test.exs +++ b/test/plugs/legacy_authentication_plug_test.exs @@ -5,19 +5,18 @@ defmodule Pleroma.Plugs.LegacyAuthenticationPlugTest do use Pleroma.Web.ConnCase + import Pleroma.Factory + alias Pleroma.Plugs.LegacyAuthenticationPlug alias Pleroma.User - import Mock - setup do - # password is "password" - user = %User{ - id: 1, - name: "dude", - password_hash: - "$6$9psBWV8gxkGOZWBz$PmfCycChoxeJ3GgGzwvhlgacb9mUoZ.KUXNCssekER4SJ7bOK53uXrHNb2e4i8yPFgSKyzaW9CcmrDXWIEMtD1" - } + user = + insert(:user, + password: "password", + password_hash: + "$6$9psBWV8gxkGOZWBz$PmfCycChoxeJ3GgGzwvhlgacb9mUoZ.KUXNCssekER4SJ7bOK53uXrHNb2e4i8yPFgSKyzaW9CcmrDXWIEMtD1" + ) %{user: user} end @@ -36,6 +35,7 @@ test "it does nothing if a user is assigned", %{conn: conn, user: user} do assert ret_conn == conn end + @tag :skip_on_mac test "it authenticates the auth_user if present and password is correct and resets the password", %{ conn: conn, @@ -46,22 +46,12 @@ test "it authenticates the auth_user if present and password is correct and rese |> assign(:auth_credentials, %{username: "dude", password: "password"}) |> assign(:auth_user, user) - conn = - with_mocks([ - {:crypt, [], [crypt: fn _password, password_hash -> password_hash end]}, - {User, [], - [ - reset_password: fn user, %{password: password, password_confirmation: password} -> - {:ok, user} - end - ]} - ]) do - LegacyAuthenticationPlug.call(conn, %{}) - end + conn = LegacyAuthenticationPlug.call(conn, %{}) - assert conn.assigns.user == user + assert conn.assigns.user.id == user.id end + @tag :skip_on_mac test "it does nothing if the password is wrong", %{ conn: conn, user: user diff --git a/test/test_helper.exs b/test/test_helper.exs index 3e33f0335..a927b2c3d 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -2,7 +2,8 @@ # Copyright © 2017-2018 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -ExUnit.start() +os_exclude = if :os.type() == {:unix, :darwin}, do: [skip_on_mac: true], else: [] +ExUnit.start(exclude: os_exclude) Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, :manual) Mox.defmock(Pleroma.ReverseProxy.ClientMock, for: Pleroma.ReverseProxy.Client) {:ok, _} = Application.ensure_all_started(:ex_machina) From eacf61d823f8bc4398dee883aa86171ec4757fe9 Mon Sep 17 00:00:00 2001 From: Sadposter Date: Tue, 23 Jul 2019 15:02:18 +0100 Subject: [PATCH 30/45] fix unauthenticated req to favourited/rebloggd_by --- .../mastodon_api/mastodon_api_controller.ex | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index d660f3f05..ccebcd415 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -884,9 +884,12 @@ def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do %Object{data: %{"likes" => likes}} <- Object.normalize(object) do q = from(u in User, where: u.ap_id in ^likes) - users = - Repo.all(q) - |> Enum.filter(&(not User.blocks?(user, &1))) + users = Repo.all(q) + users = if is_nil(user) do + users + else + Enum.filter(users, &(not User.blocks?(user, &1))) + end conn |> put_view(AccountView) @@ -901,9 +904,12 @@ def reblogged_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do %Object{data: %{"announcements" => announces}} <- Object.normalize(object) do q = from(u in User, where: u.ap_id in ^announces) - users = - Repo.all(q) - |> Enum.filter(&(not User.blocks?(user, &1))) + users = Repo.all(q) + users = if is_nil(user) do + users + else + Enum.filter(users, &(not User.blocks?(user, &1))) + end conn |> put_view(AccountView) From fd1fa5a2ec922575bc8b75dabe224337977c8e3e Mon Sep 17 00:00:00 2001 From: Sadposter Date: Tue, 23 Jul 2019 15:05:19 +0100 Subject: [PATCH 31/45] add tests for unauthed reqs to liked/reblogged_by --- .../mastodon_api_controller_test.exs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index a3e4c4136..00ca320d3 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -3786,6 +3786,20 @@ test "does not return users who have favorited the status but are blocked", %{ assert Enum.empty?(response) end + + test "does not fail on an unauthententicated request", %{conn: conn, activity: activity} do + other_user = insert(:user) + {:ok, _, _} = CommonAPI.favorite(activity.id, other_user) + + response = + conn + |> assign(:user, nil) + |> get("/api/v1/#{activity.id}/favourited_by") + |> json_response(:ok) + + [%{"id" => id}] = response + assert id == other_user.id + end end describe "GET /api/v1/statuses/:id/reblogged_by" do @@ -3843,6 +3857,20 @@ test "does not return users who have reblogged the status but are blocked", %{ assert Enum.empty?(response) end + + test "does not fail on an unauthententicated request", %{conn: conn, activity: activity} do + other_user = insert(:user) + {:ok, _, _} = CommonAPI.favorite(activity.id, other_user) + + response = + conn + |> assign(:user, nil) + |> get("/api/v1/#{activity.id}/reblogged_by") + |> json_response(:ok) + + [%{"id" => id}] = response + assert id == other_user.id + end end describe "POST /auth/password, with valid parameters" do From 452980652dc749d71e96b1cbb17d68d393121a78 Mon Sep 17 00:00:00 2001 From: Sadposter Date: Tue, 23 Jul 2019 15:13:05 +0100 Subject: [PATCH 32/45] Mix format --- .../mastodon_api/mastodon_api_controller.ex | 24 +++++++------ .../mastodon_api_controller_test.exs | 36 +++++++++---------- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index ccebcd415..9269a5a29 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -885,11 +885,13 @@ def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do q = from(u in User, where: u.ap_id in ^likes) users = Repo.all(q) - users = if is_nil(user) do - users - else - Enum.filter(users, &(not User.blocks?(user, &1))) - end + + users = + if is_nil(user) do + users + else + Enum.filter(users, &(not User.blocks?(user, &1))) + end conn |> put_view(AccountView) @@ -905,11 +907,13 @@ def reblogged_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do q = from(u in User, where: u.ap_id in ^announces) users = Repo.all(q) - users = if is_nil(user) do - users - else - Enum.filter(users, &(not User.blocks?(user, &1))) - end + + users = + if is_nil(user) do + users + else + Enum.filter(users, &(not User.blocks?(user, &1))) + end conn |> put_view(AccountView) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 00ca320d3..49650b1de 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -3788,17 +3788,17 @@ test "does not return users who have favorited the status but are blocked", %{ end test "does not fail on an unauthententicated request", %{conn: conn, activity: activity} do - other_user = insert(:user) - {:ok, _, _} = CommonAPI.favorite(activity.id, other_user) + other_user = insert(:user) + {:ok, _, _} = CommonAPI.favorite(activity.id, other_user) - response = - conn - |> assign(:user, nil) - |> get("/api/v1/#{activity.id}/favourited_by") - |> json_response(:ok) + response = + conn + |> assign(:user, nil) + |> get("/api/v1/#{activity.id}/favourited_by") + |> json_response(:ok) - [%{"id" => id}] = response - assert id == other_user.id + [%{"id" => id}] = response + assert id == other_user.id end end @@ -3859,17 +3859,17 @@ test "does not return users who have reblogged the status but are blocked", %{ end test "does not fail on an unauthententicated request", %{conn: conn, activity: activity} do - other_user = insert(:user) - {:ok, _, _} = CommonAPI.favorite(activity.id, other_user) + other_user = insert(:user) + {:ok, _, _} = CommonAPI.favorite(activity.id, other_user) - response = - conn - |> assign(:user, nil) - |> get("/api/v1/#{activity.id}/reblogged_by") - |> json_response(:ok) + response = + conn + |> assign(:user, nil) + |> get("/api/v1/#{activity.id}/reblogged_by") + |> json_response(:ok) - [%{"id" => id}] = response - assert id == other_user.id + [%{"id" => id}] = response + assert id == other_user.id end end From 7026018c8c604ce9e077b13e14c35bd8d7052e2c Mon Sep 17 00:00:00 2001 From: Sadposter Date: Tue, 23 Jul 2019 15:31:35 +0100 Subject: [PATCH 33/45] Use correct URL for tests --- test/web/mastodon_api/mastodon_api_controller_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 49650b1de..28d3f4117 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -3794,7 +3794,7 @@ test "does not fail on an unauthententicated request", %{conn: conn, activity: a response = conn |> assign(:user, nil) - |> get("/api/v1/#{activity.id}/favourited_by") + |> get("/api/v1/statuses/#{activity.id}/favourited_by") |> json_response(:ok) [%{"id" => id}] = response @@ -3865,7 +3865,7 @@ test "does not fail on an unauthententicated request", %{conn: conn, activity: a response = conn |> assign(:user, nil) - |> get("/api/v1/#{activity.id}/reblogged_by") + |> get("/api/v1/statuses/#{activity.id}/reblogged_by") |> json_response(:ok) [%{"id" => id}] = response From 299c0e965b4b0d917a9daf696dd39ee546b33185 Mon Sep 17 00:00:00 2001 From: Sadposter Date: Tue, 23 Jul 2019 15:38:19 +0100 Subject: [PATCH 34/45] actually reblog on the reblog test --- test/web/mastodon_api/mastodon_api_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 28d3f4117..bd756c467 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -3860,7 +3860,7 @@ test "does not return users who have reblogged the status but are blocked", %{ test "does not fail on an unauthententicated request", %{conn: conn, activity: activity} do other_user = insert(:user) - {:ok, _, _} = CommonAPI.favorite(activity.id, other_user) + {:ok, _, _} = CommonAPI.repeat(activity.id, other_user) response = conn From c4005654279fe45213a3d11b6e4767e8afd24850 Mon Sep 17 00:00:00 2001 From: Sadposter Date: Tue, 23 Jul 2019 15:47:17 +0100 Subject: [PATCH 35/45] fix test names because i cannot type --- test/web/mastodon_api/mastodon_api_controller_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index bd756c467..bc3213e0c 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -3787,7 +3787,7 @@ test "does not return users who have favorited the status but are blocked", %{ assert Enum.empty?(response) end - test "does not fail on an unauthententicated request", %{conn: conn, activity: activity} do + test "does not fail on an unauthenticated request", %{conn: conn, activity: activity} do other_user = insert(:user) {:ok, _, _} = CommonAPI.favorite(activity.id, other_user) @@ -3858,7 +3858,7 @@ test "does not return users who have reblogged the status but are blocked", %{ assert Enum.empty?(response) end - test "does not fail on an unauthententicated request", %{conn: conn, activity: activity} do + test "does not fail on an unauthenticated request", %{conn: conn, activity: activity} do other_user = insert(:user) {:ok, _, _} = CommonAPI.repeat(activity.id, other_user) From 2e697f87f45f2982b53c4a0ec73f14526b6f8cf4 Mon Sep 17 00:00:00 2001 From: oncletom Date: Tue, 23 Jul 2019 15:16:47 +0000 Subject: [PATCH 36/45] Update `prometheus_phoenix` to v1.3 in order to support `phoenix@1.4`. --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index c12b0a500..e69940c5d 100644 --- a/mix.exs +++ b/mix.exs @@ -143,7 +143,7 @@ defp deps do {:telemetry, "~> 0.3"}, {:prometheus_ex, "~> 3.0"}, {:prometheus_plugs, "~> 1.1"}, - {:prometheus_phoenix, "~> 1.2"}, + {:prometheus_phoenix, "~> 1.3"}, {:prometheus_ecto, "~> 1.4"}, {:recon, github: "ferd/recon", tag: "2.4.0"}, {:quack, "~> 0.1.1"}, From e7c64f106eb578f802d000ecd8dacbc00a357b66 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 23 Jul 2019 16:47:22 +0000 Subject: [PATCH 37/45] signature: properly deduce the actor from misskey key IDs --- lib/pleroma/signature.ex | 15 ++++++++++++--- test/signature_test.exs | 21 +++++++++++++++------ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex index 2a0823ecf..0bf49fd7c 100644 --- a/lib/pleroma/signature.ex +++ b/lib/pleroma/signature.ex @@ -10,9 +10,18 @@ defmodule Pleroma.Signature do alias Pleroma.Web.ActivityPub.ActivityPub def key_id_to_actor_id(key_id) do - URI.parse(key_id) - |> Map.put(:fragment, nil) - |> URI.to_string() + uri = + URI.parse(key_id) + |> Map.put(:fragment, nil) + + uri = + if String.ends_with?(uri.path, "/publickey") do + Map.put(uri, :path, String.replace(uri.path, "/publickey", "")) + else + uri + end + + URI.to_string(uri) end def fetch_public_key(conn) do diff --git a/test/signature_test.exs b/test/signature_test.exs index 7400cae9a..26337eaf9 100644 --- a/test/signature_test.exs +++ b/test/signature_test.exs @@ -48,16 +48,14 @@ test "it returns key" do test "it returns error when not found user" do assert capture_log(fn -> - assert Signature.fetch_public_key(make_fake_conn("test-ap_id")) == - {:error, :error} + assert Signature.fetch_public_key(make_fake_conn("test-ap_id")) == {:error, :error} end) =~ "[error] Could not decode user" end test "it returns error if public key is empty" do user = insert(:user, %{info: %{source_data: %{"publicKey" => %{}}}}) - assert Signature.fetch_public_key(make_fake_conn(user.ap_id)) == - {:error, :error} + assert Signature.fetch_public_key(make_fake_conn(user.ap_id)) == {:error, :error} end end @@ -65,8 +63,7 @@ test "it returns error if public key is empty" do test "it returns key" do ap_id = "https://mastodon.social/users/lambadalambda" - assert Signature.refetch_public_key(make_fake_conn(ap_id)) == - {:ok, @rsa_public_key} + assert Signature.refetch_public_key(make_fake_conn(ap_id)) == {:ok, @rsa_public_key} end test "it returns error when not found user" do @@ -105,4 +102,16 @@ test "it returns error" do ) == {:error, []} end end + + describe "key_id_to_actor_id/1" do + test "it properly deduces the actor id for misskey" do + assert Signature.key_id_to_actor_id("https://example.com/users/1234/publickey") == + "https://example.com/users/1234" + end + + test "it properly deduces the actor id for mastodon and pleroma" do + assert Signature.key_id_to_actor_id("https://example.com/users/1234#main-key") == + "https://example.com/users/1234" + end + end end From fd287387a042b86a62d80c41b1dd282316b6609b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 23 Jul 2019 13:14:26 -0500 Subject: [PATCH 38/45] Do not notify subscribers for messages from users which are replies to others --- lib/pleroma/web/common_api/utils.ex | 6 ++++++ test/notification_test.exs | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index fcc000969..6f0f56d96 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -439,6 +439,12 @@ def maybe_notify_mentioned_recipients( def maybe_notify_mentioned_recipients(recipients, _), do: recipients + def maybe_notify_subscribers(_, %Activity{ + data: %{"object" => %Object{data: %{"inReplyTo" => _ap_id}}} + }) do + :nothing + end + def maybe_notify_subscribers( recipients, %Activity{data: %{"actor" => actor, "type" => type}} = activity diff --git a/test/notification_test.exs b/test/notification_test.exs index dda570b49..06f0b6557 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -42,6 +42,24 @@ test "it creates a notification for subscribed users" do assert notification.user_id == subscriber.id end + + test "does not create a notification for subscribed users if status is a reply" do + user = insert(:user) + other_user = insert(:user) + subscriber = insert(:user) + + User.subscribe(subscriber, other_user) + + {:ok, activity} = CommonAPI.post(user, %{"status" => "test post"}) + + {:ok, reply_activity} = + CommonAPI.post(other_user, %{ + "status" => "test reply", + "in_reply_to_status_id" => activity.id + }) + + refute Notification.create_notification(reply_activity, subscriber) + end end describe "create_notification" do From 54a161cb7ad58da05ced24daaf0c16964f76fa4c Mon Sep 17 00:00:00 2001 From: Sadposter Date: Tue, 23 Jul 2019 19:44:47 +0100 Subject: [PATCH 39/45] move unauth'd user blocks?/2 check --- lib/pleroma/user.ex | 2 ++ .../mastodon_api/mastodon_api_controller.ex | 18 ++++-------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index a3f6add28..e017efad6 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -882,6 +882,8 @@ def blocks?(%User{info: info} = _user, %{ap_id: ap_id}) do Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, host) end + def blocks?(nil, _), do: false + def subscribed_to?(user, %{ap_id: ap_id}) do with %User{} = target <- get_cached_by_ap_id(ap_id) do Enum.member?(target.info.subscribers, user.ap_id) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 9269a5a29..d660f3f05 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -884,14 +884,9 @@ def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do %Object{data: %{"likes" => likes}} <- Object.normalize(object) do q = from(u in User, where: u.ap_id in ^likes) - users = Repo.all(q) - users = - if is_nil(user) do - users - else - Enum.filter(users, &(not User.blocks?(user, &1))) - end + Repo.all(q) + |> Enum.filter(&(not User.blocks?(user, &1))) conn |> put_view(AccountView) @@ -906,14 +901,9 @@ def reblogged_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do %Object{data: %{"announcements" => announces}} <- Object.normalize(object) do q = from(u in User, where: u.ap_id in ^announces) - users = Repo.all(q) - users = - if is_nil(user) do - users - else - Enum.filter(users, &(not User.blocks?(user, &1))) - end + Repo.all(q) + |> Enum.filter(&(not User.blocks?(user, &1))) conn |> put_view(AccountView) From 6a79bb12c38bce6287b29c79c1ad3b7f9b967b69 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 23 Jul 2019 13:53:05 -0500 Subject: [PATCH 40/45] Fix function --- lib/pleroma/web/common_api/utils.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 6f0f56d96..94462c3dd 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -439,10 +439,11 @@ def maybe_notify_mentioned_recipients( def maybe_notify_mentioned_recipients(recipients, _), do: recipients - def maybe_notify_subscribers(_, %Activity{ - data: %{"object" => %Object{data: %{"inReplyTo" => _ap_id}}} + # Do not notify subscribers if author is making a reply + def maybe_notify_subscribers(recipients, %Activity{ + object: %Object{data: %{"inReplyTo" => _ap_id}} }) do - :nothing + recipients end def maybe_notify_subscribers( From ec7b085b76996bee7eaa60c21b9d8a0cba382a65 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 23 Jul 2019 13:57:22 -0500 Subject: [PATCH 41/45] Fix test --- test/notification_test.exs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/notification_test.exs b/test/notification_test.exs index 06f0b6557..28f8df49d 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -52,13 +52,17 @@ test "does not create a notification for subscribed users if status is a reply" {:ok, activity} = CommonAPI.post(user, %{"status" => "test post"}) - {:ok, reply_activity} = + {:ok, _reply_activity} = CommonAPI.post(other_user, %{ "status" => "test reply", "in_reply_to_status_id" => activity.id }) - refute Notification.create_notification(reply_activity, subscriber) + user_notifications = Notification.for_user(user) + assert length(user_notifications) == 1 + + subscriber_notifications = Notification.for_user(subscriber) + assert Enum.empty?(subscriber_notifications) end end From c49a09ed88c3cef0f3df3e97cf4fa5367cd8f830 Mon Sep 17 00:00:00 2001 From: Maksim Date: Tue, 23 Jul 2019 19:15:48 +0000 Subject: [PATCH 42/45] tests for Pleroma.Web.ActivityPub.Publisher --- lib/pleroma/user.ex | 11 ++ lib/pleroma/web/activity_pub/publisher.ex | 8 +- lib/pleroma/web/activity_pub/visibility.ex | 13 +- test/support/factory.ex | 23 +-- test/web/activity_pub/activity_pub_test.exs | 109 -------------- test/web/activity_pub/publisher_test.exs | 154 ++++++++++++++++++++ test/web/federator_test.exs | 9 ++ 7 files changed, 197 insertions(+), 130 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index e017efad6..982ca8bc1 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -586,12 +586,23 @@ def get_followers_query(user, page) do @spec get_followers_query(User.t()) :: Ecto.Query.t() def get_followers_query(user), do: get_followers_query(user, nil) + @spec get_followers(User.t(), pos_integer()) :: {:ok, list(User.t())} def get_followers(user, page \\ nil) do q = get_followers_query(user, page) {:ok, Repo.all(q)} end + @spec get_external_followers(User.t(), pos_integer()) :: {:ok, list(User.t())} + def get_external_followers(user, page \\ nil) do + q = + user + |> get_followers_query(page) + |> User.Query.build(%{external: true}) + + {:ok, Repo.all(q)} + end + def get_followers_ids(user, page \\ nil) do q = get_followers_query(user, page) diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 0bbe6ee80..016d78216 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -97,13 +97,13 @@ defp should_federate?(inbox, public) do end end + @spec recipients(User.t(), Activity.t()) :: list(User.t()) | [] defp recipients(actor, activity) do - followers = + {:ok, followers} = if actor.follower_address in activity.recipients do - {:ok, followers} = User.get_followers(actor) - Enum.filter(followers, &(!&1.local)) + User.get_external_followers(actor) else - [] + {:ok, []} end Pleroma.Web.Salmon.remote_users(actor, activity) ++ followers diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex index 2666edc7c..097fceb08 100644 --- a/lib/pleroma/web/activity_pub/visibility.ex +++ b/lib/pleroma/web/activity_pub/visibility.ex @@ -8,14 +8,14 @@ defmodule Pleroma.Web.ActivityPub.Visibility do alias Pleroma.Repo alias Pleroma.User + @public "https://www.w3.org/ns/activitystreams#Public" + + @spec is_public?(Object.t() | Activity.t() | map()) :: boolean() def is_public?(%Object{data: %{"type" => "Tombstone"}}), do: false def is_public?(%Object{data: data}), do: is_public?(data) def is_public?(%Activity{data: data}), do: is_public?(data) def is_public?(%{"directMessage" => true}), do: false - - def is_public?(data) do - "https://www.w3.org/ns/activitystreams#Public" in (data["to"] ++ (data["cc"] || [])) - end + def is_public?(data), do: @public in (data["to"] ++ (data["cc"] || [])) def is_private?(activity) do with false <- is_public?(activity), @@ -69,15 +69,14 @@ def entire_thread_visible_for_user?(%Activity{} = activity, %User{} = user) do end def get_visibility(object) do - public = "https://www.w3.org/ns/activitystreams#Public" to = object.data["to"] || [] cc = object.data["cc"] || [] cond do - public in to -> + @public in to -> "public" - public in cc -> + @public in cc -> "unlisted" # this should use the sql for the object's activity diff --git a/test/support/factory.ex b/test/support/factory.ex index 531eb81e4..1f4239213 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -118,17 +118,20 @@ def direct_note_activity_factory do def note_activity_factory(attrs \\ %{}) do user = attrs[:user] || insert(:user) note = attrs[:note] || insert(:note, user: user) - attrs = Map.drop(attrs, [:user, :note]) + data_attrs = attrs[:data_attrs] || %{} + attrs = Map.drop(attrs, [:user, :note, :data_attrs]) - data = %{ - "id" => Pleroma.Web.ActivityPub.Utils.generate_activity_id(), - "type" => "Create", - "actor" => note.data["actor"], - "to" => note.data["to"], - "object" => note.data["id"], - "published" => DateTime.utc_now() |> DateTime.to_iso8601(), - "context" => note.data["context"] - } + data = + %{ + "id" => Pleroma.Web.ActivityPub.Utils.generate_activity_id(), + "type" => "Create", + "actor" => note.data["actor"], + "to" => note.data["to"], + "object" => note.data["id"], + "published" => DateTime.utc_now() |> DateTime.to_iso8601(), + "context" => note.data["context"] + } + |> Map.merge(data_attrs) %Pleroma.Activity{ data: data, diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 00adbc0f9..1c0b274cb 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -6,11 +6,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do use Pleroma.DataCase alias Pleroma.Activity alias Pleroma.Builders.ActivityBuilder - alias Pleroma.Instances alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.ActivityPub.Publisher alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.CommonAPI @@ -1083,113 +1081,6 @@ test "it can create a Flag activity" do } = activity end - describe "publish_one/1" do - test_with_mock "calls `Instances.set_reachable` on successful federation if `unreachable_since` is not specified", - Instances, - [:passthrough], - [] do - actor = insert(:user) - inbox = "http://200.site/users/nick1/inbox" - - assert {:ok, _} = Publisher.publish_one(%{inbox: inbox, json: "{}", actor: actor, id: 1}) - - assert called(Instances.set_reachable(inbox)) - end - - test_with_mock "calls `Instances.set_reachable` on successful federation if `unreachable_since` is set", - Instances, - [:passthrough], - [] do - actor = insert(:user) - inbox = "http://200.site/users/nick1/inbox" - - assert {:ok, _} = - Publisher.publish_one(%{ - inbox: inbox, - json: "{}", - actor: actor, - id: 1, - unreachable_since: NaiveDateTime.utc_now() - }) - - assert called(Instances.set_reachable(inbox)) - end - - test_with_mock "does NOT call `Instances.set_reachable` on successful federation if `unreachable_since` is nil", - Instances, - [:passthrough], - [] do - actor = insert(:user) - inbox = "http://200.site/users/nick1/inbox" - - assert {:ok, _} = - Publisher.publish_one(%{ - inbox: inbox, - json: "{}", - actor: actor, - id: 1, - unreachable_since: nil - }) - - refute called(Instances.set_reachable(inbox)) - end - - test_with_mock "calls `Instances.set_unreachable` on target inbox on non-2xx HTTP response code", - Instances, - [:passthrough], - [] do - actor = insert(:user) - inbox = "http://404.site/users/nick1/inbox" - - assert {:error, _} = Publisher.publish_one(%{inbox: inbox, json: "{}", actor: actor, id: 1}) - - assert called(Instances.set_unreachable(inbox)) - end - - test_with_mock "it calls `Instances.set_unreachable` on target inbox on request error of any kind", - Instances, - [:passthrough], - [] do - actor = insert(:user) - inbox = "http://connrefused.site/users/nick1/inbox" - - assert {:error, _} = Publisher.publish_one(%{inbox: inbox, json: "{}", actor: actor, id: 1}) - - assert called(Instances.set_unreachable(inbox)) - end - - test_with_mock "does NOT call `Instances.set_unreachable` if target is reachable", - Instances, - [:passthrough], - [] do - actor = insert(:user) - inbox = "http://200.site/users/nick1/inbox" - - assert {:ok, _} = Publisher.publish_one(%{inbox: inbox, json: "{}", actor: actor, id: 1}) - - refute called(Instances.set_unreachable(inbox)) - end - - test_with_mock "does NOT call `Instances.set_unreachable` if target instance has non-nil `unreachable_since`", - Instances, - [:passthrough], - [] do - actor = insert(:user) - inbox = "http://connrefused.site/users/nick1/inbox" - - assert {:error, _} = - Publisher.publish_one(%{ - inbox: inbox, - json: "{}", - actor: actor, - id: 1, - unreachable_since: NaiveDateTime.utc_now() - }) - - refute called(Instances.set_unreachable(inbox)) - end - end - test "fetch_activities/2 returns activities addressed to a list " do user = insert(:user) member = insert(:user) diff --git a/test/web/activity_pub/publisher_test.exs b/test/web/activity_pub/publisher_test.exs index 2b15b3edc..36a39c84c 100644 --- a/test/web/activity_pub/publisher_test.exs +++ b/test/web/activity_pub/publisher_test.exs @@ -6,12 +6,20 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do use Pleroma.DataCase import Pleroma.Factory + import Tesla.Mock + import Mock alias Pleroma.Activity + alias Pleroma.Instances alias Pleroma.Web.ActivityPub.Publisher @as_public "https://www.w3.org/ns/activitystreams#Public" + setup do + mock(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + describe "determine_inbox/2" do test "it returns sharedInbox for messages involving as:Public in to" do user = @@ -109,4 +117,150 @@ test "it returns inbox for messages involving single recipients in total" do assert Publisher.determine_inbox(activity, user) == "http://example.com/personal-inbox" end end + + describe "publish_one/1" do + test_with_mock "calls `Instances.set_reachable` on successful federation if `unreachable_since` is not specified", + Instances, + [:passthrough], + [] do + actor = insert(:user) + inbox = "http://200.site/users/nick1/inbox" + + assert {:ok, _} = Publisher.publish_one(%{inbox: inbox, json: "{}", actor: actor, id: 1}) + + assert called(Instances.set_reachable(inbox)) + end + + test_with_mock "calls `Instances.set_reachable` on successful federation if `unreachable_since` is set", + Instances, + [:passthrough], + [] do + actor = insert(:user) + inbox = "http://200.site/users/nick1/inbox" + + assert {:ok, _} = + Publisher.publish_one(%{ + inbox: inbox, + json: "{}", + actor: actor, + id: 1, + unreachable_since: NaiveDateTime.utc_now() + }) + + assert called(Instances.set_reachable(inbox)) + end + + test_with_mock "does NOT call `Instances.set_reachable` on successful federation if `unreachable_since` is nil", + Instances, + [:passthrough], + [] do + actor = insert(:user) + inbox = "http://200.site/users/nick1/inbox" + + assert {:ok, _} = + Publisher.publish_one(%{ + inbox: inbox, + json: "{}", + actor: actor, + id: 1, + unreachable_since: nil + }) + + refute called(Instances.set_reachable(inbox)) + end + + test_with_mock "calls `Instances.set_unreachable` on target inbox on non-2xx HTTP response code", + Instances, + [:passthrough], + [] do + actor = insert(:user) + inbox = "http://404.site/users/nick1/inbox" + + assert {:error, _} = Publisher.publish_one(%{inbox: inbox, json: "{}", actor: actor, id: 1}) + + assert called(Instances.set_unreachable(inbox)) + end + + test_with_mock "it calls `Instances.set_unreachable` on target inbox on request error of any kind", + Instances, + [:passthrough], + [] do + actor = insert(:user) + inbox = "http://connrefused.site/users/nick1/inbox" + + assert {:error, _} = Publisher.publish_one(%{inbox: inbox, json: "{}", actor: actor, id: 1}) + + assert called(Instances.set_unreachable(inbox)) + end + + test_with_mock "does NOT call `Instances.set_unreachable` if target is reachable", + Instances, + [:passthrough], + [] do + actor = insert(:user) + inbox = "http://200.site/users/nick1/inbox" + + assert {:ok, _} = Publisher.publish_one(%{inbox: inbox, json: "{}", actor: actor, id: 1}) + + refute called(Instances.set_unreachable(inbox)) + end + + test_with_mock "does NOT call `Instances.set_unreachable` if target instance has non-nil `unreachable_since`", + Instances, + [:passthrough], + [] do + actor = insert(:user) + inbox = "http://connrefused.site/users/nick1/inbox" + + assert {:error, _} = + Publisher.publish_one(%{ + inbox: inbox, + json: "{}", + actor: actor, + id: 1, + unreachable_since: NaiveDateTime.utc_now() + }) + + refute called(Instances.set_unreachable(inbox)) + end + end + + describe "publish/2" do + test_with_mock "publishes an activity with BCC to all relevant peers.", + Pleroma.Web.Federator.Publisher, + [:passthrough], + [] do + follower = + insert(:user, + local: false, + info: %{ + ap_enabled: true, + source_data: %{"inbox" => "https://domain.com/users/nick1/inbox"} + } + ) + + actor = insert(:user, follower_address: follower.ap_id) + user = insert(:user) + + {:ok, _follower_one} = Pleroma.User.follow(follower, actor) + actor = refresh_record(actor) + + note_activity = + insert(:note_activity, + recipients: [follower.ap_id], + data_attrs: %{"bcc" => [user.ap_id]} + ) + + res = Publisher.publish(actor, note_activity) + assert res == :ok + + assert called( + Pleroma.Web.Federator.Publisher.enqueue_one(Publisher, %{ + inbox: "https://domain.com/users/nick1/inbox", + actor: actor, + id: note_activity.data["id"] + }) + ) + end + end end diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index 69dd4d747..6e143eee4 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -22,6 +22,15 @@ defmodule Pleroma.Web.FederatorTest do :ok end + describe "Publisher.perform" do + test "call `perform` with unknown task" do + assert { + :error, + "Don't know what to do with this" + } = Pleroma.Web.Federator.Publisher.perform("test", :ok, :ok) + end + end + describe "Publish an activity" do setup do user = insert(:user) From 5e72554f3c6490ebdaaa8238f34860fa362016fc Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 23 Jul 2019 19:17:00 +0000 Subject: [PATCH 43/45] Admin config fix --- docs/api/admin_api.md | 2 ++ lib/pleroma/web/admin_api/config.ex | 14 ++++++++++++-- test/web/admin_api/admin_api_controller_test.exs | 6 ++++-- test/web/admin_api/config_test.exs | 8 ++++++++ 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index c429da822..5ac3535c4 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -564,6 +564,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret ## `/api/pleroma/admin/config` ### List config settings +List config settings only works with `:pleroma => :instance => :dynamic_configuration` setting to `true`. - Method `GET` - Params: none - Response: @@ -582,6 +583,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret ## `/api/pleroma/admin/config` ### Update config settings +Updating config settings only works with `:pleroma => :instance => :dynamic_configuration` setting to `true`. Module name can be passed as string, which starts with `Pleroma`, e.g. `"Pleroma.Upload"`. Atom keys and values can be passed with `:` in the beginning, e.g. `":upload"`. Tuples can be passed as `{"tuple": ["first_val", Pleroma.Module, []]}`. diff --git a/lib/pleroma/web/admin_api/config.ex b/lib/pleroma/web/admin_api/config.ex index b4eb8e002..dde05ea7b 100644 --- a/lib/pleroma/web/admin_api/config.ex +++ b/lib/pleroma/web/admin_api/config.ex @@ -84,6 +84,7 @@ defp do_convert(entity) when is_map(entity) do end defp do_convert({:dispatch, [entity]}), do: %{"tuple" => [":dispatch", [inspect(entity)]]} + defp do_convert({:partial_chain, entity}), do: %{"tuple" => [":partial_chain", inspect(entity)]} defp do_convert(entity) when is_tuple(entity), do: %{"tuple" => do_convert(Tuple.to_list(entity))} @@ -113,11 +114,15 @@ def transform(entity), do: :erlang.term_to_binary(entity) defp do_transform(%Regex{} = entity) when is_map(entity), do: entity defp do_transform(%{"tuple" => [":dispatch", [entity]]}) do - cleaned_string = String.replace(entity, ~r/[^\w|^{:,[|^,|^[|^\]^}|^\/|^\.|^"]^\s/, "") - {dispatch_settings, []} = Code.eval_string(cleaned_string, [], requires: [], macros: []) + {dispatch_settings, []} = do_eval(entity) {:dispatch, [dispatch_settings]} end + defp do_transform(%{"tuple" => [":partial_chain", entity]}) do + {partial_chain, []} = do_eval(entity) + {:partial_chain, partial_chain} + end + defp do_transform(%{"tuple" => entity}) do Enum.reduce(entity, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end) end @@ -149,4 +154,9 @@ defp do_transform_string(value) do do: String.to_existing_atom("Elixir." <> value), else: value end + + defp do_eval(entity) do + cleaned_string = String.replace(entity, ~r/[^\w|^{:,[|^,|^[|^\]^}|^\/|^\.|^"]^\s/, "") + Code.eval_string(cleaned_string, [], requires: [], macros: []) + end end diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 03aa46cae..1306c341d 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -1571,7 +1571,8 @@ test "common config example", %{conn: conn} do %{"tuple" => [":method", "Pleroma.Captcha.Kocaptcha"]}, %{"tuple" => [":seconds_valid", 60]}, %{"tuple" => [":path", ""]}, - %{"tuple" => [":key1", nil]} + %{"tuple" => [":key1", nil]}, + %{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]} ] } ] @@ -1587,7 +1588,8 @@ test "common config example", %{conn: conn} do %{"tuple" => [":method", "Pleroma.Captcha.Kocaptcha"]}, %{"tuple" => [":seconds_valid", 60]}, %{"tuple" => [":path", ""]}, - %{"tuple" => [":key1", nil]} + %{"tuple" => [":key1", nil]}, + %{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]} ] } ] diff --git a/test/web/admin_api/config_test.exs b/test/web/admin_api/config_test.exs index d41666ef3..3190dc1c8 100644 --- a/test/web/admin_api/config_test.exs +++ b/test/web/admin_api/config_test.exs @@ -238,6 +238,14 @@ test "simple keyword" do assert Config.from_binary(binary) == [key: "value"] end + test "keyword with partial_chain key" do + binary = + Config.transform([%{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]}]) + + assert binary == :erlang.term_to_binary(partial_chain: &:hackney_connect.partial_chain/1) + assert Config.from_binary(binary) == [partial_chain: &:hackney_connect.partial_chain/1] + end + test "keyword" do binary = Config.transform([ From 90be91b0e091dabd6db36dff92b13ce9dc251c5c Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 22 Jul 2019 13:41:56 +0200 Subject: [PATCH 44/45] Router: Remove deprecated AdminAPI endpoints --- lib/pleroma/web/router.ex | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index a33b5ddd7..d230788d0 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -154,22 +154,12 @@ defmodule Pleroma.Web.Router do post("/users/follow", AdminAPIController, :user_follow) post("/users/unfollow", AdminAPIController, :user_unfollow) - # TODO: to be removed at version 1.0 - delete("/user", AdminAPIController, :user_delete) - post("/user", AdminAPIController, :user_create) - delete("/users", AdminAPIController, :user_delete) post("/users", AdminAPIController, :user_create) patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation) put("/users/tag", AdminAPIController, :tag_users) delete("/users/tag", AdminAPIController, :untag_users) - # TODO: to be removed at version 1.0 - get("/permission_group/:nickname", AdminAPIController, :right_get) - get("/permission_group/:nickname/:permission_group", AdminAPIController, :right_get) - post("/permission_group/:nickname/:permission_group", AdminAPIController, :right_add) - delete("/permission_group/:nickname/:permission_group", AdminAPIController, :right_delete) - get("/users/:nickname/permission_group", AdminAPIController, :right_get) get("/users/:nickname/permission_group/:permission_group", AdminAPIController, :right_get) post("/users/:nickname/permission_group/:permission_group", AdminAPIController, :right_add) @@ -190,9 +180,6 @@ defmodule Pleroma.Web.Router do post("/users/revoke_invite", AdminAPIController, :revoke_invite) post("/users/email_invite", AdminAPIController, :email_invite) - # TODO: to be removed at version 1.0 - get("/password_reset", AdminAPIController, :get_password_reset) - get("/users/:nickname/password_reset", AdminAPIController, :get_password_reset) get("/users", AdminAPIController, :list_users) From 0afaf9664030956567bdc3dba745b9bbc326bad5 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 23 Jul 2019 19:49:36 +0000 Subject: [PATCH 45/45] update mix.lock --- mix.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mix.lock b/mix.lock index 45142ba8f..5f20878d3 100644 --- a/mix.lock +++ b/mix.lock @@ -56,22 +56,22 @@ "nimble_parsec": {:hex, :nimble_parsec, "0.5.0", "90e2eca3d0266e5c53f8fbe0079694740b9c91b6747f2b7e3c5d21966bba8300", [:mix], [], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.3", "6706a148809a29c306062862c803406e88f048277f6e85b68faf73291e820b84", [:mix], [], "hexpm"}, - "phoenix": {:hex, :phoenix, "1.4.8", "c72dc3adeb49c70eb963a0ea24f7a064ec1588e651e84e1b7ad5ed8253c0b4a2", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "phoenix": {:hex, :phoenix, "1.4.9", "746d098e10741c334d88143d3c94cab1756435f94387a63441792e66ec0ee974", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.0.0", "c43117a136e7399ea04ecaac73f8f23ee0ffe3e07acfcb8062fe5f4c9f0f6531", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "phoenix_html": {:hex, :phoenix_html, "2.13.1", "fa8f034b5328e2dfa0e4131b5569379003f34bc1fafdaa84985b0b9d2f12e68b", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "1.1.2", "496c303bdf1b2e98a9d26e89af5bba3ab487ba3a3735f74bf1f4064d2a845a3e", [:mix], [], "hexpm"}, "pleroma_job_queue": {:hex, :pleroma_job_queue, "0.2.0", "879e660aa1cebe8dc6f0aaaa6aa48b4875e89cd961d4a585fd128e0773b31a18", [:mix], [], "hexpm"}, "plug": {:hex, :plug, "1.8.2", "0bcce1daa420f189a6491f3940cc77ea7fb1919761175c9c3b59800d897440fc", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm"}, - "plug_cowboy": {:hex, :plug_cowboy, "2.0.2", "6055f16868cc4882b24b6e1d63d2bada94fb4978413377a3b32ac16c18dffba2", [:mix], [{:cowboy, "~> 2.5", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, + "plug_cowboy": {:hex, :plug_cowboy, "2.1.0", "b75768153c3a8a9e8039d4b25bb9b14efbc58e9c4a6e6a270abff1cd30cbe320", [:mix], [{:cowboy, "~> 2.5", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"}, "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, - "prometheus": {:hex, :prometheus, "4.2.2", "a830e77b79dc6d28183f4db050a7cac926a6c58f1872f9ef94a35cd989aceef8", [:mix, :rebar3], [], "hexpm"}, + "prometheus": {:hex, :prometheus, "4.4.1", "1e96073b3ed7788053768fea779cbc896ddc3bdd9ba60687f2ad50b252ac87d6", [:mix, :rebar3], [], "hexpm"}, "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, - "prometheus_phoenix": {:hex, :prometheus_phoenix, "1.2.1", "964a74dfbc055f781d3a75631e06ce3816a2913976d1df7830283aa3118a797a", [:mix], [{:phoenix, "~> 1.3", [hex: :phoenix, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, + "prometheus_phoenix": {:hex, :prometheus_phoenix, "1.3.0", "c4b527e0b3a9ef1af26bdcfbfad3998f37795b9185d475ca610fe4388fdd3bb5", [:mix], [{:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, "prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm"}, "prometheus_process_collector": {:hex, :prometheus_process_collector, "1.4.0", "6dbd39e3165b9ef1c94a7a820e9ffe08479f949dcdd431ed4aaea7b250eebfde", [:rebar3], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, "quack": {:hex, :quack, "0.1.1", "cca7b4da1a233757fdb44b3334fce80c94785b3ad5a602053b7a002b5a8967bf", [:mix], [{:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}, {:tesla, "~> 1.2.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm"},