From 01c45ddc9ead715131b3c583caa14fcf20845354 Mon Sep 17 00:00:00 2001 From: lain Date: Sat, 11 May 2019 11:26:46 +0200 Subject: [PATCH 01/56] Search: Use RUM index. --- .../mastodon_api/mastodon_api_controller.ex | 6 ++-- ...510135645_add_fts_index_to_objects_two.exs | 33 +++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 956736780..32677df95 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1019,12 +1019,12 @@ def status_search(user, query) do where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients, where: fragment( - "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)", - o.data, + "? @@ plainto_tsquery('english', ?)", + o.fts_content, ^query ), limit: 20, - order_by: [desc: :id] + order_by: [fragment("? <=> now()::date", o.inserted_at)] ) Repo.all(q) ++ fetched diff --git a/priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs b/priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs new file mode 100644 index 000000000..14b964847 --- /dev/null +++ b/priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs @@ -0,0 +1,33 @@ +defmodule Pleroma.Repo.Migrations.AddFtsIndexToObjectsTwo do + use Ecto.Migration + + def up do + drop_if_exists index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) + alter table(:objects) do + add(:fts_content, :tsvector) + end + + execute("CREATE FUNCTION objects_fts_update() RETURNS trigger AS $$ + begin + new.fts_content := to_tsvector('english', new.data->>'content'); + return new; + end + $$ LANGUAGE plpgsql") + execute("create index objects_fts on objects using RUM (fts_content rum_tsvector_addon_ops, inserted_at) with (attach = 'inserted_at', to = 'fts_content');") + + execute("CREATE TRIGGER tsvectorupdate BEFORE INSERT OR UPDATE ON objects + FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()") + + execute("UPDATE objects SET updated_at = NOW()") + end + + def down do + execute "drop index objects_fts" + execute "drop trigger tsvectorupdate on objects" + execute "drop function objects_fts_update()" + alter table(:objects) do + remove(:fts_content, :tsvector) + end + create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) + end +end From f1e67bdc312ba16a37916024244d6cb9d4417c9e Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 15 May 2019 15:28:01 +0200 Subject: [PATCH 02/56] Search: Add optional rum indexing / searching. --- config/config.exs | 2 + docs/config.md | 15 +++++++ .../mastodon_api/mastodon_api_controller.ex | 43 ++++++++++++++----- mix.exs | 5 ++- mix.lock | 10 ++--- ...510135645_add_fts_index_to_objects_two.exs | 3 ++ 6 files changed, 62 insertions(+), 16 deletions(-) rename priv/repo/{migrations => optional_migrations/rum_indexing}/20190510135645_add_fts_index_to_objects_two.exs (91%) diff --git a/config/config.exs b/config/config.exs index 1e64b79a7..42e4cb4ce 100644 --- a/config/config.exs +++ b/config/config.exs @@ -476,6 +476,8 @@ token_expires_in: 600, issue_new_refresh_token: true +config :pleroma, :database, rum_enabled: false + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/docs/config.md b/docs/config.md index 43ea24d80..99cee25cd 100644 --- a/docs/config.md +++ b/docs/config.md @@ -537,3 +537,18 @@ Configure OAuth 2 provider capabilities: * `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]` * `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]` * `default_manifest`: Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download. Currently only one manifest can be added (no arrays). + +## Database options + +### RUM indexing for full text search +* `rum_enabled`: If RUM indexes should be used. Defaults to `false`. + +RUM indexes are an alternative indexing scheme that is not included in PostgreSQL by default. While they may eventually be mainlined, for now they have to be installed as a PostgreSQL extension from https://github.com/postgrespro/rum. + +Their advantage over the standard GIN indexes is that they allow efficient ordering of search results by timestamp, which makes search queries a lot faster on larger servers, by one or two orders of magnitude. They take up around 3 times as much space as GIN indexes. + +To enable them, both the `rum_enabled` flag has to be set and the following special migration has to be run: + +`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/` + +This will probably take a long time. diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 32677df95..fc0100b1e 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1000,6 +1000,30 @@ def unsubscribe(%{assigns: %{user: user}} = conn, %{"id" => id}) do end end + def status_search_query_with_gin(q, query) do + from([a, o] in q, + where: + fragment( + "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)", + o.data, + ^query + ), + order_by: [desc: :id] + ) + end + + def status_search_query_with_rum(q, query) do + from([a, o] in q, + where: + fragment( + "? @@ plainto_tsquery('english', ?)", + o.fts_content, + ^query + ), + order_by: [fragment("? <=> now()::date", o.inserted_at)] + ) + end + def status_search(user, query) do fetched = if Regex.match?(~r/https?:/, query) do @@ -1013,20 +1037,19 @@ def status_search(user, query) do end || [] q = - from( - [a, o] in Activity.with_preloaded_object(Activity), + from([a, o] in Activity.with_preloaded_object(Activity), where: fragment("?->>'type' = 'Create'", a.data), where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients, - where: - fragment( - "? @@ plainto_tsquery('english', ?)", - o.fts_content, - ^query - ), - limit: 20, - order_by: [fragment("? <=> now()::date", o.inserted_at)] + limit: 20 ) + q = + if Pleroma.Config.get([:database, :rum_enabled]) do + status_search_query_with_rum(q, query) + else + status_search_query_with_gin(q, query) + end + Repo.all(q) ++ fetched end diff --git a/mix.exs b/mix.exs index fae21f18d..f7955c6ca 100644 --- a/mix.exs +++ b/mix.exs @@ -65,7 +65,10 @@ defp deps do {:plug_cowboy, "~> 2.0"}, {:phoenix_pubsub, "~> 1.1"}, {:phoenix_ecto, "~> 4.0"}, - {:ecto_sql, "~>3.0.5"}, + {:ecto_sql, + git: "https://github.com/elixir-ecto/ecto_sql", + ref: "e839a9a327b632d73533ac8105ba360bc831cf83", + override: true}, {:postgrex, ">= 0.13.5"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, diff --git a/mix.lock b/mix.lock index 624c0fb35..d223803ab 100644 --- a/mix.lock +++ b/mix.lock @@ -16,12 +16,12 @@ "cowlib": {:hex, :cowlib, "2.7.0", "3ef16e77562f9855a2605900cedb15c1462d76fb1be6a32fc3ae91973ee543d2", [:rebar3], [], "hexpm"}, "credo": {:hex, :credo, "0.9.3", "76fa3e9e497ab282e0cf64b98a624aa11da702854c52c82db1bf24e54ab7c97a", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]}, - "db_connection": {:hex, :db_connection, "2.0.5", "ddb2ba6761a08b2bb9ca0e7d260e8f4dd39067426d835c24491a321b7f92a4da", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, + "db_connection": {:hex, :db_connection, "2.0.6", "bde2f85d047969c5b5800cb8f4b3ed6316c8cb11487afedac4aa5f93fd39abfa", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, "decimal": {:hex, :decimal, "1.7.0", "30d6b52c88541f9a66637359ddf85016df9eb266170d53105f02e4a67e00c5aa", [:mix], [], "hexpm"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"}, "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"}, - "ecto": {:hex, :ecto, "3.0.7", "44dda84ac6b17bbbdeb8ac5dfef08b7da253b37a453c34ab1a98de7f7e5fec7f", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"}, - "ecto_sql": {:hex, :ecto_sql, "3.0.5", "7e44172b4f7aca4469f38d7f6a3da394dbf43a1bcf0ca975e958cb957becd74e", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0.6", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "ecto_sql": {:git, "https://github.com/elixir-ecto/ecto_sql", "e839a9a327b632d73533ac8105ba360bc831cf83", [ref: "e839a9a327b632d73533ac8105ba360bc831cf83"]}, "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"}, "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"}, "ex_aws": {:hex, :ex_aws, "2.1.0", "b92651527d6c09c479f9013caa9c7331f19cba38a650590d82ebf2c6c16a1d8a", [:mix], [{:configparser_ex, "~> 2.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:xml_builder, "~> 0.1.0", [hex: :xml_builder, repo: "hexpm", optional: true]}], "hexpm"}, @@ -61,7 +61,7 @@ "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, - "postgrex": {:hex, :postgrex, "0.14.1", "63247d4a5ad6b9de57a0bac5d807e1c32d41e39c04b8a4156a26c63bcd8a2e49", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, "prometheus": {:hex, :prometheus, "4.2.2", "a830e77b79dc6d28183f4db050a7cac926a6c58f1872f9ef94a35cd989aceef8", [:mix, :rebar3], [], "hexpm"}, "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, @@ -74,7 +74,7 @@ "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, "swoosh": {:hex, :swoosh, "0.20.0", "9a6c13822c9815993c03b6f8fccc370fcffb3c158d9754f67b1fdee6b3a5d928", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.12", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, - "telemetry": {:hex, :telemetry, "0.3.0", "099a7f3ce31e4780f971b4630a3c22ec66d22208bc090fe33a2a3a6a67754a73", [:rebar3], [], "hexpm"}, + "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, "tesla": {:hex, :tesla, "1.2.1", "864783cc27f71dd8c8969163704752476cec0f3a51eb3b06393b3971dc9733ff", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"}, "timex": {:hex, :timex, "3.5.0", "b0a23167da02d0fe4f1a4e104d1f929a00d348502b52432c05de875d0b9cffa5", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs similarity index 91% rename from priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs rename to priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs index 14b964847..09e6cbfb1 100644 --- a/priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs +++ b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs @@ -2,6 +2,7 @@ defmodule Pleroma.Repo.Migrations.AddFtsIndexToObjectsTwo do use Ecto.Migration def up do + execute("create extension if not exists rum") drop_if_exists index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) alter table(:objects) do add(:fts_content, :tsvector) @@ -19,6 +20,7 @@ def up do FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()") execute("UPDATE objects SET updated_at = NOW()") + execute("vacuum analyze") end def down do @@ -29,5 +31,6 @@ def down do remove(:fts_content, :tsvector) end create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) + execute("vacuum analyze") end end From a3fc7294da43a7e2d17626200e8d3bf5c05808d3 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 17 May 2019 12:47:38 +0200 Subject: [PATCH 03/56] CI: Add rum variant testing. --- .gitlab-ci.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f9745122a..3ea275127 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,6 +15,7 @@ cache: stages: - build - test + - test-rum - deploy before_script: @@ -45,7 +46,7 @@ docs-build: unit-testing: stage: test services: - - name: postgres:9.6.2 + - name: lainsoykaf/postgres-with-rum command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] script: - mix deps.get @@ -54,6 +55,20 @@ unit-testing: - mix test --trace --preload-modules - mix coveralls +unit-testing-rum: + stage: test-rum + services: + - name: lainsoykaf/postgres-with-rum + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + script: + - "echo 'config :pleroma, :database, rum_enabled: true' >> config/test.exs" + - mix deps.get + - mix ecto.create + - mix ecto.migrate + - "mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/" + - mix test --trace --preload-modules + - mix coveralls + lint: stage: test script: @@ -65,7 +80,6 @@ analysis: - mix deps.get - mix credo --strict --only=warnings,todo,fixme,consistency,readability - docs-deploy: stage: deploy image: alpine:3.9 From 022e6e4b44229be70f0ec6720a66610a0e2a403a Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 17 May 2019 14:59:01 +0200 Subject: [PATCH 04/56] RUM: Remove vacuum analyze from migration Can't be run in a trnasaction. --- .../20190510135645_add_fts_index_to_objects_two.exs | 2 -- 1 file changed, 2 deletions(-) diff --git a/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs index 09e6cbfb1..b6a24441a 100644 --- a/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs +++ b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs @@ -20,7 +20,6 @@ def up do FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()") execute("UPDATE objects SET updated_at = NOW()") - execute("vacuum analyze") end def down do @@ -31,6 +30,5 @@ def down do remove(:fts_content, :tsvector) end create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) - execute("vacuum analyze") end end From 8784a7d1b40394cb1b4d5ce2d04b8cd057a57ceb Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 17 May 2019 15:06:51 +0200 Subject: [PATCH 05/56] RUM: Set rum status by the environment. --- config/test.exs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/test.exs b/config/test.exs index 40db66170..e1785d10f 100644 --- a/config/test.exs +++ b/config/test.exs @@ -63,6 +63,10 @@ config :pleroma, :http_security, report_uri: "https://endpoint.com" +rum_enabled = System.get_env("RUM_ENABLED") == "true" +config :pleroma, :database, rum_enabled: rum_enabled +IO.puts("RUM enabled: #{rum_enabled}") + try do import_config "test.secret.exs" rescue From ef63cf70883f4c7e3814ebac907f000da835ea10 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 17 May 2019 15:21:29 +0200 Subject: [PATCH 06/56] CI: Use the correct image with the correct hostname. --- .gitlab-ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3ea275127..8b5131dc3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,7 +15,6 @@ cache: stages: - build - test - - test-rum - deploy before_script: @@ -47,6 +46,7 @@ unit-testing: stage: test services: - name: lainsoykaf/postgres-with-rum + alias: postgres command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] script: - mix deps.get @@ -56,18 +56,19 @@ unit-testing: - mix coveralls unit-testing-rum: - stage: test-rum + stage: test services: - name: lainsoykaf/postgres-with-rum + alias: postgres command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + variables: + RUM_ENABLED: "true" script: - - "echo 'config :pleroma, :database, rum_enabled: true' >> config/test.exs" - mix deps.get - mix ecto.create - mix ecto.migrate - "mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/" - mix test --trace --preload-modules - - mix coveralls lint: stage: test From f959bf7aa6b878ee5b669c4caabd5cdc4cc2dc9e Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 17 May 2019 18:21:11 +0200 Subject: [PATCH 07/56] MongooseIM: Add basic integration endpoints. --- .../web/mongooseim/mongoose_im_controller.ex | 41 +++++++++++++ lib/pleroma/web/router.ex | 5 ++ .../mongoose_im_controller_test.exs | 59 +++++++++++++++++++ 3 files changed, 105 insertions(+) create mode 100644 lib/pleroma/web/mongooseim/mongoose_im_controller.ex create mode 100644 test/web/mongooseim/mongoose_im_controller_test.exs diff --git a/lib/pleroma/web/mongooseim/mongoose_im_controller.ex b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex new file mode 100644 index 000000000..f8c634653 --- /dev/null +++ b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex @@ -0,0 +1,41 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MongooseIM.MongooseIMController do + use Pleroma.Web, :controller + alias Comeonin.Pbkdf2 + alias Pleroma.User + alias Pleroma.Repo + + def user_exists(conn, %{"user" => username}) do + with %User{} <- Repo.get_by(User, nickname: username, local: true) do + conn + |> json(true) + else + _ -> + conn + |> put_status(:not_found) + |> json(false) + end + end + + def check_password(conn, %{"user" => username, "pass" => password}) do + with %User{password_hash: password_hash} <- + Repo.get_by(User, nickname: username, local: true), + true <- Pbkdf2.checkpw(password, password_hash) do + conn + |> json(true) + else + false -> + conn + |> put_status(403) + |> json(false) + + _ -> + conn + |> put_status(:not_found) + |> json(false) + end + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 6a4e4a1d4..552778c92 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -704,6 +704,11 @@ defmodule Pleroma.Web.Router do end end + scope "/", Pleroma.Web.MongooseIM do + get("/user_exists", MongooseIMController, :user_exists) + get("/check_password", MongooseIMController, :check_password) + end + scope "/", Fallback do get("/registration/:token", RedirectController, :registration_page) get("/:maybe_nickname_or_id", RedirectController, :redirector_with_meta) diff --git a/test/web/mongooseim/mongoose_im_controller_test.exs b/test/web/mongooseim/mongoose_im_controller_test.exs new file mode 100644 index 000000000..eb83999bb --- /dev/null +++ b/test/web/mongooseim/mongoose_im_controller_test.exs @@ -0,0 +1,59 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MongooseIMController do + use Pleroma.Web.ConnCase + import Pleroma.Factory + + test "/user_exists", %{conn: conn} do + _user = insert(:user, nickname: "lain") + _remote_user = insert(:user, nickname: "alice", local: false) + + res = + conn + |> get(mongoose_im_path(conn, :user_exists), user: "lain") + |> json_response(200) + + assert res == true + + res = + conn + |> get(mongoose_im_path(conn, :user_exists), user: "alice") + |> json_response(404) + + assert res == false + + res = + conn + |> get(mongoose_im_path(conn, :user_exists), user: "bob") + |> json_response(404) + + assert res == false + end + + test "/check_password", %{conn: conn} do + user = insert(:user, password_hash: Comeonin.Pbkdf2.hashpwsalt("cool")) + + res = + conn + |> get(mongoose_im_path(conn, :check_password), user: user.nickname, pass: "cool") + |> json_response(200) + + assert res == true + + res = + conn + |> get(mongoose_im_path(conn, :check_password), user: user.nickname, pass: "uncool") + |> json_response(403) + + assert res == false + + res = + conn + |> get(mongoose_im_path(conn, :check_password), user: "nobody", pass: "cool") + |> json_response(404) + + assert res == false + end +end From 075eecec907b0a623a90eed44a0378a6812d8037 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 17 May 2019 18:32:30 +0200 Subject: [PATCH 08/56] Linting. --- lib/pleroma/web/mongooseim/mongoose_im_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mongooseim/mongoose_im_controller.ex b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex index f8c634653..489d5d3a5 100644 --- a/lib/pleroma/web/mongooseim/mongoose_im_controller.ex +++ b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex @@ -5,8 +5,8 @@ defmodule Pleroma.Web.MongooseIM.MongooseIMController do use Pleroma.Web, :controller alias Comeonin.Pbkdf2 - alias Pleroma.User alias Pleroma.Repo + alias Pleroma.User def user_exists(conn, %{"user" => username}) do with %User{} <- Repo.get_by(User, nickname: username, local: true) do From 78588dbd80580fbef53819dd87ee8fcc26cb09e9 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Fri, 17 May 2019 18:49:10 +0000 Subject: [PATCH 09/56] mrf: simple policy: mark all posts instead of posts with media as sensitive if they match media_nsfw --- lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 50426e920..9627c3400 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -48,10 +48,9 @@ defp check_media_nsfw( %{host: actor_host} = _actor_info, %{ "type" => "Create", - "object" => %{"attachment" => child_attachment} = child_object + "object" => child_object } = object - ) - when length(child_attachment) > 0 do + ) do object = if Enum.member?(Pleroma.Config.get([:mrf_simple, :media_nsfw]), actor_host) do tags = (child_object["tag"] || []) ++ ["nsfw"] From 0da1233e8e093ff7c69994f9e81d58611be60507 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Fri, 17 May 2019 18:49:43 +0000 Subject: [PATCH 10/56] rich media: suppress link previews if post is marked as sensitive --- lib/pleroma/web/rich_media/helpers.ex | 1 + test/web/rich_media/helpers_test.exs | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 0162a5be9..9bc8f2559 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -24,6 +24,7 @@ defp validate_page_url(_), do: :error def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do with true <- Pleroma.Config.get([:rich_media, :enabled]), %Object{} = object <- Object.normalize(activity), + false <- object.data["sensitive"] || false, {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]), :ok <- validate_page_url(page_url), {:ok, rich_media} <- Parser.parse(page_url) do diff --git a/test/web/rich_media/helpers_test.exs b/test/web/rich_media/helpers_test.exs index 60d93768f..6e23392ca 100644 --- a/test/web/rich_media/helpers_test.exs +++ b/test/web/rich_media/helpers_test.exs @@ -1,6 +1,7 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do use Pleroma.DataCase + alias Pleroma.Object alias Pleroma.Web.CommonAPI import Pleroma.Factory @@ -59,4 +60,24 @@ test "crawls valid, complete URLs" do Pleroma.Config.put([:rich_media, :enabled], false) end + + test "refuses to crawl URLs from posts marked sensitive" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "http://example.com/ogp", + "spoiler_text" => "." + }) + + %Object{} = object = Object.normalize(activity) + + assert object.data["sensitive"] + + Pleroma.Config.put([:rich_media, :enabled], true) + + assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) + + Pleroma.Config.put([:rich_media, :enabled], false) + end end From d3b8cd342ff6d1fe87bcfe6305424faf49666252 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Fri, 17 May 2019 19:03:19 +0000 Subject: [PATCH 11/56] http: request builder: send user-agent when making requests --- lib/pleroma/http/request_builder.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index 5f2cff2c0..522728da1 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -45,8 +45,9 @@ def url(request, u) do Add headers to the request """ @spec headers(map(), list(tuple)) :: map() - def headers(request, h) do - Map.put_new(request, :headers, h) + def headers(request, header_list) do + header_list = header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}] + Map.put_new(request, :headers, header_list) end @doc """ From 290f5b2cfe91dd2acba56f79ef137f15c68a0db0 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Fri, 17 May 2019 20:28:58 +0000 Subject: [PATCH 12/56] config: make sending the user agent configurable, disable sending the user agent in tests --- config/config.exs | 1 + config/test.exs | 2 ++ lib/pleroma/http/request_builder.ex | 8 +++++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index e82f08e07..9a10b0ff7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -192,6 +192,7 @@ # Configures http settings, upstream proxy etc. config :pleroma, :http, proxy_url: nil, + send_user_agent: true, adapter: [ ssl_options: [ # We don't support TLS v1.3 yet diff --git a/config/test.exs b/config/test.exs index e1785d10f..6100989c4 100644 --- a/config/test.exs +++ b/config/test.exs @@ -63,6 +63,8 @@ config :pleroma, :http_security, report_uri: "https://endpoint.com" +config :pleroma, :http, send_user_agent: false + rum_enabled = System.get_env("RUM_ENABLED") == "true" config :pleroma, :database, rum_enabled: rum_enabled IO.puts("RUM enabled: #{rum_enabled}") diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index 522728da1..e23457999 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -46,7 +46,13 @@ def url(request, u) do """ @spec headers(map(), list(tuple)) :: map() def headers(request, header_list) do - header_list = header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}] + header_list = + if Pleroma.Config.get([:http, :send_user_agent]) do + header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}] + else + header_list + end + Map.put_new(request, :headers, header_list) end From c234ce546a769747f436c19fee99bed2a7a58f3b Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Fri, 17 May 2019 20:31:39 +0000 Subject: [PATCH 13/56] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12c439135..5de9ae292 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,6 +102,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Correct `reblogged`, `favourited`, and `bookmarked` values in the reblog status JSON - Mastodon API: Exposing default scope of the user to anyone - Mastodon API: Make `irreversible` field default to `false` [`POST /api/v1/filters`] +- User-Agent is now sent correctly for all HTTP requests. ## Removed - Configuration: `config :pleroma, :fe` in favor of the more flexible `config :pleroma, :frontend_configurations` From dc081595385084fe6b382e4b38c17cb51cf0a611 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Fri, 17 May 2019 20:42:51 +0000 Subject: [PATCH 14/56] also suppress link previews from posts marked #nsfw --- lib/pleroma/web/common_api/common_api.ex | 4 +++- lib/pleroma/web/common_api/utils.ex | 4 +++- test/web/rich_media/helpers_test.exs | 21 ++++++++++++++++++++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index a599ffee5..5a312d673 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -157,6 +157,7 @@ def post(user, %{"status" => status} = data) do {to, cc} <- to_for_user_and_mentions(user, mentions, in_reply_to, visibility), context <- make_context(in_reply_to), cw <- data["spoiler_text"] || "", + sensitive <- data["sensitive"] || Enum.member?(tags, {"#nsfw", "nsfw"}), full_payload <- String.trim(status <> cw), length when length in 1..limit <- String.length(full_payload), object <- @@ -169,7 +170,8 @@ def post(user, %{"status" => status} = data) do in_reply_to, tags, cw, - cc + cc, + sensitive ), object <- Map.put( diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index bee2fd159..8d6160976 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -223,7 +223,8 @@ def make_note_data( in_reply_to, tags, cw \\ nil, - cc \\ [] + cc \\ [], + sensitive \\ false ) do object = %{ "type" => "Note", @@ -231,6 +232,7 @@ def make_note_data( "cc" => cc, "content" => content_html, "summary" => cw, + "sensitive" => sensitive, "context" => context, "attachment" => attachments, "actor" => actor, diff --git a/test/web/rich_media/helpers_test.exs b/test/web/rich_media/helpers_test.exs index 6e23392ca..53b0596f5 100644 --- a/test/web/rich_media/helpers_test.exs +++ b/test/web/rich_media/helpers_test.exs @@ -67,7 +67,26 @@ test "refuses to crawl URLs from posts marked sensitive" do {:ok, activity} = CommonAPI.post(user, %{ "status" => "http://example.com/ogp", - "spoiler_text" => "." + "sensitive" => true + }) + + %Object{} = object = Object.normalize(activity) + + assert object.data["sensitive"] + + Pleroma.Config.put([:rich_media, :enabled], true) + + assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) + + Pleroma.Config.put([:rich_media, :enabled], false) + end + + test "refuses to crawl URLs from posts tagged NSFW" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "http://example.com/ogp #nsfw" }) %Object{} = object = Object.normalize(activity) From c4a55e167afcfd25cf4c1efb46886f2defe72c22 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Fri, 17 May 2019 20:43:31 +0000 Subject: [PATCH 15/56] add Changelog entry --- CHANGELOG.md | 1 + lib/pleroma/web/common_api/utils.ex | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12c439135..a73bf47d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Deps: Updated Ecto to 3.0.7 - Don't ship finmoji by default, they can be installed as an emoji pack - Hide deactivated users and their statuses +- Posts which are marked sensitive or tagged nsfw no longer have link previews. ### Fixed - Added an FTS index on objects. Running `vacuum analyze` and setting a larger `work_mem` is recommended. diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 8d6160976..d93c0d46e 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -232,7 +232,7 @@ def make_note_data( "cc" => cc, "content" => content_html, "summary" => cw, - "sensitive" => sensitive, + "sensitive" => !Enum.member?(["false", "False", "0", false], sensitive), "context" => context, "attachment" => attachments, "actor" => actor, From 2375e9a95ba9042958ff7e8f75830df4ab53fed2 Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Mon, 20 May 2019 06:02:50 +0800 Subject: [PATCH 16/56] Add report filtering to MRF.SimplePolicy --- config/config.exs | 1 + .../web/activity_pub/mrf/simple_policy.ex | 13 ++++++++- .../activity_pub/mrf/simple_policy_test.exs | 28 +++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 9a10b0ff7..bab47a8a2 100644 --- a/config/config.exs +++ b/config/config.exs @@ -298,6 +298,7 @@ media_removal: [], media_nsfw: [], federated_timeline_removal: [], + report_removal: [], reject: [], accept: [] diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 9627c3400..7190652d2 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -94,6 +94,16 @@ defp check_ftl_removal(%{host: actor_host} = _actor_info, object) do {:ok, object} end + defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = object) do + if actor_host in Pleroma.Config.get([:mrf_simple, :report_removal]) do + {:reject, nil} + else + {:ok, object} + end + end + + defp check_report_removal(_actor_info, object), do: {:ok, object} + @impl true def filter(object) do actor_info = URI.parse(object["actor"]) @@ -102,7 +112,8 @@ def filter(object) do {:ok, object} <- check_reject(actor_info, object), {:ok, object} <- check_media_removal(actor_info, object), {:ok, object} <- check_media_nsfw(actor_info, object), - {:ok, object} <- check_ftl_removal(actor_info, object) do + {:ok, object} <- check_ftl_removal(actor_info, object), + {:ok, object} <- check_report_removal(actor_info, object) do {:ok, object} else _e -> {:reject, nil} diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs index 1e0511975..74af7dcde 100644 --- a/test/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/web/activity_pub/mrf/simple_policy_test.exs @@ -15,6 +15,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do media_removal: [], media_nsfw: [], federated_timeline_removal: [], + report_removal: [], reject: [], accept: [] ) @@ -85,6 +86,33 @@ defp build_media_message do } end + describe "when :report_removal" do + test "is empty" do + Config.put([:mrf_simple, :report_removal], []) + report_message = build_report_message() + local_message = build_local_message() + + assert SimplePolicy.filter(report_message) == {:ok, report_message} + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end + + test "has a matching host" do + Config.put([:mrf_simple, :report_removal], ["remote.instance"]) + report_message = build_report_message() + local_message = build_local_message() + + assert SimplePolicy.filter(report_message) == {:reject, nil} + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end + end + + defp build_report_message do + %{ + "actor" => "https://remote.instance/users/bob", + "type" => "Flag" + } + end + describe "when :federated_timeline_removal" do test "is empty" do Config.put([:mrf_simple, :federated_timeline_removal], []) From 54e9cb5c2db580bc12441f3651fa87a7b976137d Mon Sep 17 00:00:00 2001 From: Sadposter Date: Mon, 20 May 2019 12:39:23 +0100 Subject: [PATCH 17/56] Add API endpoints for a custom user mascot --- docs/api/pleroma_api.md | 39 +++++++++++++++++++ lib/pleroma/user/info.ex | 21 ++++++++++ .../mastodon_api/mastodon_api_controller.ex | 36 ++++++++++++++++- lib/pleroma/web/router.ex | 3 ++ 4 files changed, 98 insertions(+), 1 deletion(-) diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md index dd0b6ca73..4d99a2d2b 100644 --- a/docs/api/pleroma_api.md +++ b/docs/api/pleroma_api.md @@ -252,6 +252,45 @@ See [Admin-API](Admin-API.md) ] ``` +## `/api/v1/pleroma/mascot` +### Gets user mascot image +* Method `GET` +* Authentication: required + +* Response: JSON. Returns a mastodon media attachment entity. +* Example response: +```json +{ + "id": "abcdefg", + "url": "https://pleroma.example.org/media/abcdefg.png", + "type": "image", + "pleroma": { + "mime_type": "image/png" + } +} +``` + +### Updates user mascot image +* Method `PUT` +* Authentication: required +* Params: + * `image`: Multipart image +* Response: JSON. Returns a mastodon media attachment entity + when successful, otherwise returns HTTP 415 `{"error": "error_msg"}` +* Example response: +```json +{ + "id": "abcdefg", + "url": "https://pleroma.example.org/media/abcdefg.png", + "type": "image", + "pleroma": { + "mime_type": "image/png" + } +} +``` +* Note: Behaves exactly the same as `POST /api/v1/upload`. + Can only accept images - any attempt to upload non-image files will be met with `HTTP 415 Unsupported Media Type`. + ## `/api/pleroma/notification_settings` ### Updates user notification settings * Method `PUT` diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 5f0cefc00..ffcd06e3e 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -43,6 +43,19 @@ defmodule Pleroma.User.Info do field(:hide_favorites, :boolean, default: true) field(:pinned_activities, {:array, :string}, default: []) field(:flavour, :string, default: nil) + + field(:mascot, :map, + default: %{ + id: "pleromatan", + url: "/images/pleroma-fox-tan-smol.png", + type: "image", + preview_url: "/images/pleroma-fox-tan-smol.png", + pleroma: %{ + mime_type: "image/png" + } + } + ) + field(:emoji, {:array, :map}, default: []) field(:notification_settings, :map, @@ -248,6 +261,14 @@ def mastodon_flavour_update(info, flavour) do |> validate_required([:flavour]) end + def mascot_update(info, url) do + params = %{mascot: url} + + info + |> cast(params, [:mascot]) + |> validate_required([:mascot]) + end + def set_source_data(info, source_data) do params = %{source_data: source_data} diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 1051861ff..67f363859 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -707,6 +707,40 @@ def upload(%{assigns: %{user: user}} = conn, %{"file" => file} = data) do end end + def set_mascot(%{assigns: %{user: user}} = conn, %{"file" => file}) do + with {:ok, object} <- ActivityPub.upload(file, actor: User.ap_id(user)), + %{} = attachment_data <- Map.put(object.data, "id", object.id), + %{type: type} = rendered <- + StatusView.render("attachment.json", %{attachment: attachment_data}) do + # Reject if not an image + if type == "image" do + # Sure! + # Save to the user's info + info_changeset = User.Info.mascot_update(user.info, rendered) + + user_changeset = + user + |> Ecto.Changeset.change() + |> Ecto.Changeset.put_embed(:info, info_changeset) + + {:ok, _user} = User.update_and_set_cache(user_changeset) + + conn + |> json(rendered) + else + conn + |> send_resp(415, Jason.encode!(%{"error" => "mascots can only be images"})) + end + end + end + + def get_mascot(%{assigns: %{user: user}} = conn, _params) do + %{info: %{mascot: mascot}} = user + + conn + |> json(mascot) + end + def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id), %Object{data: %{"likes" => likes}} <- Object.normalize(object) do @@ -1329,7 +1363,7 @@ def index(%{assigns: %{user: user}} = conn, _params) do display_sensitive_media: false, reduce_motion: false, max_toot_chars: limit, - mascot: "/images/pleroma-fox-tan-smol.png" + mascot: Map.get(user.info.mascot, "url", "/images/pleroma-fox-tan-smol.png") }, rights: %{ delete_others_notice: present?(user.info.is_moderator), diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 6a4e4a1d4..4c29b24eb 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -352,6 +352,9 @@ defmodule Pleroma.Web.Router do post("/pleroma/flavour/:flavour", MastodonAPIController, :set_flavour) + get("/pleroma/mascot", MastodonAPIController, :get_mascot) + put("/pleroma/mascot", MastodonAPIController, :set_mascot) + post("/reports", MastodonAPIController, :reports) end From e81f0fc6d45249dd70656c58af926c21c70c482f Mon Sep 17 00:00:00 2001 From: Sadposter Date: Mon, 20 May 2019 12:58:06 +0100 Subject: [PATCH 18/56] Add mascot get/set tests --- .../mastodon_api/mastodon_api_controller.ex | 1 + test/fixtures/sound.mp3 | Bin 0 -> 521 bytes .../mastodon_api_controller_test.exs | 65 ++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 test/fixtures/sound.mp3 diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 67f363859..d7f095a1f 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -729,6 +729,7 @@ def set_mascot(%{assigns: %{user: user}} = conn, %{"file" => file}) do |> json(rendered) else conn + |> put_resp_content_type("application/json") |> send_resp(415, Jason.encode!(%{"error" => "mascots can only be images"})) end end diff --git a/test/fixtures/sound.mp3 b/test/fixtures/sound.mp3 new file mode 100644 index 0000000000000000000000000000000000000000..9f0f661a3bc34b6366304e9fd1d4fe8513411083 GIT binary patch literal 521 zcmezWd%_V0bP$o5mkt!;2VzDB1}091|Fj1{y8?V1eO-<93=IrecEX$_s-VK;;K;!E q0OXGES_-dW5+jBF|6AY)1M>j}#w9=>D=;vaG%zr*zym6jY5)LcXO3V1 literal 0 HcmV?d00001 diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index cbff141c8..87e1c105d 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1455,6 +1455,71 @@ test "media upload", %{conn: conn} do assert object.data["actor"] == User.ap_id(user) end + test "mascot upload", %{conn: conn} do + user = insert(:user) + + non_image_file = %Plug.Upload{ + content_type: "audio/mpeg", + path: Path.absname("test/fixtures/sound.mp3"), + filename: "sound.mp3" + } + + conn = + conn + |> assign(:user, user) + |> put("/api/v1/pleroma/mascot", %{"file" => non_image_file}) + + assert json_response(conn, 415) + + file = %Plug.Upload{ + content_type: "image/jpg", + path: Path.absname("test/fixtures/image.jpg"), + filename: "an_image.jpg" + } + + conn = + build_conn() + |> assign(:user, user) + |> put("/api/v1/pleroma/mascot", %{"file" => file}) + + assert %{"id" => _, "type" => image} = json_response(conn, 200) + end + + test "mascot retrieving", %{conn: conn} do + user = insert(:user) + # When user hasn't set a mascot, we should just get pleroma tan back + conn = + conn + |> assign(:user, user) + |> get("/api/v1/pleroma/mascot") + + assert %{"url" => url} = json_response(conn, 200) + assert url =~ "pleroma-fox-tan-smol" + + # When a user sets their mascot, we should get that back + file = %Plug.Upload{ + content_type: "image/jpg", + path: Path.absname("test/fixtures/image.jpg"), + filename: "an_image.jpg" + } + + conn = + build_conn() + |> assign(:user, user) + |> put("/api/v1/pleroma/mascot", %{"file" => file}) + assert json_response(conn, 200) + + user = User.get_cached_by_id(user.id) + + conn = + build_conn() + |> assign(:user, user) + |> get("/api/v1/pleroma/mascot") + + assert %{"url" => url, "type" => "image"} = json_response(conn, 200) + assert url =~ "an_image" + end + test "hashtag timeline", %{conn: conn} do following = insert(:user) From dc916ba15f0fd77afa015084849f082065ed6f74 Mon Sep 17 00:00:00 2001 From: Sadposter Date: Mon, 20 May 2019 12:58:17 +0100 Subject: [PATCH 19/56] Format mascot tests --- test/web/mastodon_api/mastodon_api_controller_test.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 87e1c105d..1d9f5a816 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1503,10 +1503,11 @@ test "mascot retrieving", %{conn: conn} do filename: "an_image.jpg" } - conn = + conn = build_conn() |> assign(:user, user) |> put("/api/v1/pleroma/mascot", %{"file" => file}) + assert json_response(conn, 200) user = User.get_cached_by_id(user.id) From 3d0d9e7a5680bcb1b79e5f4aab0e8c514fc9e5ff Mon Sep 17 00:00:00 2001 From: Sadposter Date: Mon, 20 May 2019 13:10:04 +0100 Subject: [PATCH 20/56] Use string map for default mascot --- lib/pleroma/user/info.ex | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index ffcd06e3e..e76d04d7f 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -46,12 +46,12 @@ defmodule Pleroma.User.Info do field(:mascot, :map, default: %{ - id: "pleromatan", - url: "/images/pleroma-fox-tan-smol.png", - type: "image", - preview_url: "/images/pleroma-fox-tan-smol.png", - pleroma: %{ - mime_type: "image/png" + "id" => "pleromatan", + "url" => "/images/pleroma-fox-tan-smol.png", + "type" => "image", + "preview_url" => "/images/pleroma-fox-tan-smol.png", + "pleroma" => %{ + "mime_type" => "image/png" } } ) From d835810610e5e7660a56d305bc8509e38c642942 Mon Sep 17 00:00:00 2001 From: Sadposter Date: Mon, 20 May 2019 14:19:42 +0100 Subject: [PATCH 21/56] Add changelog entry for mascot config --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87101db30..07e6bce31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Configuration: `report_uri` option - Pleroma API: User subscriptions - Pleroma API: Healthcheck endpoint +- Pleroma API: `/api/v1/pleroma/mascot` per-user frontend mascot configuration endpoints - Admin API: Endpoints for listing/revoking invite tokens - Admin API: Endpoints for making users follow/unfollow each other - Admin API: added filters (role, tags, email, name) for users endpoint From daeae8e2e7c506b72c66dea6ac790408f948ec16 Mon Sep 17 00:00:00 2001 From: Sadposter Date: Mon, 20 May 2019 16:12:55 +0100 Subject: [PATCH 22/56] Move default mascot configuration to `config/` --- config/config.exs | 13 ++++++++++++ docs/config.md | 10 ++++++++++ lib/pleroma/user.ex | 20 +++++++++++++++++++ lib/pleroma/user/info.ex | 14 +------------ .../mastodon_api/mastodon_api_controller.ex | 4 ++-- 5 files changed, 46 insertions(+), 15 deletions(-) diff --git a/config/config.exs b/config/config.exs index bab47a8a2..72908266d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -276,6 +276,19 @@ showInstanceSpecificPanel: true } +config :pleroma, :assets, + mascots: [ + pleroma_fox_tan: %{ + url: "/images/pleroma-fox-tan-smol.png", + mime_type: "image/png" + }, + pleroma_fox_tan_shy: %{ + url: "/images/pleroma-fox-tan-shy.png", + mime_type: "image/png" + } + ], + default_mascot: :pleroma_fox_tan + config :pleroma, :activitypub, accept_blocks: true, unfollow_blocked: true, diff --git a/docs/config.md b/docs/config.md index 450d73fda..197326bbd 100644 --- a/docs/config.md +++ b/docs/config.md @@ -203,6 +203,16 @@ This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:i * `hide_post_stats`: Hide notices statistics(repeats, favorites, …) * `hide_user_stats`: Hide profile statistics(posts, posts per day, followers, followings, …) +## :assets + +This section configures assets to be used with various frontends. Currently the only option +relates to mascots on the mastodon frontend + +* `mascots`: KeywordList of mascots, each element __MUST__ contain both a `url` and a + `mime_type` key. +* `default_mascot`: An element from `mascots` - This will be used as the default mascot + on MastoFE (default: `:pleroma_fox_tan`) + ## :mrf_simple * `media_removal`: List of instances to remove medias from * `media_nsfw`: List of instances to put medias as NSFW(sensitive) from diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 28da310ee..05fe58f7c 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1402,4 +1402,24 @@ def toggle_confirmation(%User{} = user) do |> put_embed(:info, info_changeset) |> update_and_set_cache() end + + def get_mascot(%{info: %{mascot: %{} = mascot}}) when not is_nil(mascot) do + mascot + end + + def get_mascot(%{info: %{mascot: mascot}}) when is_nil(mascot) do + # use instance-default + config = Pleroma.Config.get([:assets, :mascots]) + default_mascot = Pleroma.Config.get([:assets, :default_mascot]) + mascot = Keyword.get(config, default_mascot) + + %{ + "id" => "default-mascot", + "url" => mascot[:url], + "preview_url" => mascot[:url], + "pleroma" => %{ + "mime_type" => mascot[:mime_type] + } + } + end end diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index e76d04d7f..6397e2737 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -43,19 +43,7 @@ defmodule Pleroma.User.Info do field(:hide_favorites, :boolean, default: true) field(:pinned_activities, {:array, :string}, default: []) field(:flavour, :string, default: nil) - - field(:mascot, :map, - default: %{ - "id" => "pleromatan", - "url" => "/images/pleroma-fox-tan-smol.png", - "type" => "image", - "preview_url" => "/images/pleroma-fox-tan-smol.png", - "pleroma" => %{ - "mime_type" => "image/png" - } - } - ) - + field(:mascot, :map, default: nil) field(:emoji, {:array, :map}, default: []) field(:notification_settings, :map, diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index d7f095a1f..1ec0f30a1 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -736,7 +736,7 @@ def set_mascot(%{assigns: %{user: user}} = conn, %{"file" => file}) do end def get_mascot(%{assigns: %{user: user}} = conn, _params) do - %{info: %{mascot: mascot}} = user + mascot = User.get_mascot(user) conn |> json(mascot) @@ -1364,7 +1364,7 @@ def index(%{assigns: %{user: user}} = conn, _params) do display_sensitive_media: false, reduce_motion: false, max_toot_chars: limit, - mascot: Map.get(user.info.mascot, "url", "/images/pleroma-fox-tan-smol.png") + mascot: User.get_mascot(user)["url"] }, rights: %{ delete_others_notice: present?(user.info.is_moderator), From eb02edcad9cb0d65fc216408960aec63713e5d2b Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Tue, 21 May 2019 00:35:46 +0800 Subject: [PATCH 23/56] Add virtual :thread_muted? field that may be set when fetching activities --- lib/pleroma/activity.ex | 12 +++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 8 ++++++ .../web/mastodon_api/views/status_view.ex | 8 +++++- .../web/twitter_api/views/activity_view.ex | 8 +++++- test/activity_test.exs | 26 +++++++++++++++++++ test/user_test.exs | 2 +- 6 files changed, 61 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 4e54b15ba..99589590c 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Activity do alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.ThreadMute alias Pleroma.User import Ecto.Changeset @@ -37,6 +38,7 @@ defmodule Pleroma.Activity do field(:local, :boolean, default: true) field(:actor, :string) field(:recipients, {:array, :string}, default: []) + field(:thread_muted?, :boolean, virtual: true) # This is a fake relation, do not use outside of with_preloaded_bookmark/get_bookmark has_one(:bookmark, Bookmark) has_many(:notifications, Notification, on_delete: :delete_all) @@ -90,6 +92,16 @@ def with_preloaded_bookmark(query, %User{} = user) do def with_preloaded_bookmark(query, _), do: query + def with_set_thread_muted_field(query, %User{} = user) do + from([a] in query, + left_join: tm in ThreadMute, + on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data), + select: %Activity{a | thread_muted?: not is_nil(tm.id)} + ) + end + + def with_set_thread_muted_field(query, _), do: query + def get_by_ap_id(ap_id) do Repo.one( from( diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 5c3156978..3d9679ec0 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -834,6 +834,13 @@ defp maybe_preload_bookmarks(query, opts) do |> Activity.with_preloaded_bookmark(opts["user"]) end + defp maybe_set_thread_muted_field(query, %{"skip_preload" => true}), do: query + + defp maybe_set_thread_muted_field(query, opts) do + query + |> Activity.with_set_thread_muted_field(opts["user"]) + end + defp maybe_order(query, %{order: :desc}) do query |> order_by(desc: :id) @@ -852,6 +859,7 @@ def fetch_activities_query(recipients, opts \\ %{}) do base_query |> maybe_preload_objects(opts) |> maybe_preload_bookmarks(opts) + |> maybe_set_thread_muted_field(opts) |> maybe_order(opts) |> restrict_recipients(recipients, opts["user"]) |> restrict_tag(opts) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index c93d915e5..e55f9b96e 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -157,6 +157,12 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity bookmarked = Activity.get_bookmark(activity, opts[:for]) != nil + thread_muted? = + case activity.thread_muted? do + thread_muted? when is_boolean(thread_muted?) -> thread_muted? + nil -> CommonAPI.thread_muted?(user, activity) + end + attachment_data = object.data["attachment"] || [] attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment) @@ -228,7 +234,7 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity reblogged: reblogged?(activity, opts[:for]), favourited: present?(favorited), bookmarked: present?(bookmarked), - muted: CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user), + muted: thread_muted? || User.mutes?(opts[:for], user), pinned: pinned?(activity, user), sensitive: sensitive, spoiler_text: summary_html, diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex index 44bcafe0e..e84af84dc 100644 --- a/lib/pleroma/web/twitter_api/views/activity_view.ex +++ b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -284,6 +284,12 @@ def render( Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) ) + thread_muted? = + case activity.thread_muted? do + thread_muted? when is_boolean(thread_muted?) -> thread_muted? + nil -> CommonAPI.thread_muted?(user, activity) + end + %{ "id" => activity.id, "uri" => object.data["id"], @@ -314,7 +320,7 @@ def render( "summary" => summary, "summary_html" => summary |> Formatter.emojify(object.data["emoji"]), "card" => card, - "muted" => CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user) + "muted" => thread_muted? || User.mutes?(opts[:for], user) } end diff --git a/test/activity_test.exs b/test/activity_test.exs index 7e91d534b..15c95502a 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.ActivityTest do use Pleroma.DataCase alias Pleroma.Activity alias Pleroma.Bookmark + alias Pleroma.ThreadMute import Pleroma.Factory test "returns an activity by it's AP id" do @@ -47,6 +48,31 @@ test "preloading a bookmark" do assert queried_activity.bookmark == bookmark3 end + test "setting thread_muted?" do + activity = insert(:note_activity) + user = insert(:user) + annoyed_user = insert(:user) + {:ok, _} = ThreadMute.add_mute(annoyed_user.id, activity.data["context"]) + + activity_with_unset_thread_muted_field = + Ecto.Query.from(Activity) + |> Repo.one() + + activity_for_user = + Ecto.Query.from(Activity) + |> Activity.with_set_thread_muted_field(user) + |> Repo.one() + + activity_for_annoyed_user = + Ecto.Query.from(Activity) + |> Activity.with_set_thread_muted_field(annoyed_user) + |> Repo.one() + + assert activity_with_unset_thread_muted_field.thread_muted? == nil + assert activity_for_user.thread_muted? == false + assert activity_for_annoyed_user.thread_muted? == true + end + describe "getting a bookmark" do test "when association is loaded" do user = insert(:user) diff --git a/test/user_test.exs b/test/user_test.exs index 10e463ff8..cb6afbe07 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -902,7 +902,7 @@ test "hide a user's statuses from timelines and notifications" do assert [activity] == ActivityPub.fetch_public_activities(%{}) |> Repo.preload(:bookmark) - assert [activity] == + assert [%{activity | thread_muted?: CommonAPI.thread_muted?(user2, activity)}] == ActivityPub.fetch_activities([user2.ap_id | user2.following], %{"user" => user2}) {:ok, _user} = User.deactivate(user) From 5aa107d9126c484f0ae06eddec6995b535623b32 Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Tue, 21 May 2019 00:59:12 +0800 Subject: [PATCH 24/56] Document MRF.Simple :report_removal --- docs/config/mrf.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/config/mrf.md b/docs/config/mrf.md index 2cc16cef0..45be18fc5 100644 --- a/docs/config/mrf.md +++ b/docs/config/mrf.md @@ -5,11 +5,12 @@ Possible uses include: * marking incoming messages with media from a given account or instance as sensitive * rejecting messages from a specific instance +* rejecting reports (flags) from a specific instance * removing/unlisting messages from the public timelines * removing media from messages * sending only public messages to a specific instance -The MRF provides user-configurable policies. The default policy is `NoOpPolicy`, which disables the MRF functionality. Pleroma also includes an easy to use policy called `SimplePolicy` which maps messages matching certain pre-defined criterion to actions built into the policy module. +The MRF provides user-configurable policies. The default policy is `NoOpPolicy`, which disables the MRF functionality. Pleroma also includes an easy to use policy called `SimplePolicy` which maps messages matching certain pre-defined criterion to actions built into the policy module. It is possible to use multiple, active MRF policies at the same time. ## Quarantine Instances @@ -41,12 +42,13 @@ Once `SimplePolicy` is enabled, you can configure various groups in the `:mrf_si * `media_nsfw`: Servers in this group will have the #nsfw tag and sensitive setting injected into incoming messages which contain media. * `reject`: Servers in this group will have their messages rejected. * `federated_timeline_removal`: Servers in this group will have their messages unlisted from the public timelines by flipping the `to` and `cc` fields. +* `report_removal`: Servers in this group will have their reports (flags) rejected. Servers should be configured as lists. ### Example -This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com` and remove messages from `spam.university` from the federated timeline: +This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com`, remove messages from `spam.university` from the federated timeline and block reports (flags) from `whiny.whiner`: ``` config :pleroma, :instance, @@ -56,7 +58,8 @@ config :pleroma, :mrf_simple, media_removal: ["illegalporn.biz"], media_nsfw: ["porn.biz", "porn.business"], reject: ["spam.com"], - federated_timeline_removal: ["spam.university"] + federated_timeline_removal: ["spam.university"], + report_removal: ["whiny.whiner"] ``` From 75c7bb9289066bfaebe7d96aaa474d34ec4d5a5f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 20 May 2019 17:18:59 -0500 Subject: [PATCH 25/56] Additional reserved usernames --- config/config.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/config.exs b/config/config.exs index 61e2648a9..387c1a5a7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -369,6 +369,7 @@ "activities", "api", "auth", + "check_password", "dev", "friend-requests", "inbox", @@ -389,6 +390,7 @@ "status", "tag", "user-search", + "user_exists", "users", "web" ] From f96e9b28bb5ee241a3f0ca6a622b925ca560c141 Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Tue, 21 May 2019 07:30:18 +0800 Subject: [PATCH 26/56] Fix prometheus-ecto error when not configured --- lib/pleroma/application.ex | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index eeb415084..dab45a0b2 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -131,19 +131,22 @@ def start(_type, _args) do defp setup_instrumenters do require Prometheus.Registry - :ok = - :telemetry.attach( - "prometheus-ecto", - [:pleroma, :repo, :query], - &Pleroma.Repo.Instrumenter.handle_event/4, - %{} - ) + if Application.get_env(:prometheus, Pleroma.Repo.Instrumenter) do + :ok = + :telemetry.attach( + "prometheus-ecto", + [:pleroma, :repo, :query], + &Pleroma.Repo.Instrumenter.handle_event/4, + %{} + ) + + Pleroma.Repo.Instrumenter.setup() + end Prometheus.Registry.register_collector(:prometheus_process_collector) Pleroma.Web.Endpoint.MetricsExporter.setup() Pleroma.Web.Endpoint.PipelineInstrumenter.setup() Pleroma.Web.Endpoint.Instrumenter.setup() - Pleroma.Repo.Instrumenter.setup() end def enabled_hackney_pools do From c972d0bb14ee5a65f053b8c9629d93fc9b94ca78 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 21 May 2019 04:58:26 +0000 Subject: [PATCH 27/56] http: bump connection timeout to 10 seconds --- CHANGELOG.md | 1 + lib/pleroma/http/connection.ex | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07e6bce31..256df91b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Don't ship finmoji by default, they can be installed as an emoji pack - Hide deactivated users and their statuses - Posts which are marked sensitive or tagged nsfw no longer have link previews. +- HTTP connection timeout is now set to 10 seconds. ### Fixed - Added an FTS index on objects. Running `vacuum analyze` and setting a larger `work_mem` is recommended. diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index c0173465a..558005c19 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -8,7 +8,7 @@ defmodule Pleroma.HTTP.Connection do """ @hackney_options [ - connect_timeout: 2_000, + connect_timeout: 10_000, recv_timeout: 20_000, follow_redirect: true, pool: :federation From d378b342ba0d7f54be3b47f031c602a578191dfd Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 21 May 2019 18:57:36 +0200 Subject: [PATCH 28/56] MongooseIM: Add documentation. --- CHANGELOG.md | 1 + docs/config/howto_mongooseim.md | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 docs/config/howto_mongooseim.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 12c439135..b50ca895c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [unreleased] ### Added +- [MongooseIM](https://github.com/esl/MongooseIM) http authentication support. - LDAP authentication - External OAuth provider authentication - A [job queue](https://git.pleroma.social/pleroma/pleroma_job_queue) for federation, emails, web push, etc. diff --git a/docs/config/howto_mongooseim.md b/docs/config/howto_mongooseim.md new file mode 100644 index 000000000..a33e590a1 --- /dev/null +++ b/docs/config/howto_mongooseim.md @@ -0,0 +1,10 @@ +# Configuring MongooseIM (XMPP Server) to use Pleroma for authentication + +If you want to give your Pleroma users an XMPP (chat) account, you can configure [MongooseIM](https://github.com/esl/MongooseIM) to use your Pleroma server for user authentication, automatically giving every local user an XMPP account. + +In general, you just have to follow the configuration described at [https://mongooseim.readthedocs.io/en/latest/authentication-backends/HTTP-authentication-module/](https://mongooseim.readthedocs.io/en/latest/authentication-backends/HTTP-authentication-module/) and do these changes to your mongooseim.cfg. + +1. Set the auth_method to `{auth_method, http}`. +2. Add the http auth pool like this: `{http, global, auth, [{workers, 50}], [{server, "https://yourpleromainstance.com"}]}` + +Restart your MongooseIM server, your users should now be able to connect with their Pleroma credentials. From c2b0b82e6a6d40f945feacc001ad984a17e23336 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 21 May 2019 00:41:40 +0000 Subject: [PATCH 29/56] object: add Object.prune() --- lib/pleroma/object.ex | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 740d687a3..cc6fc9c5d 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -130,6 +130,13 @@ def delete(%Object{data: %{"id" => id}} = object) do end end + def prune(%Object{data: %{"id" => id}} = object) do + with {:ok, object} <- Repo.delete(object), + {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do + {:ok, object} + end + end + def set_cache(%Object{data: %{"id" => ap_id}} = object) do Cachex.put(:object_cache, "object:#{ap_id}", object) {:ok, object} From 73df9d690d5c1a9c11f0f04b8d877c0677022591 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 21 May 2019 00:41:58 +0000 Subject: [PATCH 30/56] object: fetcher: add support for reinjecting pruned objects --- lib/pleroma/object/fetcher.ex | 22 ++++++++++++++++++++-- test/object/fetcher_test.exs | 19 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 8d4bcc95e..bb9388d4f 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -8,6 +8,19 @@ defmodule Pleroma.Object.Fetcher do @httpoison Application.get_env(:pleroma, :httpoison) + defp reinject_object(data) do + Logger.debug("Reinjecting object #{data["id"]}") + + with data <- Transmogrifier.fix_object(data), + {:ok, object} <- Object.create(data) do + {:ok, object} + else + e -> + Logger.error("Error while processing object: #{inspect(e)}") + {:error, e} + end + end + # TODO: # This will create a Create activity, which we need internally at the moment. def fetch_object_from_id(id) do @@ -26,12 +39,17 @@ def fetch_object_from_id(id) do "object" => data }, :ok <- Containment.contain_origin(id, params), - {:ok, activity} <- Transmogrifier.handle_incoming(params) do - {:ok, Object.normalize(activity, false)} + {:ok, activity} <- Transmogrifier.handle_incoming(params), + {:object, _data, %Object{} = object} <- + {:object, data, Object.normalize(activity, false)} do + {:ok, object} else {:error, {:reject, nil}} -> {:reject, nil} + {:object, data, nil} -> + reinject_object(data) + object = %Object{} -> {:ok, object} diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs index 72f616782..d604fd5f5 100644 --- a/test/object/fetcher_test.exs +++ b/test/object/fetcher_test.exs @@ -87,4 +87,23 @@ test "all objects with fake directions are rejected by the object fetcher" do ) end end + + describe "pruning" do + test "it can refetch pruned objects" do + object_id = "http://mastodon.example.org/@admin/99541947525187367" + + {:ok, object} = Fetcher.fetch_object_from_id(object_id) + + assert object + + {:ok, _object} = Object.prune(object) + + refute Object.get_by_ap_id(object_id) + + {:ok, %Object{} = object_two} = Fetcher.fetch_object_from_id(object_id) + + assert object.data["id"] == object_two.data["id"] + assert object.id != object_two.id + end + end end From 16b260fb19cca02463766c2e36a41bfcc823af9b Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 21 May 2019 01:21:28 +0000 Subject: [PATCH 31/56] add mix task to prune the object database using a configured retention period --- config/config.exs | 3 ++- docs/config.md | 1 + lib/mix/tasks/pleroma/database.ex | 40 +++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 72908266d..466a6e9b7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -239,7 +239,8 @@ welcome_message: nil, max_report_comment_size: 1000, safe_dm_mentions: false, - healthcheck: false + healthcheck: false, + remote_post_retention_days: 90 config :pleroma, :app_account_creation, enabled: true, max_requests: 25, interval: 1800 diff --git a/docs/config.md b/docs/config.md index 197326bbd..a050068f4 100644 --- a/docs/config.md +++ b/docs/config.md @@ -104,6 +104,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `max_report_comment_size`: The maximum size of the report comment (Default: `1000`) * `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). (Default: `false`) * `healthcheck`: if set to true, system data will be shown on ``/api/pleroma/healthcheck``. +* `remote_post_retention_days`: the default amount of days to retain remote posts when pruning the database ## :app_account_creation REST API for creating an account settings diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index f650b447d..fdb216037 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -23,6 +23,10 @@ defmodule Mix.Tasks.Pleroma.Database do Options: - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references + ## Prune old objects from the database + + mix pleroma.database prune_objects + ## Create a conversation for all existing DMs. Can be safely re-run. mix pleroma.database bump_all_conversations @@ -72,4 +76,40 @@ def run(["update_users_following_followers_counts"]) do Enum.each(users, &User.remove_duplicated_following/1) Enum.each(users, &User.update_follower_count/1) end + + def run(["prune_objects" | args]) do + {options, [], []} = + OptionParser.parse( + args, + strict: [ + vacuum: :boolean + ] + ) + + Common.start_pleroma() + + deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + + Logger.info("Pruning objects older than #{deadline} days") + + time_deadline = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(-(deadline * 86_400)) + + Repo.query!( + "DELETE FROM objects WHERE inserted_at < $1 AND split_part(data->>'actor', '/', 3) != $2", + [time_deadline, Pleroma.Web.Endpoint.host()], + timeout: :infinity + ) + + if Keyword.get(options, :vacuum) do + Logger.info("Runnning VACUUM FULL") + + Repo.query!( + "vacuum full;", + [], + timeout: :infinity + ) + end + end end From f446f94290a6cdae531859c2efa55f55dbaeb7e8 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 21 May 2019 01:22:27 +0000 Subject: [PATCH 32/56] add changelog entry for object pruning --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 256df91b7..a21c4bff2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Metadata: RelMe provider - OAuth: added support for refresh tokens - Emoji packs and emoji pack manager +- Object pruning (`mix pleroma.database prune_objects`) ### Changed - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer From a13d449b241b6e5fa14fb741694e63cd21569b2c Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Tue, 21 May 2019 09:39:32 +0800 Subject: [PATCH 33/56] Add tests for fallback routes --- test/web/fallback_test.exs | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 test/web/fallback_test.exs diff --git a/test/web/fallback_test.exs b/test/web/fallback_test.exs new file mode 100644 index 000000000..514923a20 --- /dev/null +++ b/test/web/fallback_test.exs @@ -0,0 +1,46 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.FallbackTest do + use Pleroma.Web.ConnCase + import Pleroma.Factory + + test "GET /registration/:token", %{conn: conn} do + assert conn + |> get("/registration/foo") + |> html_response(200) =~ "" + end + + test "GET /:maybe_nickname_or_id", %{conn: conn} do + user = insert(:user) + + assert conn + |> get("/foo") + |> html_response(200) =~ "" + + refute conn + |> get("/" <> user.nickname) + |> html_response(200) =~ "" + end + + test "GET /*path", %{conn: conn} do + assert conn + |> get("/foo") + |> html_response(200) =~ "" + + assert conn + |> get("/foo/bar") + |> html_response(200) =~ "" + end + + test "OPTIONS /*path", %{conn: conn} do + assert conn + |> options("/foo") + |> response(204) == "" + + assert conn + |> options("/foo/bar") + |> response(204) == "" + end +end From 3ab9255eda21a5f8a25047375af9608e0c0c7592 Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Tue, 21 May 2019 09:40:29 +0800 Subject: [PATCH 34/56] Respond with a 404 Not implemented JSON error message when requested API is not implemented --- CHANGELOG.md | 1 + lib/pleroma/web/router.ex | 7 +++++++ test/web/fallback_test.exs | 6 ++++++ 3 files changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 256df91b7..2ed380102 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Hide deactivated users and their statuses - Posts which are marked sensitive or tagged nsfw no longer have link previews. - HTTP connection timeout is now set to 10 seconds. +- Respond with a 404 Not implemented JSON error message when requested API is not implemented ### Fixed - Added an FTS index on objects. Running `vacuum analyze` and setting a larger `work_mem` is recommended. diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 4c29b24eb..49e28cc2d 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -710,6 +710,7 @@ defmodule Pleroma.Web.Router do scope "/", Fallback do get("/registration/:token", RedirectController, :registration_page) get("/:maybe_nickname_or_id", RedirectController, :redirector_with_meta) + get("/api*path", RedirectController, :api_not_implemented) get("/*path", RedirectController, :redirector) options("/*path", RedirectController, :empty) @@ -721,6 +722,12 @@ defmodule Fallback.RedirectController do alias Pleroma.User alias Pleroma.Web.Metadata + def api_not_implemented(conn, _params) do + conn + |> put_status(404) + |> json(%{error: "Not implemented"}) + end + def redirector(conn, _params, code \\ 200) do conn |> put_resp_content_type("text/html") diff --git a/test/web/fallback_test.exs b/test/web/fallback_test.exs index 514923a20..cc78b3ae1 100644 --- a/test/web/fallback_test.exs +++ b/test/web/fallback_test.exs @@ -24,6 +24,12 @@ test "GET /:maybe_nickname_or_id", %{conn: conn} do |> html_response(200) =~ "" end + test "GET /api*path", %{conn: conn} do + assert conn + |> get("/api/foo") + |> json_response(404) == %{"error" => "Not implemented"} + end + test "GET /*path", %{conn: conn} do assert conn |> get("/foo") From f76268135c014c20a482d30a7c9596ec2e7d6a69 Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Wed, 22 May 2019 07:11:09 +0800 Subject: [PATCH 35/56] Fix failing test --- test/web/admin_api/admin_api_controller_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index ca12c7215..c15c67e31 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -397,14 +397,14 @@ test "it returns 500 if `registrations_open` is enabled", %{conn: conn, user: us end end - test "/api/pleroma/admin/invite_token" do + test "/api/pleroma/admin/users/invite_token" do admin = insert(:user, info: %{is_admin: true}) conn = build_conn() |> assign(:user, admin) |> put_req_header("accept", "application/json") - |> get("/api/pleroma/admin/invite_token") + |> get("/api/pleroma/admin/users/invite_token") assert conn.status == 200 end From a023ca004cbd90e330cab35e4dfda16346d08668 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 03:12:48 +0000 Subject: [PATCH 36/56] prune objects task: use Repo.delete_all() --- lib/mix/tasks/pleroma/database.ex | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index fdb216037..f9bafb277 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -5,6 +5,7 @@ defmodule Mix.Tasks.Pleroma.Database do alias Mix.Tasks.Pleroma.Common alias Pleroma.Conversation + alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User require Logger @@ -78,6 +79,8 @@ def run(["update_users_following_followers_counts"]) do end def run(["prune_objects" | args]) do + import Ecto.Query + {options, [], []} = OptionParser.parse( args, @@ -96,11 +99,15 @@ def run(["prune_objects" | args]) do NaiveDateTime.utc_now() |> NaiveDateTime.add(-(deadline * 86_400)) - Repo.query!( - "DELETE FROM objects WHERE inserted_at < $1 AND split_part(data->>'actor', '/', 3) != $2", - [time_deadline, Pleroma.Web.Endpoint.host()], - timeout: :infinity + public = "https://www.w3.org/ns/activitystreams#Public" + + from(o in Object, + where: fragment("?->'to' \\? ? OR ?->'cc' \\? ?", o.data, ^public, o.data, ^public), + where: o.inserted_at < ^time_deadline, + where: + fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host()) ) + |> Repo.delete_all() if Keyword.get(options, :vacuum) do Logger.info("Runnning VACUUM FULL") From 045803346d70c1f9c6ea770485904fd7cc52969a Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 03:58:15 +0000 Subject: [PATCH 37/56] move key generation functions into Pleroma.Keys module --- lib/pleroma/keys.ex | 44 +++++++++++++++++++ lib/pleroma/signature.ex | 7 ++- lib/pleroma/user.ex | 21 +++++++++ .../activity_pub/activity_pub_controller.ex | 14 +++--- .../web/activity_pub/views/user_view.ex | 11 +++-- lib/pleroma/web/federator/federator.ex | 6 +-- lib/pleroma/web/salmon/salmon.ex | 44 ++----------------- lib/pleroma/web/web_finger/web_finger.ex | 26 +---------- test/keys_test.exs | 20 +++++++++ test/user_test.exs | 15 +++++++ test/web/activity_pub/activity_pub_test.exs | 2 +- .../web/activity_pub/views/user_view_test.exs | 13 +++--- test/web/salmon/salmon_test.exs | 19 ++------ test/web/web_finger/web_finger_test.exs | 15 ------- 14 files changed, 133 insertions(+), 124 deletions(-) create mode 100644 lib/pleroma/keys.ex create mode 100644 test/keys_test.exs diff --git a/lib/pleroma/keys.ex b/lib/pleroma/keys.ex new file mode 100644 index 000000000..b7bc7a4da --- /dev/null +++ b/lib/pleroma/keys.ex @@ -0,0 +1,44 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Keys do + # Native generation of RSA keys is only available since OTP 20+ and in default build conditions + # We try at compile time to generate natively an RSA key otherwise we fallback on the old way. + try do + _ = :public_key.generate_key({:rsa, 2048, 65_537}) + + def generate_rsa_pem do + key = :public_key.generate_key({:rsa, 2048, 65_537}) + entry = :public_key.pem_entry_encode(:RSAPrivateKey, key) + pem = :public_key.pem_encode([entry]) |> String.trim_trailing() + {:ok, pem} + end + rescue + _ -> + def generate_rsa_pem do + port = Port.open({:spawn, "openssl genrsa"}, [:binary]) + + {:ok, pem} = + receive do + {^port, {:data, pem}} -> {:ok, pem} + end + + Port.close(port) + + if Regex.match?(~r/RSA PRIVATE KEY/, pem) do + {:ok, pem} + else + :error + end + end + end + + def keys_from_pem(pem) do + [private_key_code] = :public_key.pem_decode(pem) + private_key = :public_key.pem_entry_decode(private_key_code) + {:RSAPrivateKey, _, modulus, exponent, _, _, _, _, _, _, _} = private_key + public_key = {:RSAPublicKey, modulus, exponent} + {:ok, private_key, public_key} + end +end diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex index b7ecf00a0..1a4d54c62 100644 --- a/lib/pleroma/signature.ex +++ b/lib/pleroma/signature.ex @@ -5,11 +5,10 @@ defmodule Pleroma.Signature do @behaviour HTTPSignatures.Adapter + alias Pleroma.Keys alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils - alias Pleroma.Web.Salmon - alias Pleroma.Web.WebFinger def fetch_public_key(conn) do with actor_id <- Utils.get_ap_id(conn.params["actor"]), @@ -33,8 +32,8 @@ def refetch_public_key(conn) do end def sign(%User{} = user, headers) do - with {:ok, %{info: %{keys: keys}}} <- WebFinger.ensure_keys_present(user), - {:ok, private_key, _} <- Salmon.keys_from_pem(keys) do + with {:ok, %{info: %{keys: keys}}} <- User.ensure_keys_present(user), + {:ok, private_key, _} <- Keys.keys_from_pem(keys) do HTTPSignatures.sign(private_key, user.ap_id <> "#main-key", headers) end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 05fe58f7c..653dec95f 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -10,6 +10,7 @@ defmodule Pleroma.User do alias Comeonin.Pbkdf2 alias Pleroma.Activity + alias Pleroma.Keys alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Registration @@ -1422,4 +1423,24 @@ def get_mascot(%{info: %{mascot: mascot}}) when is_nil(mascot) do } } end + + def ensure_keys_present(user) do + info = user.info + + if info.keys do + {:ok, user} + else + {:ok, pem} = Keys.generate_rsa_pem() + + info_cng = + info + |> User.Info.set_keys(pem) + + cng = + Ecto.Changeset.change(user) + |> Ecto.Changeset.put_embed(:info, info_cng) + + update_and_set_cache(cng) + end + end end diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index c967ab7a9..ad2ca1e54 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -39,7 +39,7 @@ def relay_active?(conn, _) do def user(conn, %{"nickname" => nickname}) do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do + {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_header("content-type", "application/activity+json") |> json(UserView.render("user.json", %{user: user})) @@ -106,7 +106,7 @@ def activity(conn, %{"uuid" => uuid}) do def following(conn, %{"nickname" => nickname, "page" => page}) do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do + {:ok, user} <- User.ensure_keys_present(user) do {page, _} = Integer.parse(page) conn @@ -117,7 +117,7 @@ def following(conn, %{"nickname" => nickname, "page" => page}) do def following(conn, %{"nickname" => nickname}) do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do + {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_header("content-type", "application/activity+json") |> json(UserView.render("following.json", %{user: user})) @@ -126,7 +126,7 @@ def following(conn, %{"nickname" => nickname}) do def followers(conn, %{"nickname" => nickname, "page" => page}) do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do + {:ok, user} <- User.ensure_keys_present(user) do {page, _} = Integer.parse(page) conn @@ -137,7 +137,7 @@ def followers(conn, %{"nickname" => nickname, "page" => page}) do def followers(conn, %{"nickname" => nickname}) do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do + {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_header("content-type", "application/activity+json") |> json(UserView.render("followers.json", %{user: user})) @@ -146,7 +146,7 @@ def followers(conn, %{"nickname" => nickname}) do def outbox(conn, %{"nickname" => nickname} = params) do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do + {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_header("content-type", "application/activity+json") |> json(UserView.render("outbox.json", %{user: user, max_id: params["max_id"]})) @@ -195,7 +195,7 @@ def inbox(conn, params) do def relay(conn, _params) do with %User{} = user <- Relay.get_actor(), - {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do + {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_header("content-type", "application/activity+json") |> json(UserView.render("user.json", %{user: user})) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 1254fdf6c..327e0e05b 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do use Pleroma.Web, :view + alias Pleroma.Keys alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub @@ -12,8 +13,6 @@ defmodule Pleroma.Web.ActivityPub.UserView do alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Endpoint alias Pleroma.Web.Router.Helpers - alias Pleroma.Web.Salmon - alias Pleroma.Web.WebFinger import Ecto.Query @@ -34,8 +33,8 @@ def render("endpoints.json", _), do: %{} # the instance itself is not a Person, but instead an Application def render("user.json", %{user: %{nickname: nil} = user}) do - {:ok, user} = WebFinger.ensure_keys_present(user) - {:ok, _, public_key} = Salmon.keys_from_pem(user.info.keys) + {:ok, user} = User.ensure_keys_present(user) + {:ok, _, public_key} = Keys.keys_from_pem(user.info.keys) public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key) public_key = :public_key.pem_encode([public_key]) @@ -62,8 +61,8 @@ def render("user.json", %{user: %{nickname: nil} = user}) do end def render("user.json", %{user: user}) do - {:ok, user} = WebFinger.ensure_keys_present(user) - {:ok, _, public_key} = Salmon.keys_from_pem(user.info.keys) + {:ok, user} = User.ensure_keys_present(user) + {:ok, _, public_key} = Keys.keys_from_pem(user.info.keys) public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key) public_key = :public_key.pem_encode([public_key]) diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 169fdf4dc..6b0b75284 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -11,7 +11,6 @@ defmodule Pleroma.Web.Federator do alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Federator.Publisher alias Pleroma.Web.Federator.RetryQueue - alias Pleroma.Web.WebFinger alias Pleroma.Web.Websub require Logger @@ -77,9 +76,8 @@ def perform(:request_subscription, websub) do def perform(:publish, activity) do Logger.debug(fn -> "Running publish for #{activity.data["id"]}" end) - with actor when not is_nil(actor) <- User.get_cached_by_ap_id(activity.data["actor"]) do - {:ok, actor} = WebFinger.ensure_keys_present(actor) - + with %User{} = actor <- User.get_cached_by_ap_id(activity.data["actor"]), + {:ok, actor} <- User.ensure_keys_present(actor) do Publisher.publish(actor, activity) end end diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index 42709ab47..fa30f73cd 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.Salmon do use Bitwise alias Pleroma.Activity + alias Pleroma.Keys alias Pleroma.Instances alias Pleroma.User alias Pleroma.Web.ActivityPub.Visibility @@ -89,45 +90,6 @@ def encode_key({:RSAPublicKey, modulus, exponent}) do "RSA.#{modulus_enc}.#{exponent_enc}" end - # Native generation of RSA keys is only available since OTP 20+ and in default build conditions - # We try at compile time to generate natively an RSA key otherwise we fallback on the old way. - try do - _ = :public_key.generate_key({:rsa, 2048, 65_537}) - - def generate_rsa_pem do - key = :public_key.generate_key({:rsa, 2048, 65_537}) - entry = :public_key.pem_entry_encode(:RSAPrivateKey, key) - pem = :public_key.pem_encode([entry]) |> String.trim_trailing() - {:ok, pem} - end - rescue - _ -> - def generate_rsa_pem do - port = Port.open({:spawn, "openssl genrsa"}, [:binary]) - - {:ok, pem} = - receive do - {^port, {:data, pem}} -> {:ok, pem} - end - - Port.close(port) - - if Regex.match?(~r/RSA PRIVATE KEY/, pem) do - {:ok, pem} - else - :error - end - end - end - - def keys_from_pem(pem) do - [private_key_code] = :public_key.pem_decode(pem) - private_key = :public_key.pem_entry_decode(private_key_code) - {:RSAPrivateKey, _, modulus, exponent, _, _, _, _, _, _, _} = private_key - public_key = {:RSAPublicKey, modulus, exponent} - {:ok, private_key, public_key} - end - def encode(private_key, doc) do type = "application/atom+xml" encoding = "base64url" @@ -227,7 +189,7 @@ def publish(%{info: %{keys: keys}} = user, %{data: %{"type" => type}} = activity |> :xmerl.export_simple(:xmerl_xml) |> to_string - {:ok, private, _} = keys_from_pem(keys) + {:ok, private, _} = Keys.keys_from_pem(keys) {:ok, feed} = encode(private, feed) remote_users = remote_users(activity) @@ -253,7 +215,7 @@ def publish(%{info: %{keys: keys}} = user, %{data: %{"type" => type}} = activity def publish(%{id: id}, _), do: Logger.debug(fn -> "Keys missing for user #{id}" end) def gather_webfinger_links(%User{} = user) do - {:ok, _private, public} = keys_from_pem(user.info.keys) + {:ok, _private, public} = Keys.keys_from_pem(user.info.keys) magic_key = encode_key(public) [ diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 1239b962a..c5b7d4acb 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.WebFinger do alias Pleroma.User alias Pleroma.Web alias Pleroma.Web.Federator.Publisher - alias Pleroma.Web.Salmon alias Pleroma.Web.XML alias Pleroma.XmlBuilder require Jason @@ -61,7 +60,7 @@ defp gather_links(%User{} = user) do end def represent_user(user, "JSON") do - {:ok, user} = ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) %{ "subject" => "acct:#{user.nickname}@#{Pleroma.Web.Endpoint.host()}", @@ -71,7 +70,7 @@ def represent_user(user, "JSON") do end def represent_user(user, "XML") do - {:ok, user} = ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) links = gather_links(user) @@ -88,27 +87,6 @@ def represent_user(user, "XML") do |> XmlBuilder.to_doc() end - # This seems a better fit in Salmon - def ensure_keys_present(user) do - info = user.info - - if info.keys do - {:ok, user} - else - {:ok, pem} = Salmon.generate_rsa_pem() - - info_cng = - info - |> User.Info.set_keys(pem) - - cng = - Ecto.Changeset.change(user) - |> Ecto.Changeset.put_embed(:info, info_cng) - - User.update_and_set_cache(cng) - end - end - defp get_magic_key(magic_key) do "data:application/magic-public-key," <> magic_key = magic_key {:ok, magic_key} diff --git a/test/keys_test.exs b/test/keys_test.exs new file mode 100644 index 000000000..776fdea6f --- /dev/null +++ b/test/keys_test.exs @@ -0,0 +1,20 @@ +defmodule Pleroma.KeysTest do + use Pleroma.DataCase + + alias Pleroma.Keys + + test "generates an RSA private key pem" do + {:ok, key} = Keys.generate_rsa_pem() + + assert is_binary(key) + assert Regex.match?(~r/RSA/, key) + end + + test "returns a public and private key from a pem" do + pem = File.read!("test/fixtures/private_key.pem") + {:ok, private, public} = Keys.keys_from_pem(pem) + + assert elem(private, 0) == :RSAPrivateKey + assert elem(public, 0) == :RSAPublicKey + end +end diff --git a/test/user_test.exs b/test/user_test.exs index cb6afbe07..019f2b56d 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1251,4 +1251,19 @@ test "if user is unconfirmed" do refute user.info.confirmation_token end end + + describe "ensure_keys_present" do + test "it creates keys for a user and stores them in info" do + user = insert(:user) + refute is_binary(user.info.keys) + {:ok, user} = User.ensure_keys_present(user) + assert is_binary(user.info.keys) + end + + test "it doesn't create keys if there already are some" do + user = insert(:user, %{info: %{keys: "xxx"}}) + {:ok, user} = User.ensure_keys_present(user) + assert user.info.keys == "xxx" + end + end end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index c18e0ab5f..f743f380b 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -1005,7 +1005,7 @@ test "it filters broken threads" do describe "update" do test "it creates an update activity with the new user data" do user = insert(:user) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) user_data = Pleroma.Web.ActivityPub.UserView.render("user.json", %{user: user}) {:ok, update} = diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index 9fb9455d2..e6483db8b 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -2,11 +2,12 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do use Pleroma.DataCase import Pleroma.Factory + alias Pleroma.User alias Pleroma.Web.ActivityPub.UserView test "Renders a user, including the public key" do user = insert(:user) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) result = UserView.render("user.json", %{user: user}) @@ -18,7 +19,7 @@ test "Renders a user, including the public key" do test "Does not add an avatar image if the user hasn't set one" do user = insert(:user) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) result = UserView.render("user.json", %{user: user}) refute result["icon"] @@ -32,7 +33,7 @@ test "Does not add an avatar image if the user hasn't set one" do } ) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) result = UserView.render("user.json", %{user: user}) assert result["icon"]["url"] == "https://someurl" @@ -42,7 +43,7 @@ test "Does not add an avatar image if the user hasn't set one" do describe "endpoints" do test "local users have a usable endpoints structure" do user = insert(:user) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) result = UserView.render("user.json", %{user: user}) @@ -58,7 +59,7 @@ test "local users have a usable endpoints structure" do test "remote users have an empty endpoints structure" do user = insert(:user, local: false) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) result = UserView.render("user.json", %{user: user}) @@ -68,7 +69,7 @@ test "remote users have an empty endpoints structure" do test "instance users do not expose oAuth endpoints" do user = insert(:user, nickname: nil, local: true) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) result = UserView.render("user.json", %{user: user}) diff --git a/test/web/salmon/salmon_test.exs b/test/web/salmon/salmon_test.exs index 232082779..e86e76fe9 100644 --- a/test/web/salmon/salmon_test.exs +++ b/test/web/salmon/salmon_test.exs @@ -5,6 +5,7 @@ defmodule Pleroma.Web.Salmon.SalmonTest do use Pleroma.DataCase alias Pleroma.Activity + alias Pleroma.Keys alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.Federator.Publisher @@ -34,12 +35,6 @@ test "errors on wrong magic key" do assert Salmon.decode_and_validate(@wrong_magickey, salmon) == :error end - test "generates an RSA private key pem" do - {:ok, key} = Salmon.generate_rsa_pem() - assert is_binary(key) - assert Regex.match?(~r/RSA/, key) - end - test "it encodes a magic key from a public key" do key = Salmon.decode_key(@magickey) magic_key = Salmon.encode_key(key) @@ -51,18 +46,10 @@ test "it decodes a friendica public key" do _key = Salmon.decode_key(@magickey_friendica) end - test "returns a public and private key from a pem" do - pem = File.read!("test/fixtures/private_key.pem") - {:ok, private, public} = Salmon.keys_from_pem(pem) - - assert elem(private, 0) == :RSAPrivateKey - assert elem(public, 0) == :RSAPublicKey - end - test "encodes an xml payload with a private key" do doc = File.read!("test/fixtures/incoming_note_activity.xml") pem = File.read!("test/fixtures/private_key.pem") - {:ok, private, public} = Salmon.keys_from_pem(pem) + {:ok, private, public} = Keys.keys_from_pem(pem) # Let's try a roundtrip. {:ok, salmon} = Salmon.encode(private, doc) @@ -105,7 +92,7 @@ test "it gets a magic key" do {:ok, activity} = Repo.insert(%Activity{data: activity_data, recipients: activity_data["to"]}) user = User.get_cached_by_ap_id(activity.data["actor"]) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) Salmon.publish(user, activity) diff --git a/test/web/web_finger/web_finger_test.exs b/test/web/web_finger/web_finger_test.exs index 6b20d8d56..335c95b18 100644 --- a/test/web/web_finger/web_finger_test.exs +++ b/test/web/web_finger/web_finger_test.exs @@ -105,19 +105,4 @@ test "it gets the xrd endpoint for statusnet" do assert template == "http://status.alpicola.com/main/xrd?uri={uri}" end end - - describe "ensure_keys_present" do - test "it creates keys for a user and stores them in info" do - user = insert(:user) - refute is_binary(user.info.keys) - {:ok, user} = WebFinger.ensure_keys_present(user) - assert is_binary(user.info.keys) - end - - test "it doesn't create keys if there already are some" do - user = insert(:user, %{info: %{keys: "xxx"}}) - {:ok, user} = WebFinger.ensure_keys_present(user) - assert user.info.keys == "xxx" - end - end end From 913484817076bf5ca4bdbe7c3c1ff34f7debd3e5 Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Wed, 22 May 2019 04:04:20 +0000 Subject: [PATCH 38/56] Do not truncate DM when it contains newlines and safe_dm_mentions is set to true --- lib/pleroma/formatter.ex | 2 +- test/formatter_test.exs | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 3d7c36d21..3e3b9fe97 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Formatter do alias Pleroma.User alias Pleroma.Web.MediaProxy - @safe_mention_regex ~r/^(\s*(?@.+?\s+)+)(?.*)/ + @safe_mention_regex ~r/^(\s*(?@.+?\s+)+)(?.*)/s @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/ diff --git a/test/formatter_test.exs b/test/formatter_test.exs index 5e7011160..47b91b121 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -206,6 +206,15 @@ test "given the 'safe_mention' option, it will still work without any mention" d assert mentions == [] assert expected_text == text end + + test "given the 'safe_mention' option, it will keep text after newlines" do + user = insert(:user) + text = " @#{user.nickname}\n hey dude\n\nhow are you doing?" + + {expected_text, _, _} = Formatter.linkify(text, safe_mention: true) + + assert expected_text =~ "how are you doing?" + end end describe ".parse_tags" do From 0e2c215a006c7ca5756e80a357ff6395a4325946 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 22 May 2019 07:22:19 +0200 Subject: [PATCH 39/56] MastoAPI AccountView: fill source.note with plaintext version of note Closes: https://git.pleroma.social/pleroma/pleroma/issues/926 --- lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +- test/web/mastodon_api/account_view_test.exs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 134c07b7e..b82d3319b 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -112,7 +112,7 @@ defp do_render("account.json", %{user: user} = opts) do fields: fields, bot: bot, source: %{ - note: "", + note: HTML.strip_tags((user.bio || "") |> String.replace("
", "\n")), sensitive: false, pleroma: %{} }, diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs index a24f2a050..aaf2261bb 100644 --- a/test/web/mastodon_api/account_view_test.exs +++ b/test/web/mastodon_api/account_view_test.exs @@ -55,7 +55,7 @@ test "Represent a user account" do fields: [], bot: false, source: %{ - note: "", + note: "valid html", sensitive: false, pleroma: %{} }, @@ -120,7 +120,7 @@ test "Represent a Service(bot) account" do fields: [], bot: true, source: %{ - note: "", + note: user.bio, sensitive: false, pleroma: %{} }, @@ -209,7 +209,7 @@ test "represent an embedded relationship" do fields: [], bot: true, source: %{ - note: "", + note: user.bio, sensitive: false, pleroma: %{} }, From 1344c85e2fe636fd6b9d033eb7add1c3a9701c7f Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 05:58:51 +0000 Subject: [PATCH 40/56] salmon: fix credo --- lib/pleroma/web/salmon/salmon.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index fa30f73cd..f25d92fad 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -10,8 +10,8 @@ defmodule Pleroma.Web.Salmon do use Bitwise alias Pleroma.Activity - alias Pleroma.Keys alias Pleroma.Instances + alias Pleroma.Keys alias Pleroma.User alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.Federator.Publisher From f9e0d09ec0082a096dcd4980bc5ffebe8e3139ae Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 22 May 2019 10:17:32 +0200 Subject: [PATCH 41/56] Changelog: Add SSH mode. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ff70e6e5..3d1e7640d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [unreleased] ### Added +- Optional SSH access mode. - [MongooseIM](https://github.com/esl/MongooseIM) http authentication support. - LDAP authentication - External OAuth provider authentication From b6cf62ddeab04db6bd2695c5537c81e0fb1aecaf Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 22 May 2019 10:28:50 +0200 Subject: [PATCH 42/56] Mix: Don't start esshd application if we don't need it. --- mix.exs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 95c052c34..bc5b6204f 100644 --- a/mix.exs +++ b/mix.exs @@ -40,9 +40,16 @@ def project do # # Type `mix help compile.app` for more information. def application do + extra_applications = [:logger, :runtime_tools, :comeonin, :quack] + extra_applications = if Application.get_env(:esshd, :enabled, false) do + [:esshd | extra_applications] + else + extra_applications + end + [ mod: {Pleroma.Application, []}, - extra_applications: [:logger, :runtime_tools, :comeonin, :esshd, :quack], + extra_applications: extra_applications, included_applications: [:ex_syslogger] ] end From db9a82d168cfc452611a44d92df2b81a5e6d1e69 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 22 May 2019 10:40:15 +0200 Subject: [PATCH 43/56] Linting. --- mix.exs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/mix.exs b/mix.exs index bc5b6204f..25ec46a46 100644 --- a/mix.exs +++ b/mix.exs @@ -41,11 +41,13 @@ def project do # Type `mix help compile.app` for more information. def application do extra_applications = [:logger, :runtime_tools, :comeonin, :quack] - extra_applications = if Application.get_env(:esshd, :enabled, false) do - [:esshd | extra_applications] - else - extra_applications - end + + extra_applications = + if Application.get_env(:esshd, :enabled, false) do + [:esshd | extra_applications] + else + extra_applications + end [ mod: {Pleroma.Application, []}, From b22145cbc40b57cf83f6389f063a76a03625ff16 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 22 May 2019 10:44:26 +0200 Subject: [PATCH 44/56] Documentation: Specify PEM format for SSH keys. Otherwise openssh-client 7.9 will generate a different format that can't be used by esshd. --- docs/config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config.md b/docs/config.md index 197326bbd..63ca61d1e 100644 --- a/docs/config.md +++ b/docs/config.md @@ -477,7 +477,7 @@ config :esshd, password_authenticator: "Pleroma.BBS.Authenticator" ``` -Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT` +Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -m PEM -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT` ## :auth From 3b12e1ba7c99382c678ce17629352135f44dcb9f Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 22 May 2019 11:01:10 +0200 Subject: [PATCH 45/56] Changelog: Add tip for debian users. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d1e7640d..b88edd072 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [unreleased] ### Added -- Optional SSH access mode. +- Optional SSH access mode. (Needs `erlang-ssh` package on Debian). - [MongooseIM](https://github.com/esl/MongooseIM) http authentication support. - LDAP authentication - External OAuth provider authentication From f4cfcead8868481c19ebd93b0fcd2b942dc0e477 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 22 May 2019 11:44:17 +0200 Subject: [PATCH 46/56] Mix: Bring ecto-sql back to mainline. --- mix.exs | 5 +---- mix.lock | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/mix.exs b/mix.exs index 95c052c34..9149b241f 100644 --- a/mix.exs +++ b/mix.exs @@ -66,10 +66,7 @@ defp deps do {:plug_cowboy, "~> 2.0"}, {:phoenix_pubsub, "~> 1.1"}, {:phoenix_ecto, "~> 4.0"}, - {:ecto_sql, - git: "https://github.com/elixir-ecto/ecto_sql", - ref: "14cb065a74c488d737d973f7a91bc036c6245f78", - override: true}, + {:ecto_sql, "~> 3.1"}, {:postgrex, ">= 0.13.5"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, diff --git a/mix.lock b/mix.lock index bacc09787..857bfca79 100644 --- a/mix.lock +++ b/mix.lock @@ -21,7 +21,7 @@ "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"}, "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"}, "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, - "ecto_sql": {:git, "https://github.com/elixir-ecto/ecto_sql", "14cb065a74c488d737d973f7a91bc036c6245f78", [ref: "14cb065a74c488d737d973f7a91bc036c6245f78"]}, + "ecto_sql": {:hex, :ecto_sql, "3.1.3", "2c536139190492d9de33c5fefac7323c5eaaa82e1b9bf93482a14649042f7cd9", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"}, "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"}, "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"}, From f323031927ecaf155e661b17cc9b96333fb9e4ad Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 22 May 2019 12:57:20 +0200 Subject: [PATCH 47/56] Mix: Only start sshd when needed, second try. --- mix.exs | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/mix.exs b/mix.exs index 25ec46a46..b2c075c85 100644 --- a/mix.exs +++ b/mix.exs @@ -40,18 +40,9 @@ def project do # # Type `mix help compile.app` for more information. def application do - extra_applications = [:logger, :runtime_tools, :comeonin, :quack] - - extra_applications = - if Application.get_env(:esshd, :enabled, false) do - [:esshd | extra_applications] - else - extra_applications - end - [ mod: {Pleroma.Application, []}, - extra_applications: extra_applications, + extra_applications: [:logger, :runtime_tools, :comeonin, :quack], included_applications: [:ex_syslogger] ] end @@ -129,7 +120,7 @@ defp deps do {:recon, github: "ferd/recon", tag: "2.4.0"}, {:quack, "~> 0.1.1"}, {:benchee, "~> 1.0"}, - {:esshd, "~> 0.1.0"}, + {:esshd, "~> 0.1.0", runtime: Application.get_env(:esshd, :enabled, false)}, {:ex_rated, "~> 1.2"}, {:plug_static_index_html, "~> 1.0.0"}, {:excoveralls, "~> 0.11.1", only: :test} From 78ac8ee56139ed98625c54ce627eb37047a361f0 Mon Sep 17 00:00:00 2001 From: lambda Date: Wed, 22 May 2019 11:07:51 +0000 Subject: [PATCH 48/56] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b88edd072..bb2306fc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [unreleased] ### Added -- Optional SSH access mode. (Needs `erlang-ssh` package on Debian). +- Optional SSH access mode. (Needs `erlang-ssh` package on some distributions). - [MongooseIM](https://github.com/esl/MongooseIM) http authentication support. - LDAP authentication - External OAuth provider authentication From 620908a2db86942a00bc0ba9c71c037061e26967 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 22 May 2019 15:44:50 +0000 Subject: [PATCH 49/56] [#699] add worker to clean expired oauth tokens --- CHANGELOG.md | 1 + config/config.exs | 4 +- docs/config.md | 2 + lib/pleroma/application.ex | 1 + lib/pleroma/web/oauth/token.ex | 36 ++++++-------- lib/pleroma/web/oauth/token/clean_worker.ex | 41 +++++++++++++++ lib/pleroma/web/oauth/token/query.ex | 55 +++++++++++++++++++++ test/web/oauth/token_test.exs | 13 +++++ 8 files changed, 132 insertions(+), 21 deletions(-) create mode 100644 lib/pleroma/web/oauth/token/clean_worker.ex create mode 100644 lib/pleroma/web/oauth/token/query.ex diff --git a/CHANGELOG.md b/CHANGELOG.md index b5c42d1fd..02d64a850 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - OAuth: added support for refresh tokens - Emoji packs and emoji pack manager - Object pruning (`mix pleroma.database prune_objects`) +- OAuth: added job to clean expired access tokens ### Changed - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer diff --git a/config/config.exs b/config/config.exs index a05f8b1d2..33b7e713d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -481,7 +481,9 @@ config :pleroma, :oauth2, token_expires_in: 600, - issue_new_refresh_token: true + issue_new_refresh_token: true, + clean_expired_tokens: false, + clean_expired_tokens_interval: 86_400_000 config :pleroma, :database, rum_enabled: false diff --git a/docs/config.md b/docs/config.md index a050068f4..264b65499 100644 --- a/docs/config.md +++ b/docs/config.md @@ -550,6 +550,8 @@ Configure OAuth 2 provider capabilities: * `token_expires_in` - The lifetime in seconds of the access token. * `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token. +* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`. +* `clean_expired_tokens_interval` - Interval to run the job to clean expired tokens. Defaults to `86_400_000` (24 hours). ## :emoji * `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]` diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index dab45a0b2..76df3945e 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -110,6 +110,7 @@ def start(_type, _args) do hackney_pool_children() ++ [ worker(Pleroma.Web.Federator.RetryQueue, []), + worker(Pleroma.Web.OAuth.Token.CleanWorker, []), worker(Pleroma.Stats, []), worker(Task, [&Pleroma.Web.Push.init/0], restart: :temporary, id: :web_push_init), worker(Task, [&Pleroma.Web.Federator.init/0], restart: :temporary, id: :federator_init) diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex index 66c95c2e9..f412f7eb2 100644 --- a/lib/pleroma/web/oauth/token.ex +++ b/lib/pleroma/web/oauth/token.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.OAuth.Token do use Ecto.Schema - import Ecto.Query import Ecto.Changeset alias Pleroma.Repo @@ -13,6 +12,7 @@ defmodule Pleroma.Web.OAuth.Token do alias Pleroma.Web.OAuth.App alias Pleroma.Web.OAuth.Authorization alias Pleroma.Web.OAuth.Token + alias Pleroma.Web.OAuth.Token.Query @expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600) @type t :: %__MODULE__{} @@ -31,17 +31,17 @@ defmodule Pleroma.Web.OAuth.Token do @doc "Gets token for app by access token" @spec get_by_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found} def get_by_token(%App{id: app_id} = _app, token) do - from(t in __MODULE__, where: t.app_id == ^app_id and t.token == ^token) + Query.get_by_app(app_id) + |> Query.get_by_token(token) |> Repo.find_resource() end @doc "Gets token for app by refresh token" @spec get_by_refresh_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found} def get_by_refresh_token(%App{id: app_id} = _app, token) do - from(t in __MODULE__, - where: t.app_id == ^app_id and t.refresh_token == ^token, - preload: [:user] - ) + Query.get_by_app(app_id) + |> Query.get_by_refresh_token(token) + |> Query.preload([:user]) |> Repo.find_resource() end @@ -97,29 +97,25 @@ def create_token(%App{} = app, %User{} = user, attrs \\ %{}) do end def delete_user_tokens(%User{id: user_id}) do - from( - t in Token, - where: t.user_id == ^user_id - ) + Query.get_by_user(user_id) |> Repo.delete_all() end def delete_user_token(%User{id: user_id}, token_id) do - from( - t in Token, - where: t.user_id == ^user_id, - where: t.id == ^token_id - ) + Query.get_by_user(user_id) + |> Query.get_by_id(token_id) + |> Repo.delete_all() + end + + def delete_expired_tokens do + Query.get_expired_tokens() |> Repo.delete_all() end def get_user_tokens(%User{id: user_id}) do - from( - t in Token, - where: t.user_id == ^user_id - ) + Query.get_by_user(user_id) + |> Query.preload([:app]) |> Repo.all() - |> Repo.preload(:app) end def is_expired?(%__MODULE__{valid_until: valid_until}) do diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex new file mode 100644 index 000000000..dca852449 --- /dev/null +++ b/lib/pleroma/web/oauth/token/clean_worker.ex @@ -0,0 +1,41 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.OAuth.Token.CleanWorker do + @moduledoc """ + The module represents functions to clean an expired oauth tokens. + """ + + # 10 seconds + @start_interval 10_000 + @interval Pleroma.Config.get( + # 24 hours + [:oauth2, :clean_expired_tokens_interval], + 86_400_000 + ) + @queue :background + + alias Pleroma.Web.OAuth.Token + + def start_link, do: GenServer.start_link(__MODULE__, nil) + + def init(_) do + if Pleroma.Config.get([:oauth2, :clean_expired_tokens], false) do + Process.send_after(self(), :perform, @start_interval) + {:ok, nil} + else + :ignore + end + end + + @doc false + def handle_info(:perform, state) do + Process.send_after(self(), :perform, @interval) + PleromaJobQueue.enqueue(@queue, __MODULE__, [:clean]) + {:noreply, state} + end + + # Job Worker Callbacks + def perform(:clean), do: Token.delete_expired_tokens() +end diff --git a/lib/pleroma/web/oauth/token/query.ex b/lib/pleroma/web/oauth/token/query.ex new file mode 100644 index 000000000..d92e1f071 --- /dev/null +++ b/lib/pleroma/web/oauth/token/query.ex @@ -0,0 +1,55 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.OAuth.Token.Query do + @moduledoc """ + Contains queries for OAuth Token. + """ + + import Ecto.Query, only: [from: 2] + + @type query :: Ecto.Queryable.t() | Token.t() + + alias Pleroma.Web.OAuth.Token + + @spec get_by_refresh_token(query, String.t()) :: query + def get_by_refresh_token(query \\ Token, refresh_token) do + from(q in query, where: q.refresh_token == ^refresh_token) + end + + @spec get_by_token(query, String.t()) :: query + def get_by_token(query \\ Token, token) do + from(q in query, where: q.token == ^token) + end + + @spec get_by_app(query, String.t()) :: query + def get_by_app(query \\ Token, app_id) do + from(q in query, where: q.app_id == ^app_id) + end + + @spec get_by_id(query, String.t()) :: query + def get_by_id(query \\ Token, id) do + from(q in query, where: q.id == ^id) + end + + @spec get_expired_tokens(query, DateTime.t() | nil) :: query + def get_expired_tokens(query \\ Token, date \\ nil) do + expired_date = date || Timex.now() + from(q in query, where: fragment("?", q.valid_until) < ^expired_date) + end + + @spec get_by_user(query, String.t()) :: query + def get_by_user(query \\ Token, user_id) do + from(q in query, where: q.user_id == ^user_id) + end + + @spec preload(query, any) :: query + def preload(query \\ Token, assoc_preload \\ []) + + def preload(query, assoc_preload) when is_list(assoc_preload) do + from(q in query, preload: ^assoc_preload) + end + + def preload(query, _assoc_preload), do: query +end diff --git a/test/web/oauth/token_test.exs b/test/web/oauth/token_test.exs index ad2a49f09..3c07309b7 100644 --- a/test/web/oauth/token_test.exs +++ b/test/web/oauth/token_test.exs @@ -69,4 +69,17 @@ test "deletes all tokens of a user" do assert tokens == 2 end + + test "deletes expired tokens" do + insert(:oauth_token, valid_until: Timex.shift(Timex.now(), days: -3)) + insert(:oauth_token, valid_until: Timex.shift(Timex.now(), days: -3)) + t3 = insert(:oauth_token) + t4 = insert(:oauth_token, valid_until: Timex.shift(Timex.now(), minutes: 10)) + {tokens, _} = Token.delete_expired_tokens() + assert tokens == 2 + available_tokens = Pleroma.Repo.all(Token) + + token_ids = available_tokens |> Enum.map(& &1.id) + assert token_ids == [t3.id, t4.id] + end end From 54e10a3e55fe46b71ef7f330605baf8bcccd5a44 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 22 May 2019 20:10:52 +0300 Subject: [PATCH 50/56] Disable timeouts for object pruning query --- lib/mix/tasks/pleroma/database.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index f9bafb277..4d480ac3f 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -107,7 +107,7 @@ def run(["prune_objects" | args]) do where: fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host()) ) - |> Repo.delete_all() + |> Repo.delete_all(timeout: :infinity) if Keyword.get(options, :vacuum) do Logger.info("Runnning VACUUM FULL") From 75b6c4b00433560fb5ee502f13e8261b4b8a246a Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 04:15:59 +0000 Subject: [PATCH 51/56] mrf: defang policy modules for filtering user profile objects --- lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 6 ++++-- lib/pleroma/web/activity_pub/mrf/user_allowlist.ex | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 7190652d2..ffaa4b7db 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -105,8 +105,8 @@ defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} defp check_report_removal(_actor_info, object), do: {:ok, object} @impl true - def filter(object) do - actor_info = URI.parse(object["actor"]) + def filter(%{"actor" => actor} = object) do + actor_info = URI.parse(actor) with {:ok, object} <- check_accept(actor_info, object), {:ok, object} <- check_reject(actor_info, object), @@ -119,4 +119,6 @@ def filter(object) do _e -> {:reject, nil} end end + + def filter(object), do: {:ok, object} end diff --git a/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex b/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex index f5078d818..47663414a 100644 --- a/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex +++ b/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex @@ -19,10 +19,12 @@ defp filter_by_list(%{"actor" => actor} = object, allow_list) do end @impl true - def filter(object) do - actor_info = URI.parse(object["actor"]) + def filter(%{"actor" => actor} = object) do + actor_info = URI.parse(actor) allow_list = Config.get([:mrf_user_allowlist, String.to_atom(actor_info.host)], []) filter_by_list(object, allow_list) end + + def filter(object), do: {:ok, object} end From 60f882b09f5f837546d59f8eef56b905e065ec60 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 04:33:10 +0000 Subject: [PATCH 52/56] activitypub: run user objects through MRF filters --- lib/pleroma/web/activity_pub/activity_pub.ex | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 3d9679ec0..aa0229db7 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -909,7 +909,7 @@ def upload(file, opts \\ []) do end end - def user_data_from_user_object(data) do + defp object_to_user_data(data) do avatar = data["icon"]["url"] && %{ @@ -956,9 +956,19 @@ def user_data_from_user_object(data) do {:ok, user_data} end + def user_data_from_user_object(data) do + with {:ok, data} <- MRF.filter(data), + {:ok, data} <- object_to_user_data(data) do + {:ok, data} + else + e -> {:error, e} + end + end + def fetch_and_prepare_user_from_ap_id(ap_id) do - with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do - user_data_from_user_object(data) + with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id), + {:ok, data} <- user_data_from_user_object(data) do + {:ok, data} else e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}") end From baf72d6c580e5c05ef5fea8a57c57150a5d38589 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 04:55:16 +0000 Subject: [PATCH 53/56] mrf: simple policy: add the ability to strip avatars and banners from user profiles --- config/config.exs | 4 ++- .../web/activity_pub/mrf/simple_policy.ex | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 33b7e713d..e90821d66 100644 --- a/config/config.exs +++ b/config/config.exs @@ -314,7 +314,9 @@ federated_timeline_removal: [], report_removal: [], reject: [], - accept: [] + accept: [], + avatar_removal: [], + banner_removal: [] config :pleroma, :mrf_keyword, reject: [], diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index ffaa4b7db..890d70a7a 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -104,6 +104,26 @@ defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} defp check_report_removal(_actor_info, object), do: {:ok, object} + defp check_avatar_removal(%{host: actor_host} = _actor_info, %{"icon" => _icon} = object) do + if actor_host in Pleroma.Config.get([:mrf_simple, :avatar_removal]) do + {:ok, Map.delete(object, "icon")} + else + {:ok, object} + end + end + + defp check_avatar_removal(_actor_info, object), do: {:ok, object} + + defp check_banner_removal(%{host: actor_host} = _actor_info, %{"image" => _image} = object) do + if actor_host in Pleroma.Config.get([:mrf_simple, :banner_removal]) do + {:ok, Map.delete(object, "image")} + else + {:ok, object} + end + end + + defp check_banner_removal(_actor_info, object), do: {:ok, object} + @impl true def filter(%{"actor" => actor} = object) do actor_info = URI.parse(actor) @@ -120,5 +140,17 @@ def filter(%{"actor" => actor} = object) do end end + def filter(%{"id" => actor, "type" => obj_type} = object) + when obj_type in ["Application", "Group", "Organization", "Person", "Service"] do + actor_info = URI.parse(actor) + + with {:ok, object} <- check_avatar_removal(actor_info, object), + {:ok, object} <- check_banner_removal(actor_info, object) do + {:ok, object} + else + _e -> {:reject, nil} + end + end + def filter(object), do: {:ok, object} end From 8086c7aed6cdc3b2ac1c09c6c40344e47be08ed9 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 05:08:37 +0000 Subject: [PATCH 54/56] tests: add tests for banner and avatar removal --- .../activity_pub/mrf/simple_policy_test.exs | 73 ++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs index 74af7dcde..3d1f26e60 100644 --- a/test/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/web/activity_pub/mrf/simple_policy_test.exs @@ -17,7 +17,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do federated_timeline_removal: [], report_removal: [], reject: [], - accept: [] + accept: [], + avatar_removal: [], + banner_removal: [] ) on_exit(fn -> @@ -206,6 +208,60 @@ test "has a matching host" do end end + describe "when :avatar_removal" do + test "is empty" do + Config.put([:mrf_simple, :avatar_removal], []) + + remote_user = build_remote_user() + + assert SimplePolicy.filter(remote_user) == {:ok, remote_user} + end + + test "is not empty but it doesn't have a matching host" do + Config.put([:mrf_simple, :avatar_removal], ["non.matching.remote"]) + + remote_user = build_remote_user() + + assert SimplePolicy.filter(remote_user) == {:ok, remote_user} + end + + test "has a matching host" do + Config.put([:mrf_simple, :avatar_removal], ["remote.instance"]) + + remote_user = build_remote_user() + {:ok, filtered} = SimplePolicy.filter(remote_user) + + refute filtered["icon"] + end + end + + describe "when :banner_removal" do + test "is empty" do + Config.put([:mrf_simple, :banner_removal], []) + + remote_user = build_remote_user() + + assert SimplePolicy.filter(remote_user) == {:ok, remote_user} + end + + test "is not empty but it doesn't have a matching host" do + Config.put([:mrf_simple, :banner_removal], ["non.matching.remote"]) + + remote_user = build_remote_user() + + assert SimplePolicy.filter(remote_user) == {:ok, remote_user} + end + + test "has a matching host" do + Config.put([:mrf_simple, :banner_removal], ["remote.instance"]) + + remote_user = build_remote_user() + {:ok, filtered} = SimplePolicy.filter(remote_user) + + refute filtered["image"] + end + end + defp build_local_message do %{ "actor" => "#{Pleroma.Web.base_url()}/users/alice", @@ -217,4 +273,19 @@ defp build_local_message do defp build_remote_message do %{"actor" => "https://remote.instance/users/bob"} end + + defp build_remote_user do + %{ + "id" => "https://remote.instance/users/bob", + "icon" => %{ + "url" => "http://example.com/image.jpg", + "type" => "Image" + }, + "image" => %{ + "url" => "http://example.com/image.jpg", + "type" => "Image" + }, + "type" => "Person" + } + end end From 7d9b33b3cebeed451210f754a8c34cc14a9e969b Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 05:55:09 +0000 Subject: [PATCH 55/56] update documentation for the new MRF features [no-ci] --- CHANGELOG.md | 2 ++ docs/config.md | 3 +++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02d64a850..2a2b11ddf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Emoji packs and emoji pack manager - Object pruning (`mix pleroma.database prune_objects`) - OAuth: added job to clean expired access tokens +- MRF: Support for rejecting reports from specific instances (`mrf_simple`) +- MRF: Support for stripping avatars and banner images from specific instances (`mrf_simple`) ### Changed - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer diff --git a/docs/config.md b/docs/config.md index 264b65499..1d1d24c32 100644 --- a/docs/config.md +++ b/docs/config.md @@ -220,6 +220,9 @@ relates to mascots on the mastodon frontend * `federated_timeline_removal`: List of instances to remove from Federated (aka The Whole Known Network) Timeline * `reject`: List of instances to reject any activities from * `accept`: List of instances to accept any activities from +* `report_removal`: List of instances to reject reports from +* `avatar_removal`: List of instances to strip avatars from +* `banner_removal`: List of instances to strip banners from ## :mrf_rejectnonpublic * `allow_followersonly`: whether to allow followers-only posts From 356c047759735bcd984ebd44059a21a3cf22af0e Mon Sep 17 00:00:00 2001 From: Alfie Pates Date: Thu, 23 May 2019 22:33:27 +0100 Subject: [PATCH 56/56] explicitly set reverse proxy upstream to IPv4 since Pleroma.Web.Endpoint binds on IPv4 only and `localhost.` resolves to [::0] on some systems fixes #930. --- installation/caddyfile-pleroma.example | 4 +++- installation/pleroma-apache.conf | 6 ++++-- installation/pleroma.nginx | 4 +++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/installation/caddyfile-pleroma.example b/installation/caddyfile-pleroma.example index fcf76718e..7985d9c67 100644 --- a/installation/caddyfile-pleroma.example +++ b/installation/caddyfile-pleroma.example @@ -10,7 +10,9 @@ example.tld { gzip - proxy / localhost:4000 { + # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only + # and `localhost.` resolves to [::0] on some systems: see issue #930 + proxy / 127.0.0.1:4000 { websocket transparent } diff --git a/installation/pleroma-apache.conf b/installation/pleroma-apache.conf index 2beb7c4cc..b5640ac3d 100644 --- a/installation/pleroma-apache.conf +++ b/installation/pleroma-apache.conf @@ -58,8 +58,10 @@ CustomLog ${APACHE_LOG_DIR}/access.log combined RewriteRule /(.*) ws://localhost:4000/$1 [P,L] ProxyRequests off - ProxyPass / http://localhost:4000/ - ProxyPassReverse / http://localhost:4000/ + # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only + # and `localhost.` resolves to [::0] on some systems: see issue #930 + ProxyPass / http://127.0.0.1:4000/ + ProxyPassReverse / http://127.0.0.1:4000/ RequestHeader set Host ${servername} ProxyPreserveHost On diff --git a/installation/pleroma.nginx b/installation/pleroma.nginx index cc75d78b2..7425da33f 100644 --- a/installation/pleroma.nginx +++ b/installation/pleroma.nginx @@ -69,7 +69,9 @@ server { proxy_set_header Connection "upgrade"; proxy_set_header Host $http_host; - proxy_pass http://localhost:4000; + # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only + # and `localhost.` resolves to [::0] on some systems: see issue #930 + proxy_pass http://127.0.0.1:4000; client_max_body_size 16m; }