From 452ca5250d7f7eeb51804d885122a60c8c4b84bf Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Fri, 28 Jun 2019 15:15:32 +0300 Subject: [PATCH 01/29] Merge admin and mastodon user views for reports --- .../web/admin_api/views/account_view.ex | 8 ++ .../web/admin_api/views/report_view.ex | 10 ++- .../admin_api/admin_api_controller_test.exs | 86 ++++++++++++++----- test/web/admin_api/views/report_view_test.exs | 24 +++++- 4 files changed, 100 insertions(+), 28 deletions(-) diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex index 28bb667d8..7e1b9c431 100644 --- a/lib/pleroma/web/admin_api/views/account_view.ex +++ b/lib/pleroma/web/admin_api/views/account_view.ex @@ -5,8 +5,11 @@ defmodule Pleroma.Web.AdminAPI.AccountView do use Pleroma.Web, :view + alias Pleroma.HTML + alias Pleroma.User alias Pleroma.User.Info alias Pleroma.Web.AdminAPI.AccountView + alias Pleroma.Web.MediaProxy def render("index.json", %{users: users, count: count, page_size: page_size}) do %{ @@ -17,9 +20,14 @@ def render("index.json", %{users: users, count: count, page_size: page_size}) do end def render("show.json", %{user: user}) do + avatar = User.avatar_url(user) |> MediaProxy.url() + display_name = HTML.strip_tags(user.name || user.nickname) + %{ "id" => user.id, + "avatar" => avatar, "nickname" => user.nickname, + "display_name" => display_name, "deactivated" => user.info.deactivated, "local" => user.local, "roles" => Info.roles(user.info), diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex index e7db3a8ff..73ccdc582 100644 --- a/lib/pleroma/web/admin_api/views/report_view.ex +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.AdminAPI.ReportView do alias Pleroma.HTML alias Pleroma.User alias Pleroma.Web.CommonAPI.Utils - alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.StatusView def render("index.json", %{reports: reports}) do @@ -38,12 +37,17 @@ def render("show.json", %{report: report}) do %{ id: report.id, - account: AccountView.render("account.json", %{user: account}), - actor: AccountView.render("account.json", %{user: user}), + account: merge_account_views(account), + actor: merge_account_views(user), content: content, created_at: created_at, statuses: StatusView.render("index.json", %{activities: statuses, as: :activity}), state: report.data["state"] } end + + defp merge_account_views(user) do + Pleroma.Web.MastodonAPI.AccountView.render("account.json", %{user: user}) + |> Map.merge(Pleroma.Web.AdminAPI.AccountView.render("show.json", %{user: user})) + end end diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 4278ac59d..3c46eff06 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -9,6 +9,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do alias Pleroma.User alias Pleroma.UserInviteToken alias Pleroma.Web.CommonAPI + alias Pleroma.HTML + alias Pleroma.Web.MediaProxy import Pleroma.Factory describe "/api/pleroma/admin/users" do @@ -58,7 +60,9 @@ test "Show", %{conn: conn} do "local" => true, "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) } assert expected == json_response(conn, 200) @@ -445,7 +449,9 @@ test "renders users array for the first page", %{conn: conn, admin: admin} do "nickname" => admin.nickname, "roles" => %{"admin" => true, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(admin) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(admin.name || admin.nickname) }, %{ "deactivated" => user.info.deactivated, @@ -453,7 +459,9 @@ test "renders users array for the first page", %{conn: conn, admin: admin} do "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => false, - "tags" => ["foo", "bar"] + "tags" => ["foo", "bar"], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) } ] |> Enum.sort_by(& &1["nickname"]) @@ -492,7 +500,9 @@ test "regular search", %{conn: conn} do "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) } ] } @@ -514,7 +524,9 @@ test "search by domain", %{conn: conn} do "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) } ] } @@ -536,7 +548,9 @@ test "search by full nickname", %{conn: conn} do "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) } ] } @@ -558,7 +572,9 @@ test "search by display name", %{conn: conn} do "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) } ] } @@ -580,7 +596,9 @@ test "search by email", %{conn: conn} do "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) } ] } @@ -602,7 +620,9 @@ test "regular search with page size", %{conn: conn} do "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) } ] } @@ -619,7 +639,9 @@ test "regular search with page size", %{conn: conn} do "nickname" => user2.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user2) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user2.name || user2.nickname) } ] } @@ -646,7 +668,9 @@ test "only local users" do "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) } ] } @@ -671,7 +695,9 @@ test "only local users with no query", %{admin: old_admin} do "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) }, %{ "deactivated" => admin.info.deactivated, @@ -679,7 +705,9 @@ test "only local users with no query", %{admin: old_admin} do "nickname" => admin.nickname, "roles" => %{"admin" => true, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(admin) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(admin.name || admin.nickname) }, %{ "deactivated" => false, @@ -687,7 +715,9 @@ test "only local users with no query", %{admin: old_admin} do "local" => true, "nickname" => old_admin.nickname, "roles" => %{"admin" => true, "moderator" => false}, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(old_admin) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(old_admin.name || old_admin.nickname) } ] |> Enum.sort_by(& &1["nickname"]) @@ -714,7 +744,9 @@ test "load only admins", %{conn: conn, admin: admin} do "nickname" => admin.nickname, "roles" => %{"admin" => true, "moderator" => false}, "local" => admin.local, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(admin) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(admin.name || admin.nickname) }, %{ "deactivated" => false, @@ -722,7 +754,9 @@ test "load only admins", %{conn: conn, admin: admin} do "nickname" => second_admin.nickname, "roles" => %{"admin" => true, "moderator" => false}, "local" => second_admin.local, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(second_admin) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(second_admin.name || second_admin.nickname) } ] |> Enum.sort_by(& &1["nickname"]) @@ -751,7 +785,9 @@ test "load only moderators", %{conn: conn} do "nickname" => moderator.nickname, "roles" => %{"admin" => false, "moderator" => true}, "local" => moderator.local, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(moderator) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(moderator.name || moderator.nickname) } ] } @@ -773,7 +809,9 @@ test "load users with tags list", %{conn: conn} do "nickname" => user1.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => user1.local, - "tags" => ["first"] + "tags" => ["first"], + "avatar" => User.avatar_url(user1) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user1.name || user1.nickname) }, %{ "deactivated" => false, @@ -781,7 +819,9 @@ test "load users with tags list", %{conn: conn} do "nickname" => user2.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => user2.local, - "tags" => ["second"] + "tags" => ["second"], + "avatar" => User.avatar_url(user2) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user2.name || user2.nickname) } ] |> Enum.sort_by(& &1["nickname"]) @@ -815,7 +855,9 @@ test "it works with multiple filters" do "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => user.local, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) } ] } @@ -838,7 +880,9 @@ test "PATCH /api/pleroma/admin/users/:nickname/toggle_activation" do "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) } end diff --git a/test/web/admin_api/views/report_view_test.exs b/test/web/admin_api/views/report_view_test.exs index f35f36cac..75d8bb4b5 100644 --- a/test/web/admin_api/views/report_view_test.exs +++ b/test/web/admin_api/views/report_view_test.exs @@ -18,8 +18,16 @@ test "renders a report" do expected = %{ content: nil, - actor: AccountView.render("account.json", %{user: user}), - account: AccountView.render("account.json", %{user: other_user}), + actor: + Map.merge( + AccountView.render("account.json", %{user: user}), + Pleroma.Web.AdminAPI.AccountView.render("show.json", %{user: user}) + ), + account: + Map.merge( + AccountView.render("account.json", %{user: other_user}), + Pleroma.Web.AdminAPI.AccountView.render("show.json", %{user: other_user}) + ), statuses: [], state: "open", id: activity.id @@ -42,8 +50,16 @@ test "includes reported statuses" do expected = %{ content: nil, - actor: AccountView.render("account.json", %{user: user}), - account: AccountView.render("account.json", %{user: other_user}), + actor: + Map.merge( + AccountView.render("account.json", %{user: user}), + Pleroma.Web.AdminAPI.AccountView.render("show.json", %{user: user}) + ), + account: + Map.merge( + AccountView.render("account.json", %{user: other_user}), + Pleroma.Web.AdminAPI.AccountView.render("show.json", %{user: other_user}) + ), statuses: [StatusView.render("status.json", %{activity: activity})], state: "open", id: report_activity.id From 90927e6a9a7b29f3a12335fd8b7ba899dcc0dffd Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Fri, 28 Jun 2019 15:27:18 +0300 Subject: [PATCH 02/29] Kowalski, analysis! --- test/web/admin_api/admin_api_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 3c46eff06..4ea33a6cc 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -6,10 +6,10 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do use Pleroma.Web.ConnCase alias Pleroma.Activity + alias Pleroma.HTML alias Pleroma.User alias Pleroma.UserInviteToken alias Pleroma.Web.CommonAPI - alias Pleroma.HTML alias Pleroma.Web.MediaProxy import Pleroma.Factory From e556c9666115c026541227f5dafc8b995330c66e Mon Sep 17 00:00:00 2001 From: Julien Wehmschulte Date: Fri, 28 Jun 2019 18:25:21 +0000 Subject: [PATCH 03/29] Update docs/installation/arch_linux_en.md, docs/installation/centos7_en.md, docs/installation/debian_based_en.md, docs/installation/debian_based_jp.md, docs/installation/gentoo_en.md files --- docs/installation/alpine_linux_en.md | 12 ++++++------ docs/installation/arch_linux_en.md | 12 ++++++------ docs/installation/centos7_en.md | 12 ++++++------ docs/installation/debian_based_en.md | 12 ++++++------ docs/installation/debian_based_jp.md | 8 ++++++-- docs/installation/gentoo_en.md | 12 ++++++------ 6 files changed, 36 insertions(+), 32 deletions(-) diff --git a/docs/installation/alpine_linux_en.md b/docs/installation/alpine_linux_en.md index e1d69c873..a9b5afd33 100644 --- a/docs/installation/alpine_linux_en.md +++ b/docs/installation/alpine_linux_en.md @@ -203,12 +203,12 @@ sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new Date: Thu, 27 Jun 2019 03:06:58 +0000 Subject: [PATCH 04/29] MRF: add mediaproxy warming policy --- .../mrf/mediaproxy_warming_policy.ex | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex diff --git a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex new file mode 100644 index 000000000..01d21a299 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex @@ -0,0 +1,56 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do + @moduledoc "Preloads any attachments in the MediaProxy cache by prefetching them" + @behaviour Pleroma.Web.ActivityPub.MRF + + alias Pleroma.HTTP + alias Pleroma.Web.MediaProxy + + require Logger + + @hackney_options [ + pool: :media, + recv_timeout: 10_000 + ] + + def perform(:prefetch, url) do + Logger.info("Prefetching #{inspect(url)}") + + url + |> MediaProxy.url() + |> HTTP.get([], adapter: @hackney_options) + end + + def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do + Enum.each(attachments, fn + %{"url" => url} when is_list(url) -> + url + |> Enum.each(fn + %{"href" => href} -> + PleromaJobQueue.enqueue(:background, __MODULE__, [:prefetch, href]) + + x -> + Logger.debug("Unhandled attachment URL object #{inspect(x)}") + end) + + x -> + Logger.debug("Unhandled attachment #{inspect(x)}") + end) + end + + @impl true + def filter( + %{"type" => "Create", "object" => %{"attachment" => attachments} = _object} = message + ) + when is_list(attachments) and length(attachments) > 0 do + PleromaJobQueue.enqueue(:background, __MODULE__, [:preload, message]) + + {:ok, message} + end + + @impl true + def filter(message), do: {:ok, message} +end From 074ffee0b2eeae5b3911f47f4ce17b3bbf74152d Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 27 Jun 2019 21:34:27 +0000 Subject: [PATCH 05/29] docs: add documentation for MediaProxyWarmingPolicy --- CHANGELOG.md | 4 ++++ docs/config.md | 1 + 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a65988991..a6ec8674d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [Unreleased] +### Added +- MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) + ## [1.0.0] - 2019-06-29 ### Security - Mastodon API: Fix display names not being sanitized diff --git a/docs/config.md b/docs/config.md index 7a53364ef..7d5be3980 100644 --- a/docs/config.md +++ b/docs/config.md @@ -98,6 +98,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `Pleroma.Web.ActivityPub.MRF.RejectNonPublic`: Drops posts with non-public visibility settings (See ``:mrf_rejectnonpublic`` section) * `Pleroma.Web.ActivityPub.MRF.EnsureRePrepended`: Rewrites posts to ensure that replies to posts with subjects do not have an identical subject and instead begin with re:. * `Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy`: Rejects posts from likely spambots by rejecting posts from new users that contain links. + * `Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`: Crawls attachments using their MediaProxy URLs so that the MediaProxy cache is primed. * `public`: Makes the client API in authentificated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. * `quarantined_instances`: List of ActivityPub instances where private(DMs, followers-only) activities will not be send. * `managed_config`: Whenether the config for pleroma-fe is configured in this config or in ``static/config.json`` From c22f0bfb8056889ea3180e0adbf46458eabf63a7 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Fri, 28 Jun 2019 23:41:42 +0000 Subject: [PATCH 06/29] test: add tests for mediaproxy warming policy --- .../mrf/mediaproxy_warming_policy_test.exs | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 test/web/activity_pub/mrf/mediaproxy_warming_policy_test.exs diff --git a/test/web/activity_pub/mrf/mediaproxy_warming_policy_test.exs b/test/web/activity_pub/mrf/mediaproxy_warming_policy_test.exs new file mode 100644 index 000000000..372e789be --- /dev/null +++ b/test/web/activity_pub/mrf/mediaproxy_warming_policy_test.exs @@ -0,0 +1,45 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicyTest do + use Pleroma.DataCase + + alias Pleroma.HTTP + alias Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy + + import Mock + + @message %{ + "type" => "Create", + "object" => %{ + "type" => "Note", + "content" => "content", + "attachment" => [ + %{"url" => [%{"href" => "http://example.com/image.jpg"}]} + ] + } + } + + test "it prefetches media proxy URIs" do + with_mock HTTP, get: fn _, _, _ -> {:ok, []} end do + MediaProxyWarmingPolicy.filter(@message) + assert called(HTTP.get(:_, :_, :_)) + end + end + + test "it does nothing when no attachments are present" do + object = + @message["object"] + |> Map.delete("attachment") + + message = + @message + |> Map.put("object", object) + + with_mock HTTP, get: fn _, _, _ -> {:ok, []} end do + MediaProxyWarmingPolicy.filter(message) + refute called(HTTP.get(:_, :_, :_)) + end + end +end From b2548e65dbfeb4ec20ff8fdc4230f7aedbb452f0 Mon Sep 17 00:00:00 2001 From: Yuji Nakao Date: Sat, 29 Jun 2019 04:17:49 +0000 Subject: [PATCH 07/29] Update migrating_from_source_otp_en.md: Replace `reload-daemon` with `daemon-reload`. --- docs/installation/migrating_from_source_otp_en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation/migrating_from_source_otp_en.md b/docs/installation/migrating_from_source_otp_en.md index 0b41d0c0e..df3e2f4a0 100644 --- a/docs/installation/migrating_from_source_otp_en.md +++ b/docs/installation/migrating_from_source_otp_en.md @@ -128,7 +128,7 @@ Debian/Ubuntu: cp ~pleroma/installation/pleroma.service /etc/systemd/system/pleroma.service # Reload service files -systemctl reload-daemon +systemctl daemon-reload # Reenable pleroma to start on boot systemctl reenable pleroma From ab6de9b478bf9c2b46603c2761fd99b6506f8b08 Mon Sep 17 00:00:00 2001 From: Yuji Nakao Date: Sat, 29 Jun 2019 04:24:44 +0000 Subject: [PATCH 08/29] Move all upload contents despite upload folder itself. --- docs/installation/migrating_from_source_otp_en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation/migrating_from_source_otp_en.md b/docs/installation/migrating_from_source_otp_en.md index 0b41d0c0e..5264790f3 100644 --- a/docs/installation/migrating_from_source_otp_en.md +++ b/docs/installation/migrating_from_source_otp_en.md @@ -49,7 +49,7 @@ mkdir -p /var/lib/pleroma/static chown -R pleroma /var/lib/pleroma # If you use the local uploader with default settings your uploads should be located in `~pleroma/uploads` -mv ~pleroma/uploads /var/lib/pleroma/uploads +mv ~pleroma/uploads/* /var/lib/pleroma/uploads # If you have created the custom public files directory with default settings it should be located in `~pleroma/instance/static` mv ~pleroma/instance/static /var/lib/pleroma/static From eddcebf8ae699edd9e503dd7a3c51112ca063a21 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 29 Jun 2019 11:35:42 +0300 Subject: [PATCH 09/29] Add a warning about service files assuming installation paths --- docs/installation/migrating_from_source_otp_en.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/installation/migrating_from_source_otp_en.md b/docs/installation/migrating_from_source_otp_en.md index d07a0f622..b779be8cc 100644 --- a/docs/installation/migrating_from_source_otp_en.md +++ b/docs/installation/migrating_from_source_otp_en.md @@ -122,6 +122,8 @@ su pleroma -s $SHELL -lc "./bin/pleroma stop" ## Setting up a system service OTP releases have different service files than from-source installs so they need to be copied over again. +**Warning:** The service files assume pleroma user's home directory is `/opt/pleroma`, please make sure all paths fit your installation. + Debian/Ubuntu: ```sh # Copy the service into a proper directory From ee79c67081cef80d2345fdc9b96e925354020671 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 29 Jun 2019 12:29:12 +0300 Subject: [PATCH 10/29] Build releases only on tags or develop Needed so we could push documentation updates to master without triggering a rebuild --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f8711f299..3868cb603 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -148,7 +148,7 @@ amd64: # TODO: Replace with upstream image when 1.9.0 comes out image: rinpatch/elixir:1.9.0-rc.0 only: &release-only - - master@pleroma/pleroma + - tags - develop@pleroma/pleroma artifacts: &release-artifacts name: "pleroma-$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA-$CI_JOB_NAME" From 0f6e333d8be84c6b75552411cd6838b9a0a16cad Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 29 Jun 2019 13:27:15 +0300 Subject: [PATCH 11/29] Revert "Build releases only on tags or develop" This reverts commit 54d287377054fa48c5a14cf6b056cd80e838b264. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3868cb603..f8711f299 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -148,7 +148,7 @@ amd64: # TODO: Replace with upstream image when 1.9.0 comes out image: rinpatch/elixir:1.9.0-rc.0 only: &release-only - - tags + - master@pleroma/pleroma - develop@pleroma/pleroma artifacts: &release-artifacts name: "pleroma-$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA-$CI_JOB_NAME" From 4c40ada7a5f5067f26135de5fcd97b2d97f90441 Mon Sep 17 00:00:00 2001 From: deorsum Date: Sat, 29 Jun 2019 20:39:03 +1000 Subject: [PATCH 12/29] Add a caveat for docker deployment in the config docs --- docs/config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config.md b/docs/config.md index 7d5be3980..feef43ba9 100644 --- a/docs/config.md +++ b/docs/config.md @@ -280,7 +280,7 @@ config :pleroma, :mrf_subchain, ## Pleroma.Web.Endpoint `Phoenix` endpoint configuration, all configuration options can be viewed [here](https://hexdocs.pm/phoenix/Phoenix.Endpoint.html#module-dynamic-configuration), only common options are listed here -* `http` - a list containing http protocol configuration, all configuration options can be viewed [here](https://hexdocs.pm/plug_cowboy/Plug.Cowboy.html#module-options), only common options are listed here +* `http` - a list containing http protocol configuration, all configuration options can be viewed [here](https://hexdocs.pm/plug_cowboy/Plug.Cowboy.html#module-options), only common options are listed here. For deployment using docker, you need to set this to `[ip: {0,0,0,0}, port: 4000]` to make pleroma accessible from other containers (such as your nginx server). - `ip` - a tuple consisting of 4 integers - `port` * `url` - a list containing the configuration for generating urls, accepts From bdc1592377d19da67040a78faf360770c54d4e6a Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 29 Jun 2019 14:25:47 +0300 Subject: [PATCH 13/29] Add PLEROMA_BUILD_ENV to override env for the version generator and use it in the CI for building docs This is needed because if we run with MIX_ENV=dev we will get a "+dev" suffix to the version in the docs, but if we run with MIX_ENV=prod the docs build will fail because ex_doc is dev-only dependency. --- .gitlab-ci.yml | 1 + mix.exs | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f8711f299..d0c540b16 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,6 +35,7 @@ docs-build: - develop@pleroma/pleroma variables: MIX_ENV: dev + PLEROMA_BUILD_ENV: prod script: - mix deps.get - mix compile diff --git a/mix.exs b/mix.exs index 19a80ffb2..c2618d2b2 100644 --- a/mix.exs +++ b/mix.exs @@ -207,6 +207,15 @@ defp version(version) do env_name = if Mix.env() != :prod, do: to_string(Mix.env()) + env_override = System.get_env("PLEROMA_BUILD_ENV") + + env_name = + if env_override do + if env_override != "prod", do: env_override + else + env_name + end + build = [build_name, env_name] |> Enum.filter(fn string -> string && string != "" end) From 4c60a562a7392294683caae71827d0053a3c3466 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 29 Jun 2019 22:24:03 +0300 Subject: [PATCH 14/29] Fix not being able to pin unlisted posts Closes #1038 --- CHANGELOG.md | 2 ++ lib/pleroma/web/common_api/common_api.ex | 4 ++-- test/web/common_api/common_api_test.exs | 5 +++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6ec8674d..e30ab803c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Added - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) +### Fixed +- Not being able to pin unlisted posts ## [1.0.0] - 2019-06-29 ### Security diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index f8df1e2ea..f71c67a3d 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.Web.ActivityPub.Visibility import Pleroma.Web.CommonAPI.Utils @@ -284,12 +285,11 @@ def pin(id_or_ap_id, %{ap_id: user_ap_id} = user) do }, object: %Object{ data: %{ - "to" => object_to, "type" => "Note" } } } = activity <- get_by_id_or_ap_id(id_or_ap_id), - true <- Enum.member?(object_to, "https://www.w3.org/ns/activitystreams#Public"), + true <- Visibility.is_public?(activity), %{valid?: true} = info_changeset <- User.Info.add_pinnned_activity(user.info, activity), changeset <- diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index e96106f11..6f57bbe1f 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -188,6 +188,11 @@ test "pin status", %{user: user, activity: activity} do assert %User{info: %{pinned_activities: [^id]}} = user end + test "unlisted statuses can be pinned", %{user: user} do + {:ok, activity} = CommonAPI.post(user, %{"status" => "HI!!!", "visibility" => "unlisted"}) + assert {:ok, ^activity} = CommonAPI.pin(activity.id, user) + end + test "only self-authored can be pinned", %{activity: activity} do user = insert(:user) From acd20f166b696254c6a632101d693d03416ad68d Mon Sep 17 00:00:00 2001 From: Maksim Date: Sun, 30 Jun 2019 07:28:35 +0000 Subject: [PATCH 15/29] [#1026] Filter.AnonymizeFilename added ability to retain file extension with custom text --- CHANGELOG.md | 3 ++ docs/config.md | 2 +- lib/pleroma/config.ex | 2 +- .../upload/filter/anonymize_filename.ex | 17 ++++++-- .../upload/filter/anonymize_filename_test.exs | 40 +++++++++++++++++++ 5 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 test/upload/filter/anonymize_filename_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index e30ab803c..96473b1b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Not being able to pin unlisted posts +### Changed +- Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text + ## [1.0.0] - 2019-06-29 ### Security - Mastodon API: Fix display names not being sanitized diff --git a/docs/config.md b/docs/config.md index feef43ba9..8afccb228 100644 --- a/docs/config.md +++ b/docs/config.md @@ -36,7 +36,7 @@ No specific configuration. This filter replaces the filename (not the path) of an upload. For complete obfuscation, add `Pleroma.Upload.Filter.Dedupe` before AnonymizeFilename. -* `text`: Text to replace filenames in links. If empty, `{random}.extension` will be used. +* `text`: Text to replace filenames in links. If empty, `{random}.extension` will be used. You can get the original filename extension by using `{extension}`, for example `custom-file-name.{extension}`. ## Pleroma.Emails.Mailer * `adapter`: one of the mail adapters listed in [Swoosh readme](https://github.com/swoosh/swoosh#adapters), or `Swoosh.Adapters.Local` for in-memory mailbox. diff --git a/lib/pleroma/config.ex b/lib/pleroma/config.ex index 71a47b9fb..fcc039710 100644 --- a/lib/pleroma/config.ex +++ b/lib/pleroma/config.ex @@ -38,7 +38,7 @@ def put([key], value), do: put(key, value) def put([parent_key | keys], value) do parent = - Application.get_env(:pleroma, parent_key) + Application.get_env(:pleroma, parent_key, []) |> put_in(keys, value) Application.put_env(:pleroma, parent_key, parent) diff --git a/lib/pleroma/upload/filter/anonymize_filename.ex b/lib/pleroma/upload/filter/anonymize_filename.ex index 5ca53a79b..a8516811c 100644 --- a/lib/pleroma/upload/filter/anonymize_filename.ex +++ b/lib/pleroma/upload/filter/anonymize_filename.ex @@ -10,10 +10,19 @@ defmodule Pleroma.Upload.Filter.AnonymizeFilename do """ @behaviour Pleroma.Upload.Filter - def filter(upload) do - extension = List.last(String.split(upload.name, ".")) - name = Pleroma.Config.get([__MODULE__, :text], random(extension)) - {:ok, %Pleroma.Upload{upload | name: name}} + alias Pleroma.Config + alias Pleroma.Upload + + def filter(%Upload{name: name} = upload) do + extension = List.last(String.split(name, ".")) + name = predefined_name(extension) || random(extension) + {:ok, %Upload{upload | name: name}} + end + + @spec predefined_name(String.t()) :: String.t() | nil + defp predefined_name(extension) do + with name when not is_nil(name) <- Config.get([__MODULE__, :text]), + do: String.replace(name, "{extension}", extension) end defp random(extension) do diff --git a/test/upload/filter/anonymize_filename_test.exs b/test/upload/filter/anonymize_filename_test.exs new file mode 100644 index 000000000..02241cfa4 --- /dev/null +++ b/test/upload/filter/anonymize_filename_test.exs @@ -0,0 +1,40 @@ +defmodule Pleroma.Upload.Filter.AnonymizeFilenameTest do + use Pleroma.DataCase + + alias Pleroma.Config + alias Pleroma.Upload + + setup do + custom_filename = Config.get([Upload.Filter.AnonymizeFilename, :text]) + + on_exit(fn -> + Config.put([Upload.Filter.AnonymizeFilename, :text], custom_filename) + end) + + upload_file = %Upload{ + name: "an… image.jpg", + content_type: "image/jpg", + path: Path.absname("test/fixtures/image_tmp.jpg") + } + + %{upload_file: upload_file} + end + + test "it replaces filename on pre-defined text", %{upload_file: upload_file} do + Config.put([Upload.Filter.AnonymizeFilename, :text], "custom-file.png") + {:ok, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file) + assert name == "custom-file.png" + end + + test "it replaces filename on pre-defined text expression", %{upload_file: upload_file} do + Config.put([Upload.Filter.AnonymizeFilename, :text], "custom-file.{extension}") + {:ok, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file) + assert name == "custom-file.jpg" + end + + test "it replaces filename on random text", %{upload_file: upload_file} do + {:ok, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file) + assert <<_::bytes-size(14)>> <> ".jpg" = name + refute name == "an… image.jpg" + end +end From 1f76740e104e6b5f50fbb813b5d7b42499eb3466 Mon Sep 17 00:00:00 2001 From: Eugenij Date: Sun, 30 Jun 2019 09:08:46 +0000 Subject: [PATCH 16/29] Add hashtag filter to user statuses (GET /api/v1/accounts/:id/statuses) --- CHANGELOG.md | 2 ++ .../web/mastodon_api/mastodon_api_controller.ex | 4 ++++ .../mastodon_api/mastodon_api_controller_test.exs | 13 +++++++++++++ 3 files changed, 19 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96473b1b8..663d99ffd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Added - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) +- Mastodon API: Support for the [`tagged` filter](https://github.com/tootsuite/mastodon/pull/9755) in [`GET /api/v1/accounts/:id/statuses`](https://docs.joinmastodon.org/api/rest/accounts/#get-api-v1-accounts-id-statuses) + ### Fixed - Not being able to pin unlisted posts diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 7cdba4cc0..ceb88511b 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -356,6 +356,10 @@ def public_timeline(%{assigns: %{user: user}} = conn, params) do def user_statuses(%{assigns: %{user: reading_user}} = conn, params) do with %User{} = user <- User.get_cached_by_id(params["id"]) do + params = + params + |> Map.put("tag", params["tagged"]) + activities = ActivityPub.fetch_user_activities(user, reading_user, params) conn diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 03f57dbfa..b7487c68c 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1408,6 +1408,19 @@ test "gets a user's statuses without reblogs", %{conn: conn} do assert [%{"id" => id}] = json_response(conn, 200) assert id == to_string(post.id) end + + test "filters user's statuses by a hashtag", %{conn: conn} do + user = insert(:user) + {:ok, post} = CommonAPI.post(user, %{"status" => "#hashtag"}) + {:ok, _post} = CommonAPI.post(user, %{"status" => "hashtag"}) + + conn = + conn + |> get("/api/v1/accounts/#{user.id}/statuses", %{"tagged" => "hashtag"}) + + assert [%{"id" => id}] = json_response(conn, 200) + assert id == to_string(post.id) + end end describe "user relationships" do From 044eb039c22f0cdececee85922b0c194f0944f42 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 30 Jun 2019 13:06:10 +0300 Subject: [PATCH 17/29] OTP Release install docs: Remove --dry-run in cron certbot command --- docs/installation/otp_en.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index fa71f23e1..9b851e395 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -207,7 +207,7 @@ certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ -- # Add it to the daily cron echo '#!/bin/sh -certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ --dry-run --post-hook "systemctl reload nginx" +certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ --post-hook "systemctl reload nginx" ' > /etc/cron.daily/renew-pleroma-cert chmod +x /etc/cron.daily/renew-pleroma-cert @@ -228,7 +228,7 @@ certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ -- # Add it to the daily cron echo '#!/bin/sh -certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ --dry-run --post-hook "rc-service nginx reload" +certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ --post-hook "rc-service nginx reload" ' > /etc/periodic/daily/renew-pleroma-cert chmod +x /etc/periodic/daily/renew-pleroma-cert From 611f6665ac8c9553ceeb3cb731f6a3f0482af829 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Mon, 1 Jul 2019 03:36:42 +0300 Subject: [PATCH 18/29] Update changelog and admin API docs with tags, display_name, avatar additions --- CHANGELOG.md | 2 ++ docs/api/admin_api.md | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 663d99ffd..ae7e4c0ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Admin API: Endpoints for managing reports - Admin API: Endpoints for deleting and changing the scope of individual reported statuses - Admin API: Endpoints to view and change config settings. +- Admin API: Return users' tags when querying reports +- Admin API: Return avatar and display name when querying users - AdminFE: initial release with basic user management accessible at /pleroma/admin/ - Mastodon API: Add chat token to `verify_credentials` response - Mastodon API: Add background image setting to `update_credentials` diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index 4be0ab0f8..74bde3ece 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -38,7 +38,9 @@ Authentication is required and the user must be an admin. "moderator": bool }, "local": bool, - "tags": array + "tags": array, + "avatar": string, + "display_name": string }, ... ] @@ -331,6 +333,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret "pleroma": {}, "sensitive": false }, + "tags": ["force_unlisted"], "statuses_count": 3, "url": "https://pleroma.example.org/users/user", "username": "user" @@ -366,6 +369,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret "pleroma": {}, "sensitive": false }, + "tags": ["force_unlisted"], "statuses_count": 1, "url": "https://pleroma.example.org/users/lain", "username": "lain" From 85c7916c94017c6a3657a3540f1e6c3afcb78225 Mon Sep 17 00:00:00 2001 From: Maksim Date: Mon, 1 Jul 2019 01:08:07 +0000 Subject: [PATCH 19/29] [#1033] Fix database migrations --- priv/repo/migrations/20170320193800_create_user.exs | 2 +- .../repo/migrations/20170321074828_create_activity.exs | 4 ++-- priv/repo/migrations/20170321074832_create_object.exs | 2 +- .../migrations/20170330153447_add_index_to_objects.exs | 2 +- ...15141210_add_unique_index_to_email_and_nickname.exs | 4 ++-- ...170418200143_create_webssub_server_subscription.exs | 2 +- ...0170426154155_create_websub_client_subscription.exs | 2 +- ...d_contraints_to_activities_and_objects_part_two.exs | 8 +++++--- .../20170502083023_add_local_field_to_activities.exs | 2 +- .../20170506222027_add_unique_index_to_apid.exs | 2 +- priv/repo/migrations/20170529093232_longer_bios.exs | 9 ++++++++- .../20170620095947_remove_activities_index.exs | 2 +- .../20170620133028_add_object_activity_index.exs | 2 +- ...170620142420_add_object_activity_index_part_two.exs | 2 +- .../20170701142005_add_actor_index_to_activity.exs | 2 +- .../migrations/20170906120646_add_mastodon_apps.exs | 2 +- .../20170906143140_create_o_auth_authorizations.exs | 2 +- .../migrations/20170906152508_create_o_auth_token.exs | 2 +- .../migrations/20170911123607_create_notifications.exs | 4 ++-- .../20171019141706_create_password_reset_tokens.exs | 2 +- .../20171109091239_add_actor_to_activity.exs | 2 +- .../20171130135819_add_local_index_to_user.exs | 2 +- .../20171212163643_add_recipients_to_activities.exs | 2 +- .../20171212164525_fill_recipients_in_activities.exs | 2 ++ .../20180221210540_make_following_postgres_array.exs | 4 +++- .../20180327174350_drop_local_index_on_activities.exs | 2 +- .../20180327175831_actually_drop_local_index.exs | 2 +- priv/repo/migrations/20180429094642_create_lists.exs | 4 ++-- .../20180516154905_create_user_trigram_index.exs | 2 +- .../20180530123448_add_list_follow_index.exs | 2 +- .../20180612110515_create_user_invite_tokens.exs | 2 +- priv/repo/migrations/20180813003722_create_filters.exs | 6 +++--- ...6_add_recipients_to_and_cc_fields_to_activities.exs | 4 ++-- .../20180829182612_activities_add_to_cc_indices.exs | 4 ++-- ...ove_recipients_to_and_cc_fields_from_activities.exs | 9 ++++++++- .../20180903114437_users_add_is_moderator_index.exs | 2 +- .../20180918182427_create_push_subscriptions.exs | 4 ++-- .../migrations/20181201104428_add_uuid_extension.exs | 4 +++- .../20181201105617_add_uui_ds_to_user_info.exs | 4 +++- .../migrations/20181206125616_add_tags_to_users.exs | 2 +- .../20181218172826_users_and_activities_flake_id.exs | 4 +++- .../20190109152453_add_visibility_function.exs | 4 ++-- .../20190115085500_create_user_fts_index.exs | 2 +- .../20190118074940_fix_user_trigram_index.exs | 4 ++-- .../20190123092341_users_add_is_admin_index.exs | 2 +- .../migrations/20190123125546_create_instances.exs | 6 +++--- priv/repo/migrations/20190123125839_fix_info_ids.exs | 4 +++- ...0190126160540_change_push_subscriptions_varchar.exs | 8 +++++++- .../migrations/20190204200237_add_correct_dm_index.exs | 2 +- .../migrations/20190205114625_create_thread_mutes.exs | 4 ++-- .../migrations/20190315101315_create_registrations.exs | 6 +++--- .../20190325185009_create_notification_id_index.exs | 2 +- .../20190328053912_create_scheduled_activities.exs | 6 +++--- .../20190403131720_add_oauth_token_indexes.exs | 6 +++--- .../20190405160700_add_index_on_subscribers.exs | 2 +- .../migrations/20190408123347_create_conversations.exs | 10 +++++----- ...190410152859_add_participation_updated_at_index.exs | 2 +- ...190411094120_add_index_on_user_info_deactivated.exs | 2 +- .../migrations/20190413082658_create_bookmarks.exs | 4 ++-- .../20190414125034_migrate_old_bookmarks.exs | 10 ++++++++-- .../20190501125843_add_fts_index_to_objects.exs | 2 +- ...20190501133552_add_refresh_token_index_to_token.exs | 2 +- ...190513175809_change_hide_column_in_filter_table.exs | 8 +++++++- .../20190515222404_add_thread_visibility_function.exs | 2 +- priv/repo/migrations/20190518032627_create_config.exs | 4 ++-- ...d_non_followers_fields_to_notification_settings.exs | 4 +++- .../20190603115238_add_index_on_activities_local.exs | 2 +- .../20190603173419_add_tag_index_to_objects.exs | 2 +- .../20190622151019_add_group_key_to_config.exs | 4 ++-- .../20190510135645_add_fts_index_to_objects_two.exs | 10 +++++----- 70 files changed, 151 insertions(+), 103 deletions(-) diff --git a/priv/repo/migrations/20170320193800_create_user.exs b/priv/repo/migrations/20170320193800_create_user.exs index 2d2f7fbf0..089964a26 100644 --- a/priv/repo/migrations/20170320193800_create_user.exs +++ b/priv/repo/migrations/20170320193800_create_user.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreatePleroma.User do use Ecto.Migration def change do - create table(:users) do + create_if_not_exists table(:users) do add :email, :string add :password_hash, :string add :name, :string diff --git a/priv/repo/migrations/20170321074828_create_activity.exs b/priv/repo/migrations/20170321074828_create_activity.exs index 6e875ae43..f5c872721 100644 --- a/priv/repo/migrations/20170321074828_create_activity.exs +++ b/priv/repo/migrations/20170321074828_create_activity.exs @@ -2,13 +2,13 @@ defmodule Pleroma.Repo.Migrations.CreatePleroma.Activity do use Ecto.Migration def change do - create table(:activities) do + create_if_not_exists table(:activities) do add :data, :map timestamps() end - create index(:activities, [:data], using: :gin) + create_if_not_exists index(:activities, [:data], using: :gin) end end diff --git a/priv/repo/migrations/20170321074832_create_object.exs b/priv/repo/migrations/20170321074832_create_object.exs index b8bd49747..b184672ad 100644 --- a/priv/repo/migrations/20170321074832_create_object.exs +++ b/priv/repo/migrations/20170321074832_create_object.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreatePleroma.Object do use Ecto.Migration def change do - create table(:objects) do + create_if_not_exists table(:objects) do add :data, :map timestamps() diff --git a/priv/repo/migrations/20170330153447_add_index_to_objects.exs b/priv/repo/migrations/20170330153447_add_index_to_objects.exs index 30ed61f51..25e308533 100644 --- a/priv/repo/migrations/20170330153447_add_index_to_objects.exs +++ b/priv/repo/migrations/20170330153447_add_index_to_objects.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddIndexToObjects do use Ecto.Migration def change do - create index(:objects, [:data], using: :gin) + create_if_not_exists index(:objects, [:data], using: :gin) end end diff --git a/priv/repo/migrations/20170415141210_add_unique_index_to_email_and_nickname.exs b/priv/repo/migrations/20170415141210_add_unique_index_to_email_and_nickname.exs index 361ca04da..42da88954 100644 --- a/priv/repo/migrations/20170415141210_add_unique_index_to_email_and_nickname.exs +++ b/priv/repo/migrations/20170415141210_add_unique_index_to_email_and_nickname.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.AddUniqueIndexToEmailAndNickname do use Ecto.Migration def change do - create unique_index(:users, [:email]) - create unique_index(:users, [:nickname]) + create_if_not_exists unique_index(:users, [:email]) + create_if_not_exists unique_index(:users, [:nickname]) end end diff --git a/priv/repo/migrations/20170418200143_create_webssub_server_subscription.exs b/priv/repo/migrations/20170418200143_create_webssub_server_subscription.exs index fe2fa2304..243280378 100644 --- a/priv/repo/migrations/20170418200143_create_webssub_server_subscription.exs +++ b/priv/repo/migrations/20170418200143_create_webssub_server_subscription.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateWebsubServerSubscription do use Ecto.Migration def change do - create table(:websub_server_subscriptions) do + create_if_not_exists table(:websub_server_subscriptions) do add :topic, :string add :callback, :string add :secret, :string diff --git a/priv/repo/migrations/20170426154155_create_websub_client_subscription.exs b/priv/repo/migrations/20170426154155_create_websub_client_subscription.exs index 89d3af7ae..4b79d7506 100644 --- a/priv/repo/migrations/20170426154155_create_websub_client_subscription.exs +++ b/priv/repo/migrations/20170426154155_create_websub_client_subscription.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateWebsubClientSubscription do use Ecto.Migration def change do - create table(:websub_client_subscriptions) do + create_if_not_exists table(:websub_client_subscriptions) do add :topic, :string add :secret, :string add :valid_until, :naive_datetime_usec diff --git a/priv/repo/migrations/20170501133231_add_id_contraints_to_activities_and_objects_part_two.exs b/priv/repo/migrations/20170501133231_add_id_contraints_to_activities_and_objects_part_two.exs index 1472b60b4..f5e5cd269 100644 --- a/priv/repo/migrations/20170501133231_add_id_contraints_to_activities_and_objects_part_two.exs +++ b/priv/repo/migrations/20170501133231_add_id_contraints_to_activities_and_objects_part_two.exs @@ -1,10 +1,12 @@ defmodule Pleroma.Repo.Migrations.AddIdContraintsToActivitiesAndObjectsPartTwo do use Ecto.Migration - def change do + def up do drop_if_exists index(:objects, ["(data->>\"id\")"], name: :objects_unique_apid_index) drop_if_exists index(:activities, ["(data->>\"id\")"], name: :activities_unique_apid_index) - create unique_index(:objects, ["(data->>'id')"], name: :objects_unique_apid_index) - create unique_index(:activities, ["(data->>'id')"], name: :activities_unique_apid_index) + create_if_not_exists unique_index(:objects, ["(data->>'id')"], name: :objects_unique_apid_index) + create_if_not_exists unique_index(:activities, ["(data->>'id')"], name: :activities_unique_apid_index) end + + def down, do: :ok end diff --git a/priv/repo/migrations/20170502083023_add_local_field_to_activities.exs b/priv/repo/migrations/20170502083023_add_local_field_to_activities.exs index 088d68f67..cebc11d21 100644 --- a/priv/repo/migrations/20170502083023_add_local_field_to_activities.exs +++ b/priv/repo/migrations/20170502083023_add_local_field_to_activities.exs @@ -6,6 +6,6 @@ def change do add :local, :boolean, default: true end - create index(:activities, [:local]) + create_if_not_exists index(:activities, [:local]) end end diff --git a/priv/repo/migrations/20170506222027_add_unique_index_to_apid.exs b/priv/repo/migrations/20170506222027_add_unique_index_to_apid.exs index 864b5e47d..1b7e33b70 100644 --- a/priv/repo/migrations/20170506222027_add_unique_index_to_apid.exs +++ b/priv/repo/migrations/20170506222027_add_unique_index_to_apid.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddUniqueIndexToAPID do use Ecto.Migration def change do - create unique_index(:users, [:ap_id]) + create_if_not_exists unique_index(:users, [:ap_id]) end end diff --git a/priv/repo/migrations/20170529093232_longer_bios.exs b/priv/repo/migrations/20170529093232_longer_bios.exs index f8c14b117..9188f4bee 100644 --- a/priv/repo/migrations/20170529093232_longer_bios.exs +++ b/priv/repo/migrations/20170529093232_longer_bios.exs @@ -1,9 +1,16 @@ defmodule Pleroma.Repo.Migrations.LongerBios do use Ecto.Migration - def change do + def up do alter table(:users) do modify :bio, :text end end + + def down do + alter table(:users) do + modify :bio, :string + end + end + end diff --git a/priv/repo/migrations/20170620095947_remove_activities_index.exs b/priv/repo/migrations/20170620095947_remove_activities_index.exs index 85feabeeb..e7d41eac4 100644 --- a/priv/repo/migrations/20170620095947_remove_activities_index.exs +++ b/priv/repo/migrations/20170620095947_remove_activities_index.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.RemoveActivitiesIndex do use Ecto.Migration def change do - drop index(:activities, [:data]) + drop_if_exists index(:activities, [:data]) end end diff --git a/priv/repo/migrations/20170620133028_add_object_activity_index.exs b/priv/repo/migrations/20170620133028_add_object_activity_index.exs index df0b8391e..5c312b8f4 100644 --- a/priv/repo/migrations/20170620133028_add_object_activity_index.exs +++ b/priv/repo/migrations/20170620133028_add_object_activity_index.exs @@ -3,6 +3,6 @@ defmodule Pleroma.Repo.Migrations.AddObjectActivityIndex do def change do # This was wrong, now a noop - # create index(:objects, ["(data->'object'->>'id')", "(data->>'type')"], name: :activities_create_objects_index) + # create_if_not_exists index(:objects, ["(data->'object'->>'id')", "(data->>'type')"], name: :activities_create_objects_index) end end diff --git a/priv/repo/migrations/20170620142420_add_object_activity_index_part_two.exs b/priv/repo/migrations/20170620142420_add_object_activity_index_part_two.exs index 8d5a95c45..c95218fad 100644 --- a/priv/repo/migrations/20170620142420_add_object_activity_index_part_two.exs +++ b/priv/repo/migrations/20170620142420_add_object_activity_index_part_two.exs @@ -3,6 +3,6 @@ defmodule Pleroma.Repo.Migrations.AddObjectActivityIndexPartTwo do def change do drop_if_exists index(:objects, ["(data->'object'->>'id')", "(data->>'type')"], name: :activities_create_objects_index) - create index(:activities, ["(data->'object'->>'id')", "(data->>'type')"], name: :activities_create_objects_index) + create_if_not_exists index(:activities, ["(data->'object'->>'id')", "(data->>'type')"], name: :activities_create_objects_index) end end diff --git a/priv/repo/migrations/20170701142005_add_actor_index_to_activity.exs b/priv/repo/migrations/20170701142005_add_actor_index_to_activity.exs index 82c64396f..807fe3728 100644 --- a/priv/repo/migrations/20170701142005_add_actor_index_to_activity.exs +++ b/priv/repo/migrations/20170701142005_add_actor_index_to_activity.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddActorIndexToActivity do use Ecto.Migration def change do - create index(:activities, ["(data->>'actor')", "inserted_at desc"], name: :activities_actor_index) + create_if_not_exists index(:activities, ["(data->>'actor')", "inserted_at desc"], name: :activities_actor_index) end end diff --git a/priv/repo/migrations/20170906120646_add_mastodon_apps.exs b/priv/repo/migrations/20170906120646_add_mastodon_apps.exs index d3dd317dd..ccd5e3fe2 100644 --- a/priv/repo/migrations/20170906120646_add_mastodon_apps.exs +++ b/priv/repo/migrations/20170906120646_add_mastodon_apps.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.AddMastodonApps do use Ecto.Migration def change do - create table(:apps) do + create_if_not_exists table(:apps) do add :client_name, :string add :redirect_uris, :string add :scopes, :string diff --git a/priv/repo/migrations/20170906143140_create_o_auth_authorizations.exs b/priv/repo/migrations/20170906143140_create_o_auth_authorizations.exs index ead1d023e..63b25c537 100644 --- a/priv/repo/migrations/20170906143140_create_o_auth_authorizations.exs +++ b/priv/repo/migrations/20170906143140_create_o_auth_authorizations.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateOAuthAuthorizations do use Ecto.Migration def change do - create table(:oauth_authorizations) do + create_if_not_exists table(:oauth_authorizations) do add :app_id, references(:apps) add :user_id, references(:users) add :token, :string diff --git a/priv/repo/migrations/20170906152508_create_o_auth_token.exs b/priv/repo/migrations/20170906152508_create_o_auth_token.exs index ed56bbf36..08471bbf8 100644 --- a/priv/repo/migrations/20170906152508_create_o_auth_token.exs +++ b/priv/repo/migrations/20170906152508_create_o_auth_token.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateOAuthToken do use Ecto.Migration def change do - create table(:oauth_tokens) do + create_if_not_exists table(:oauth_tokens) do add :app_id, references(:apps) add :user_id, references(:users) add :token, :string diff --git a/priv/repo/migrations/20170911123607_create_notifications.exs b/priv/repo/migrations/20170911123607_create_notifications.exs index 5be809fb8..50de9c5f1 100644 --- a/priv/repo/migrations/20170911123607_create_notifications.exs +++ b/priv/repo/migrations/20170911123607_create_notifications.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateNotifications do use Ecto.Migration def change do - create table(:notifications) do + create_if_not_exists table(:notifications) do add :user_id, references(:users, on_delete: :delete_all) add :activity_id, references(:activities, on_delete: :delete_all) add :seen, :boolean, default: false @@ -10,6 +10,6 @@ def change do timestamps() end - create index(:notifications, [:user_id]) + create_if_not_exists index(:notifications, [:user_id]) end end diff --git a/priv/repo/migrations/20171019141706_create_password_reset_tokens.exs b/priv/repo/migrations/20171019141706_create_password_reset_tokens.exs index 2d9be3aab..dde0f945f 100644 --- a/priv/repo/migrations/20171019141706_create_password_reset_tokens.exs +++ b/priv/repo/migrations/20171019141706_create_password_reset_tokens.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreatePasswordResetTokens do use Ecto.Migration def change do - create table(:password_reset_tokens) do + create_if_not_exists table(:password_reset_tokens) do add :token, :string add :user_id, references(:users) add :used, :boolean, default: false diff --git a/priv/repo/migrations/20171109091239_add_actor_to_activity.exs b/priv/repo/migrations/20171109091239_add_actor_to_activity.exs index 2d8b60a91..fb5f80c98 100644 --- a/priv/repo/migrations/20171109091239_add_actor_to_activity.exs +++ b/priv/repo/migrations/20171109091239_add_actor_to_activity.exs @@ -12,7 +12,7 @@ def up do end def down do - drop index(:activities, [:actor, "id DESC NULLS LAST"]) + drop_if_exists index(:activities, [:actor, "id DESC NULLS LAST"]) alter table(:activities) do remove :actor end diff --git a/priv/repo/migrations/20171130135819_add_local_index_to_user.exs b/priv/repo/migrations/20171130135819_add_local_index_to_user.exs index 25716be21..3438bbbc4 100644 --- a/priv/repo/migrations/20171130135819_add_local_index_to_user.exs +++ b/priv/repo/migrations/20171130135819_add_local_index_to_user.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddLocalIndexToUser do use Ecto.Migration def change do - create index(:users, [:local]) + create_if_not_exists index(:users, [:local]) end end diff --git a/priv/repo/migrations/20171212163643_add_recipients_to_activities.exs b/priv/repo/migrations/20171212163643_add_recipients_to_activities.exs index 7bce78108..4520b398e 100644 --- a/priv/repo/migrations/20171212163643_add_recipients_to_activities.exs +++ b/priv/repo/migrations/20171212163643_add_recipients_to_activities.exs @@ -6,6 +6,6 @@ def change do add :recipients, {:array, :string} end - create index(:activities, [:recipients], using: :gin) + create_if_not_exists index(:activities, [:recipients], using: :gin) end end diff --git a/priv/repo/migrations/20171212164525_fill_recipients_in_activities.exs b/priv/repo/migrations/20171212164525_fill_recipients_in_activities.exs index 1fcc0dabb..87de64ca5 100644 --- a/priv/repo/migrations/20171212164525_fill_recipients_in_activities.exs +++ b/priv/repo/migrations/20171212164525_fill_recipients_in_activities.exs @@ -18,4 +18,6 @@ def up do end) end end + + def down, do: :ok end diff --git a/priv/repo/migrations/20180221210540_make_following_postgres_array.exs b/priv/repo/migrations/20180221210540_make_following_postgres_array.exs index 98ca7d9d7..5a8f8f669 100644 --- a/priv/repo/migrations/20180221210540_make_following_postgres_array.exs +++ b/priv/repo/migrations/20180221210540_make_following_postgres_array.exs @@ -1,7 +1,7 @@ defmodule Pleroma.Repo.Migrations.MakeFollowingPostgresArray do use Ecto.Migration - def change do + def up do alter table(:users) do add :following_temp, {:array, :string} end @@ -15,4 +15,6 @@ def change do end rename table(:users), :following_temp, to: :following end + + def down, do: :ok end diff --git a/priv/repo/migrations/20180327174350_drop_local_index_on_activities.exs b/priv/repo/migrations/20180327174350_drop_local_index_on_activities.exs index 70f2c9fe6..35c4ce62f 100644 --- a/priv/repo/migrations/20180327174350_drop_local_index_on_activities.exs +++ b/priv/repo/migrations/20180327174350_drop_local_index_on_activities.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.DropLocalIndexOnActivities do use Ecto.Migration def change do - drop index(:users, [:local]) + drop_if_exists index(:users, [:local]) end end diff --git a/priv/repo/migrations/20180327175831_actually_drop_local_index.exs b/priv/repo/migrations/20180327175831_actually_drop_local_index.exs index 2da65689c..7556336ed 100644 --- a/priv/repo/migrations/20180327175831_actually_drop_local_index.exs +++ b/priv/repo/migrations/20180327175831_actually_drop_local_index.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.ActuallyDropLocalIndex do use Ecto.Migration def change do - create index(:users, [:local]) + create_if_not_exists index(:users, [:local]) drop_if_exists index("activities", :local) end end diff --git a/priv/repo/migrations/20180429094642_create_lists.exs b/priv/repo/migrations/20180429094642_create_lists.exs index 64c62250e..9d3ce50b3 100644 --- a/priv/repo/migrations/20180429094642_create_lists.exs +++ b/priv/repo/migrations/20180429094642_create_lists.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateLists do use Ecto.Migration def change do - create table(:lists) do + create_if_not_exists table(:lists) do add :user_id, references(:users, on_delete: :delete_all) add :title, :string add :following, {:array, :string} @@ -10,6 +10,6 @@ def change do timestamps() end - create index(:lists, [:user_id]) + create_if_not_exists index(:lists, [:user_id]) end end diff --git a/priv/repo/migrations/20180516154905_create_user_trigram_index.exs b/priv/repo/migrations/20180516154905_create_user_trigram_index.exs index abfa4b3cc..58622a87e 100644 --- a/priv/repo/migrations/20180516154905_create_user_trigram_index.exs +++ b/priv/repo/migrations/20180516154905_create_user_trigram_index.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.CreateUserTrigramIndex do use Ecto.Migration def change do - create index(:users, ["(nickname || name) gist_trgm_ops"], name: :users_trigram_index, using: :gist) + create_if_not_exists index(:users, ["(nickname || name) gist_trgm_ops"], name: :users_trigram_index, using: :gist) end end diff --git a/priv/repo/migrations/20180530123448_add_list_follow_index.exs b/priv/repo/migrations/20180530123448_add_list_follow_index.exs index d6603e916..86b8de30a 100644 --- a/priv/repo/migrations/20180530123448_add_list_follow_index.exs +++ b/priv/repo/migrations/20180530123448_add_list_follow_index.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddListFollowIndex do use Ecto.Migration def change do - create index(:lists, [:following]) + create_if_not_exists index(:lists, [:following]) end end diff --git a/priv/repo/migrations/20180612110515_create_user_invite_tokens.exs b/priv/repo/migrations/20180612110515_create_user_invite_tokens.exs index d0a1cf784..faee379f0 100644 --- a/priv/repo/migrations/20180612110515_create_user_invite_tokens.exs +++ b/priv/repo/migrations/20180612110515_create_user_invite_tokens.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateUserInviteTokens do use Ecto.Migration def change do - create table(:user_invite_tokens) do + create_if_not_exists table(:user_invite_tokens) do add :token, :string add :used, :boolean, default: false diff --git a/priv/repo/migrations/20180813003722_create_filters.exs b/priv/repo/migrations/20180813003722_create_filters.exs index 8e7129f34..541cf46a1 100644 --- a/priv/repo/migrations/20180813003722_create_filters.exs +++ b/priv/repo/migrations/20180813003722_create_filters.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateFilters do use Ecto.Migration def change do - create table(:filters) do + create_if_not_exists table(:filters) do add :user_id, references(:users, on_delete: :delete_all) add :filter_id, :integer add :hide, :boolean @@ -14,7 +14,7 @@ def change do timestamps() end - create index(:filters, [:user_id]) - create index(:filters, [:phrase], where: "hide = true", name: :hided_phrases_index) + create_if_not_exists index(:filters, [:user_id]) + create_if_not_exists index(:filters, [:phrase], where: "hide = true", name: :hided_phrases_index) end end diff --git a/priv/repo/migrations/20180829082446_add_recipients_to_and_cc_fields_to_activities.exs b/priv/repo/migrations/20180829082446_add_recipients_to_and_cc_fields_to_activities.exs index 96af412f0..af9d521c0 100644 --- a/priv/repo/migrations/20180829082446_add_recipients_to_and_cc_fields_to_activities.exs +++ b/priv/repo/migrations/20180829082446_add_recipients_to_and_cc_fields_to_activities.exs @@ -7,7 +7,7 @@ def change do add :recipients_cc, {:array, :string} end - create index(:activities, [:recipients_to], using: :gin) - create index(:activities, [:recipients_cc], using: :gin) + create_if_not_exists index(:activities, [:recipients_to], using: :gin) + create_if_not_exists index(:activities, [:recipients_cc], using: :gin) end end diff --git a/priv/repo/migrations/20180829182612_activities_add_to_cc_indices.exs b/priv/repo/migrations/20180829182612_activities_add_to_cc_indices.exs index f6c622e3e..9d31f6779 100644 --- a/priv/repo/migrations/20180829182612_activities_add_to_cc_indices.exs +++ b/priv/repo/migrations/20180829182612_activities_add_to_cc_indices.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.ActivitiesAddToCcIndices do use Ecto.Migration def change do - create index(:activities, ["(data->'to')"], name: :activities_to_index, using: :gin) - create index(:activities, ["(data->'cc')"], name: :activities_cc_index, using: :gin) + create_if_not_exists index(:activities, ["(data->'to')"], name: :activities_to_index, using: :gin) + create_if_not_exists index(:activities, ["(data->'cc')"], name: :activities_cc_index, using: :gin) end end diff --git a/priv/repo/migrations/20180829183529_remove_recipients_to_and_cc_fields_from_activities.exs b/priv/repo/migrations/20180829183529_remove_recipients_to_and_cc_fields_from_activities.exs index ed4f5af30..017ef161f 100644 --- a/priv/repo/migrations/20180829183529_remove_recipients_to_and_cc_fields_from_activities.exs +++ b/priv/repo/migrations/20180829183529_remove_recipients_to_and_cc_fields_from_activities.exs @@ -1,10 +1,17 @@ defmodule Pleroma.Repo.Migrations.RemoveRecipientsToAndCcFieldsFromActivities do use Ecto.Migration - def change do + def up do alter table(:activities) do remove :recipients_to remove :recipients_cc end end + + def down do + alter table(:activities) do + add :recipients_to, {:array, :string} + add :recipients_cc, {:array, :string} + end + end end diff --git a/priv/repo/migrations/20180903114437_users_add_is_moderator_index.exs b/priv/repo/migrations/20180903114437_users_add_is_moderator_index.exs index ba6b90ea9..adce28bdf 100644 --- a/priv/repo/migrations/20180903114437_users_add_is_moderator_index.exs +++ b/priv/repo/migrations/20180903114437_users_add_is_moderator_index.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.UsersAddIsModeratorIndex do use Ecto.Migration def change do - create index(:users, ["(info->'is_moderator')"], name: :users_is_moderator_index, using: :gin) + create_if_not_exists index(:users, ["(info->'is_moderator')"], name: :users_is_moderator_index, using: :gin) end end diff --git a/priv/repo/migrations/20180918182427_create_push_subscriptions.exs b/priv/repo/migrations/20180918182427_create_push_subscriptions.exs index 0cc7afa54..36bdf322a 100644 --- a/priv/repo/migrations/20180918182427_create_push_subscriptions.exs +++ b/priv/repo/migrations/20180918182427_create_push_subscriptions.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreatePushSubscriptions do use Ecto.Migration def change do - create table("push_subscriptions") do + create_if_not_exists table("push_subscriptions") do add :user_id, references("users", on_delete: :delete_all) add :token_id, references("oauth_tokens", on_delete: :delete_all) add :endpoint, :string @@ -13,6 +13,6 @@ def change do timestamps() end - create index("push_subscriptions", [:user_id, :token_id], unique: true) + create_if_not_exists index("push_subscriptions", [:user_id, :token_id], unique: true) end end diff --git a/priv/repo/migrations/20181201104428_add_uuid_extension.exs b/priv/repo/migrations/20181201104428_add_uuid_extension.exs index 2509e558d..99fcb957c 100644 --- a/priv/repo/migrations/20181201104428_add_uuid_extension.exs +++ b/priv/repo/migrations/20181201104428_add_uuid_extension.exs @@ -1,7 +1,9 @@ defmodule Pleroma.Repo.Migrations.AddUUIDExtension do use Ecto.Migration - def change do + def up do execute("create extension if not exists \"uuid-ossp\"") end + + def down, do: :ok end diff --git a/priv/repo/migrations/20181201105617_add_uui_ds_to_user_info.exs b/priv/repo/migrations/20181201105617_add_uui_ds_to_user_info.exs index 9571a1e4d..964383668 100644 --- a/priv/repo/migrations/20181201105617_add_uui_ds_to_user_info.exs +++ b/priv/repo/migrations/20181201105617_add_uui_ds_to_user_info.exs @@ -1,7 +1,9 @@ defmodule Pleroma.Repo.Migrations.AddUUIDsToUserInfo do use Ecto.Migration - def change do + def up do execute("update users set info = jsonb_set(info, '{\"id\"}', to_jsonb(uuid_generate_v4()))") end + + def down, do: :ok end diff --git a/priv/repo/migrations/20181206125616_add_tags_to_users.exs b/priv/repo/migrations/20181206125616_add_tags_to_users.exs index 1502f63b6..7d42a0fba 100644 --- a/priv/repo/migrations/20181206125616_add_tags_to_users.exs +++ b/priv/repo/migrations/20181206125616_add_tags_to_users.exs @@ -6,6 +6,6 @@ def change do add :tags, {:array, :string} end - create index(:users, [:tags], using: :gin) + create_if_not_exists index(:users, [:tags], using: :gin) end end diff --git a/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs b/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs index 47d2d02da..a5b4c543d 100644 --- a/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs +++ b/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs @@ -12,7 +12,7 @@ defmodule Pleroma.Repo.Migrations.UsersAndActivitiesFlakeId do # 4- update relation pkeys with the new ids # 5- rename the temporary column to id # 6- re-create the constraints - def change do + def up do # Old serial int ids are transformed to 128bits with extra padding. # The application (in `Pleroma.FlakeId`) handles theses IDs properly as integers; to keep compatibility # with previously issued ids. @@ -75,6 +75,8 @@ def change do stop_clippy_heartbeats(clippy) end + def down, do: :ok + defp start_clippy_heartbeats() do count = from(a in "activities", select: count(a.id)) |> Repo.one! diff --git a/priv/repo/migrations/20190109152453_add_visibility_function.exs b/priv/repo/migrations/20190109152453_add_visibility_function.exs index 3aadabcd7..b6a4e752b 100644 --- a/priv/repo/migrations/20190109152453_add_visibility_function.exs +++ b/priv/repo/migrations/20190109152453_add_visibility_function.exs @@ -37,12 +37,12 @@ def up do end def down do - drop( + drop_if_exists( index(:activities, ["activity_visibility(actor, recipients, data)"], name: :activities_visibility_index ) ) - execute("drop function activity_visibility(actor varchar, recipients varchar[], data jsonb)") + execute("drop function if exists activity_visibility(actor varchar, recipients varchar[], data jsonb)") end end diff --git a/priv/repo/migrations/20190115085500_create_user_fts_index.exs b/priv/repo/migrations/20190115085500_create_user_fts_index.exs index 499d67113..cff975318 100644 --- a/priv/repo/migrations/20190115085500_create_user_fts_index.exs +++ b/priv/repo/migrations/20190115085500_create_user_fts_index.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateUserFtsIndex do use Ecto.Migration def change do - create index( + create_if_not_exists index( :users, [ """ diff --git a/priv/repo/migrations/20190118074940_fix_user_trigram_index.exs b/priv/repo/migrations/20190118074940_fix_user_trigram_index.exs index b4e8c984c..b0d272802 100644 --- a/priv/repo/migrations/20190118074940_fix_user_trigram_index.exs +++ b/priv/repo/migrations/20190118074940_fix_user_trigram_index.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Repo.Migrations.FixUserTrigramIndex do def up do drop_if_exists(index(:users, [], name: :users_trigram_index)) - create( + create_if_not_exists( index(:users, ["(trim(nickname || ' ' || coalesce(name, ''))) gist_trgm_ops"], name: :users_trigram_index, using: :gist @@ -15,7 +15,7 @@ def up do def down do drop_if_exists(index(:users, [], name: :users_trigram_index)) - create( + create_if_not_exists( index(:users, ["(nickname || name) gist_trgm_ops"], name: :users_trigram_index, using: :gist) ) end diff --git a/priv/repo/migrations/20190123092341_users_add_is_admin_index.exs b/priv/repo/migrations/20190123092341_users_add_is_admin_index.exs index ba6ff78b5..25f248c59 100644 --- a/priv/repo/migrations/20190123092341_users_add_is_admin_index.exs +++ b/priv/repo/migrations/20190123092341_users_add_is_admin_index.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.UsersAddIsAdminIndex do use Ecto.Migration def change do - create(index(:users, ["(info->'is_admin')"], name: :users_is_admin_index, using: :gin)) + create_if_not_exists(index(:users, ["(info->'is_admin')"], name: :users_is_admin_index, using: :gin)) end end diff --git a/priv/repo/migrations/20190123125546_create_instances.exs b/priv/repo/migrations/20190123125546_create_instances.exs index 3d23b343e..a9b356bc3 100644 --- a/priv/repo/migrations/20190123125546_create_instances.exs +++ b/priv/repo/migrations/20190123125546_create_instances.exs @@ -2,14 +2,14 @@ defmodule Pleroma.Repo.Migrations.CreateInstances do use Ecto.Migration def change do - create table(:instances) do + create_if_not_exists table(:instances) do add :host, :string add :unreachable_since, :naive_datetime_usec timestamps() end - create unique_index(:instances, [:host]) - create index(:instances, [:unreachable_since]) + create_if_not_exists unique_index(:instances, [:host]) + create_if_not_exists index(:instances, [:unreachable_since]) end end diff --git a/priv/repo/migrations/20190123125839_fix_info_ids.exs b/priv/repo/migrations/20190123125839_fix_info_ids.exs index 2b4c2b5a9..5b263b3b3 100644 --- a/priv/repo/migrations/20190123125839_fix_info_ids.exs +++ b/priv/repo/migrations/20190123125839_fix_info_ids.exs @@ -1,9 +1,11 @@ defmodule Pleroma.Repo.Migrations.FixInfoIds do use Ecto.Migration - def change do + def up do execute( "update users set info = jsonb_set(info, '{id}', to_jsonb(uuid_generate_v4())) where info->'id' is null;" ) end + + def down, do: :ok end diff --git a/priv/repo/migrations/20190126160540_change_push_subscriptions_varchar.exs b/priv/repo/migrations/20190126160540_change_push_subscriptions_varchar.exs index 337fed156..8e1c0e630 100644 --- a/priv/repo/migrations/20190126160540_change_push_subscriptions_varchar.exs +++ b/priv/repo/migrations/20190126160540_change_push_subscriptions_varchar.exs @@ -1,9 +1,15 @@ defmodule Pleroma.Repo.Migrations.ChangePushSubscriptionsVarchar do use Ecto.Migration - def change do + def up do alter table(:push_subscriptions) do modify(:endpoint, :varchar) end end + + def down do + alter table(:push_subscriptions) do + modify(:endpoint, :string) + end + end end diff --git a/priv/repo/migrations/20190204200237_add_correct_dm_index.exs b/priv/repo/migrations/20190204200237_add_correct_dm_index.exs index 558732cd2..1facb37b8 100644 --- a/priv/repo/migrations/20190204200237_add_correct_dm_index.exs +++ b/priv/repo/migrations/20190204200237_add_correct_dm_index.exs @@ -19,7 +19,7 @@ def up do end def down do - drop( + drop_if_exists( index(:activities, ["activity_visibility(actor, recipients, data)", "id DESC"], name: :activities_visibility_index, concurrently: true, diff --git a/priv/repo/migrations/20190205114625_create_thread_mutes.exs b/priv/repo/migrations/20190205114625_create_thread_mutes.exs index 8e9eccbae..7e44db121 100644 --- a/priv/repo/migrations/20190205114625_create_thread_mutes.exs +++ b/priv/repo/migrations/20190205114625_create_thread_mutes.exs @@ -2,11 +2,11 @@ defmodule Pleroma.Repo.Migrations.CreateThreadMutes do use Ecto.Migration def change do - create table(:thread_mutes) do + create_if_not_exists table(:thread_mutes) do add :user_id, references(:users, type: :uuid, on_delete: :delete_all) add :context, :string end - create unique_index(:thread_mutes, [:user_id, :context], name: :unique_index) + create_if_not_exists unique_index(:thread_mutes, [:user_id, :context], name: :unique_index) end end diff --git a/priv/repo/migrations/20190315101315_create_registrations.exs b/priv/repo/migrations/20190315101315_create_registrations.exs index 6b28cbdd3..34a390a93 100644 --- a/priv/repo/migrations/20190315101315_create_registrations.exs +++ b/priv/repo/migrations/20190315101315_create_registrations.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateRegistrations do use Ecto.Migration def change do - create table(:registrations, primary_key: false) do + create_if_not_exists table(:registrations, primary_key: false) do add :id, :uuid, primary_key: true add :user_id, references(:users, type: :uuid, on_delete: :delete_all) add :provider, :string @@ -12,7 +12,7 @@ def change do timestamps() end - create unique_index(:registrations, [:provider, :uid]) - create unique_index(:registrations, [:user_id, :provider, :uid]) + create_if_not_exists unique_index(:registrations, [:provider, :uid]) + create_if_not_exists unique_index(:registrations, [:user_id, :provider, :uid]) end end diff --git a/priv/repo/migrations/20190325185009_create_notification_id_index.exs b/priv/repo/migrations/20190325185009_create_notification_id_index.exs index a6ab38d02..01cb30559 100644 --- a/priv/repo/migrations/20190325185009_create_notification_id_index.exs +++ b/priv/repo/migrations/20190325185009_create_notification_id_index.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.CreateNotificationIdIndex do use Ecto.Migration def change do - create index(:notifications, ["id desc nulls last"]) + create_if_not_exists index(:notifications, ["id desc nulls last"]) end end diff --git a/priv/repo/migrations/20190328053912_create_scheduled_activities.exs b/priv/repo/migrations/20190328053912_create_scheduled_activities.exs index dd737e25a..8a01bbdb1 100644 --- a/priv/repo/migrations/20190328053912_create_scheduled_activities.exs +++ b/priv/repo/migrations/20190328053912_create_scheduled_activities.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateScheduledActivities do use Ecto.Migration def change do - create table(:scheduled_activities) do + create_if_not_exists table(:scheduled_activities) do add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) add(:scheduled_at, :naive_datetime, null: false) add(:params, :map, null: false) @@ -10,7 +10,7 @@ def change do timestamps() end - create(index(:scheduled_activities, [:scheduled_at])) - create(index(:scheduled_activities, [:user_id])) + create_if_not_exists(index(:scheduled_activities, [:scheduled_at])) + create_if_not_exists(index(:scheduled_activities, [:user_id])) end end diff --git a/priv/repo/migrations/20190403131720_add_oauth_token_indexes.exs b/priv/repo/migrations/20190403131720_add_oauth_token_indexes.exs index ebcd29389..ab1bf2165 100644 --- a/priv/repo/migrations/20190403131720_add_oauth_token_indexes.exs +++ b/priv/repo/migrations/20190403131720_add_oauth_token_indexes.exs @@ -2,8 +2,8 @@ defmodule Pleroma.Repo.Migrations.AddOauthTokenIndexes do use Ecto.Migration def change do - create(unique_index(:oauth_tokens, [:token])) - create(index(:oauth_tokens, [:app_id])) - create(index(:oauth_tokens, [:user_id])) + create_if_not_exists(unique_index(:oauth_tokens, [:token])) + create_if_not_exists(index(:oauth_tokens, [:app_id])) + create_if_not_exists(index(:oauth_tokens, [:user_id])) end end diff --git a/priv/repo/migrations/20190405160700_add_index_on_subscribers.exs b/priv/repo/migrations/20190405160700_add_index_on_subscribers.exs index 232f75c92..460dafb1b 100644 --- a/priv/repo/migrations/20190405160700_add_index_on_subscribers.exs +++ b/priv/repo/migrations/20190405160700_add_index_on_subscribers.exs @@ -1,6 +1,6 @@ defmodule Pleroma.Repo.Migrations.AddIndexOnSubscribers do use Ecto.Migration - + @disable_ddl_transaction true def change do create index(:users, ["(info->'subscribers')"], name: :users_subscribers_index, using: :gin, concurrently: true) diff --git a/priv/repo/migrations/20190408123347_create_conversations.exs b/priv/repo/migrations/20190408123347_create_conversations.exs index 0e0af30ae..7b7d89da7 100644 --- a/priv/repo/migrations/20190408123347_create_conversations.exs +++ b/priv/repo/migrations/20190408123347_create_conversations.exs @@ -6,12 +6,12 @@ defmodule Pleroma.Repo.Migrations.CreateConversations do use Ecto.Migration def change do - create table(:conversations) do + create_if_not_exists table(:conversations) do add(:ap_id, :string, null: false) timestamps() end - create table(:conversation_participations) do + create_if_not_exists table(:conversation_participations) do add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) add(:conversation_id, references(:conversations, on_delete: :delete_all)) add(:read, :boolean, default: false) @@ -19,8 +19,8 @@ def change do timestamps() end - create index(:conversation_participations, [:conversation_id]) - create unique_index(:conversation_participations, [:user_id, :conversation_id]) - create unique_index(:conversations, [:ap_id]) + create_if_not_exists index(:conversation_participations, [:conversation_id]) + create_if_not_exists unique_index(:conversation_participations, [:user_id, :conversation_id]) + create_if_not_exists unique_index(:conversations, [:ap_id]) end end diff --git a/priv/repo/migrations/20190410152859_add_participation_updated_at_index.exs b/priv/repo/migrations/20190410152859_add_participation_updated_at_index.exs index 1ce688c52..b5ca2fc0f 100644 --- a/priv/repo/migrations/20190410152859_add_participation_updated_at_index.exs +++ b/priv/repo/migrations/20190410152859_add_participation_updated_at_index.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddParticipationUpdatedAtIndex do use Ecto.Migration def change do - create index(:conversation_participations, ["updated_at desc"]) + create_if_not_exists index(:conversation_participations, ["updated_at desc"]) end end diff --git a/priv/repo/migrations/20190411094120_add_index_on_user_info_deactivated.exs b/priv/repo/migrations/20190411094120_add_index_on_user_info_deactivated.exs index d701dcecc..c19427f12 100644 --- a/priv/repo/migrations/20190411094120_add_index_on_user_info_deactivated.exs +++ b/priv/repo/migrations/20190411094120_add_index_on_user_info_deactivated.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddIndexOnUserInfoDeactivated do use Ecto.Migration def change do - create(index(:users, ["(info->'deactivated')"], name: :users_deactivated_index, using: :gin)) + create_if_not_exists(index(:users, ["(info->'deactivated')"], name: :users_deactivated_index, using: :gin)) end end diff --git a/priv/repo/migrations/20190413082658_create_bookmarks.exs b/priv/repo/migrations/20190413082658_create_bookmarks.exs index 38b108158..cfd025fc5 100644 --- a/priv/repo/migrations/20190413082658_create_bookmarks.exs +++ b/priv/repo/migrations/20190413082658_create_bookmarks.exs @@ -2,13 +2,13 @@ defmodule Pleroma.Repo.Migrations.CreateBookmarks do use Ecto.Migration def change do - create table(:bookmarks) do + create_if_not_exists table(:bookmarks) do add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) add(:activity_id, references(:activities, type: :uuid, on_delete: :delete_all)) timestamps() end - create(unique_index(:bookmarks, [:user_id, :activity_id])) + create_if_not_exists(unique_index(:bookmarks, [:user_id, :activity_id])) end end diff --git a/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs b/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs index 134b7c6f7..ce4590954 100644 --- a/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs +++ b/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs @@ -6,7 +6,7 @@ defmodule Pleroma.Repo.Migrations.MigrateOldBookmarks do alias Pleroma.User alias Pleroma.Repo - def change do + def up do query = from(u in User, where: u.local == true, @@ -18,7 +18,7 @@ def change do |> Enum.each(fn %{id: user_id, bookmarks: bookmarks} -> Enum.each(bookmarks, fn ap_id -> activity = Activity.get_create_by_object_ap_id(ap_id) - unless is_nil(activity), do: {:ok, _} = Bookmark.create(user_id, activity.id) + unless is_nil(activity), do: {:ok, _} = Bookmark.create(user_id, activity.id) end) end) @@ -26,4 +26,10 @@ def change do remove(:bookmarks) end end + + def down do + alter table(:users) do + add :bookmarks, {:array, :string}, null: false, default: [] + end + end end diff --git a/priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs b/priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs index 9b274695e..d4de51691 100644 --- a/priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs +++ b/priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs @@ -3,6 +3,6 @@ defmodule Pleroma.Repo.Migrations.AddFTSIndexToObjects do def change do drop_if_exists index(:activities, ["(to_tsvector('english', data->'object'->>'content'))"], using: :gin, name: :activities_fts) - create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) + create_if_not_exists index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) end end diff --git a/priv/repo/migrations/20190501133552_add_refresh_token_index_to_token.exs b/priv/repo/migrations/20190501133552_add_refresh_token_index_to_token.exs index 449f2a3d4..44a266985 100644 --- a/priv/repo/migrations/20190501133552_add_refresh_token_index_to_token.exs +++ b/priv/repo/migrations/20190501133552_add_refresh_token_index_to_token.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddRefreshTokenIndexToToken do use Ecto.Migration def change do - create(unique_index(:oauth_tokens, [:refresh_token])) + create_if_not_exists(unique_index(:oauth_tokens, [:refresh_token])) end end diff --git a/priv/repo/migrations/20190513175809_change_hide_column_in_filter_table.exs b/priv/repo/migrations/20190513175809_change_hide_column_in_filter_table.exs index 2ffb88cc9..246b70cfb 100644 --- a/priv/repo/migrations/20190513175809_change_hide_column_in_filter_table.exs +++ b/priv/repo/migrations/20190513175809_change_hide_column_in_filter_table.exs @@ -1,9 +1,15 @@ defmodule Pleroma.Repo.Migrations.ChangeHideColumnInFilterTable do use Ecto.Migration - def change do + def up do alter table(:filters) do modify :hide, :boolean, default: false end end + + def down do + alter table(:filters) do + modify :hide, :boolean + end + end end diff --git a/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs b/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs index dc9abc998..5184c0c48 100644 --- a/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs +++ b/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs @@ -68,6 +68,6 @@ def up do end def down do - execute("drop function thread_visibility(actor varchar, activity_id varchar)") + execute("drop function if exists thread_visibility(actor varchar, activity_id varchar)") end end diff --git a/priv/repo/migrations/20190518032627_create_config.exs b/priv/repo/migrations/20190518032627_create_config.exs index 1e4e3c689..cb65f3939 100644 --- a/priv/repo/migrations/20190518032627_create_config.exs +++ b/priv/repo/migrations/20190518032627_create_config.exs @@ -2,12 +2,12 @@ defmodule Pleroma.Repo.Migrations.CreateConfig do use Ecto.Migration def change do - create table(:config) do + create_if_not_exists table(:config) do add(:key, :string) add(:value, :binary) timestamps() end - create(unique_index(:config, :key)) + create_if_not_exists(unique_index(:config, :key)) end end diff --git a/priv/repo/migrations/20190525071417_add_non_follows_and_non_followers_fields_to_notification_settings.exs b/priv/repo/migrations/20190525071417_add_non_follows_and_non_followers_fields_to_notification_settings.exs index a88b0ea61..505f0acb2 100644 --- a/priv/repo/migrations/20190525071417_add_non_follows_and_non_followers_fields_to_notification_settings.exs +++ b/priv/repo/migrations/20190525071417_add_non_follows_and_non_followers_fields_to_notification_settings.exs @@ -1,10 +1,12 @@ defmodule Pleroma.Repo.Migrations.AddNonFollowsAndNonFollowersFieldsToNotificationSettings do use Ecto.Migration - def change do + def up do execute(""" update users set info = jsonb_set(info, '{notification_settings}', '{"local": true, "remote": true, "follows": true, "followers": true, "non_follows": true, "non_followers": true}') where local=true """) end + + def down, do: :ok end diff --git a/priv/repo/migrations/20190603115238_add_index_on_activities_local.exs b/priv/repo/migrations/20190603115238_add_index_on_activities_local.exs index 89daa9705..7e6643497 100644 --- a/priv/repo/migrations/20190603115238_add_index_on_activities_local.exs +++ b/priv/repo/migrations/20190603115238_add_index_on_activities_local.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddIndexOnActivitiesLocal do use Ecto.Migration def change do - create(index("activities", [:local])) + create_if_not_exists(index("activities", [:local])) end end diff --git a/priv/repo/migrations/20190603173419_add_tag_index_to_objects.exs b/priv/repo/migrations/20190603173419_add_tag_index_to_objects.exs index c915a0213..93d57a249 100644 --- a/priv/repo/migrations/20190603173419_add_tag_index_to_objects.exs +++ b/priv/repo/migrations/20190603173419_add_tag_index_to_objects.exs @@ -3,6 +3,6 @@ defmodule Pleroma.Repo.Migrations.AddTagIndexToObjects do def change do drop_if_exists index(:activities, ["(data #> '{\"object\",\"tag\"}')"], using: :gin, name: :activities_tags) - create index(:objects, ["(data->'tag')"], using: :gin, name: :objects_tags) + create_if_not_exists index(:objects, ["(data->'tag')"], using: :gin, name: :objects_tags) end end diff --git a/priv/repo/migrations/20190622151019_add_group_key_to_config.exs b/priv/repo/migrations/20190622151019_add_group_key_to_config.exs index d7a3785d0..867dafaf3 100644 --- a/priv/repo/migrations/20190622151019_add_group_key_to_config.exs +++ b/priv/repo/migrations/20190622151019_add_group_key_to_config.exs @@ -6,7 +6,7 @@ def change do add(:group, :string) end - drop(unique_index("config", :key)) - create(unique_index("config", [:group, :key])) + drop_if_exists(unique_index("config", :key)) + create_if_not_exists(unique_index("config", [:group, :key])) end end diff --git a/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs index b6a24441a..6227769dc 100644 --- a/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs +++ b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs @@ -14,7 +14,7 @@ def up do return new; end $$ LANGUAGE plpgsql") - execute("create index objects_fts on objects using RUM (fts_content rum_tsvector_addon_ops, inserted_at) with (attach = 'inserted_at', to = 'fts_content');") + execute("create index if not exists objects_fts on objects using RUM (fts_content rum_tsvector_addon_ops, inserted_at) with (attach = 'inserted_at', to = 'fts_content');") execute("CREATE TRIGGER tsvectorupdate BEFORE INSERT OR UPDATE ON objects FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()") @@ -23,12 +23,12 @@ def up do end def down do - execute "drop index objects_fts" - execute "drop trigger tsvectorupdate on objects" - execute "drop function objects_fts_update()" + execute "drop index if exists objects_fts" + execute "drop trigger if exists tsvectorupdate on objects" + execute "drop function if exists objects_fts_update()" alter table(:objects) do remove(:fts_content, :tsvector) end - create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) + create_if_not_exists index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) end end From 0b3561ae72ea1686f940366e41d6bd1a2262d665 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Mon, 1 Jul 2019 14:04:24 +0300 Subject: [PATCH 20/29] Move changelog entries --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae7e4c0ed..8a0dad453 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) - Mastodon API: Support for the [`tagged` filter](https://github.com/tootsuite/mastodon/pull/9755) in [`GET /api/v1/accounts/:id/statuses`](https://docs.joinmastodon.org/api/rest/accounts/#get-api-v1-accounts-id-statuses) +- Admin API: Return users' tags when querying reports +- Admin API: Return avatar and display name when querying users ### Fixed - Not being able to pin unlisted posts @@ -56,8 +58,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Admin API: Endpoints for managing reports - Admin API: Endpoints for deleting and changing the scope of individual reported statuses - Admin API: Endpoints to view and change config settings. -- Admin API: Return users' tags when querying reports -- Admin API: Return avatar and display name when querying users - AdminFE: initial release with basic user management accessible at /pleroma/admin/ - Mastodon API: Add chat token to `verify_credentials` response - Mastodon API: Add background image setting to `update_credentials` From ab50c6b3ca26c87e4b7fdbe04da5d0d236d2e9ce Mon Sep 17 00:00:00 2001 From: Eugenij Date: Tue, 2 Jul 2019 01:27:00 +0000 Subject: [PATCH 21/29] Add fallback value to the admin report view. This is to avoid 500 error when the reported user no longer exists in the database. --- lib/pleroma/web/admin_api/views/report_view.ex | 4 +++- test/web/admin_api/views/report_view_test.exs | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex index 73ccdc582..a25f3f1fe 100644 --- a/lib/pleroma/web/admin_api/views/report_view.ex +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -46,8 +46,10 @@ def render("show.json", %{report: report}) do } end - defp merge_account_views(user) do + defp merge_account_views(%User{} = user) do Pleroma.Web.MastodonAPI.AccountView.render("account.json", %{user: user}) |> Map.merge(Pleroma.Web.AdminAPI.AccountView.render("show.json", %{user: user})) end + + defp merge_account_views(_), do: %{} end diff --git a/test/web/admin_api/views/report_view_test.exs b/test/web/admin_api/views/report_view_test.exs index 75d8bb4b5..a00c9c579 100644 --- a/test/web/admin_api/views/report_view_test.exs +++ b/test/web/admin_api/views/report_view_test.exs @@ -111,4 +111,20 @@ test "sanitizes report description" do refute "" == ReportView.render("show.json", %{report: activity})[:content] end + + test "doesn't error out when the user doesn't exists" do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = + CommonAPI.report(user, %{ + "account_id" => other_user.id, + "comment" => "" + }) + + Pleroma.User.delete(other_user) + Pleroma.User.invalidate_cache(other_user) + + assert %{} = ReportView.render("show.json", %{report: activity}) + end end From a02f52d12a32d78aa11a86840dfdaf94b5355dfb Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 2 Jul 2019 09:04:59 +0300 Subject: [PATCH 22/29] [#1048] Resolved violations of SemVer version format. Refactoring. --- mix.exs | 48 +++++++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/mix.exs b/mix.exs index c2618d2b2..22d3d50df 100644 --- a/mix.exs +++ b/mix.exs @@ -174,10 +174,14 @@ defp aliases do # Builds a version string made of: # * the application version # * a pre-release if ahead of the tag: the describe string (-count-commithash) - # * build info: + # * branch name + # * build metadata: # * a build name if `PLEROMA_BUILD_NAME` or `:pleroma, :build_name` is defined # * the mix environment if different than prod defp version(version) do + identifier_filter = ~r/[^0-9a-z\-]+/i + + # Pre-release version, denoted from patch version with a hyphen {git_tag, git_pre_release} = with {tag, 0} <- System.cmd("git", ["describe", "--tags", "--abbrev=0"], stderr_to_stdout: true), @@ -198,6 +202,19 @@ defp version(version) do ) end + # Branch name as pre-release version component, denoted with a dot + branch_name = + with {branch_name, 0} <- System.cmd("git", ["rev-parse", "--abbrev-ref", "HEAD"]), + branch_name <- System.get_env("PLEROMA_BUILD_BRANCH") || branch_name, + true <- branch_name != "master" do + branch_name = + branch_name + |> String.trim() + |> String.replace(identifier_filter, "-") + + "." <> branch_name + end + build_name = cond do name = Application.get_env(:pleroma, :build_name) -> name @@ -206,37 +223,26 @@ defp version(version) do end env_name = if Mix.env() != :prod, do: to_string(Mix.env()) - env_override = System.get_env("PLEROMA_BUILD_ENV") env_name = - if env_override do - if env_override != "prod", do: env_override - else - env_name + case env_override do + nil -> env_name + env_override when env_override in ["", "prod"] -> nil + env_override -> env_override end - build = + # Build metadata, denoted with a plus sign + build_metadata = [build_name, env_name] |> Enum.filter(fn string -> string && string != "" end) - |> Enum.join("-") + |> Enum.join(".") |> (fn "" -> nil - string -> "+" <> string + string -> "+" <> String.replace(string, identifier_filter, "-") end).() - branch_name = - with {branch_name, 0} <- System.cmd("git", ["rev-parse", "--abbrev-ref", "HEAD"]), - branch_name <- System.get_env("PLEROMA_BUILD_BRANCH") || branch_name, - true <- branch_name != "master" do - branch_name = - String.trim(branch_name) - |> String.replace(~r/[^0-9a-z\-\.]+/i, "-") - - "-" <> branch_name - end - - [version, git_pre_release, branch_name, build] + [version, git_pre_release, branch_name, build_metadata] |> Enum.filter(fn string -> string && string != "" end) |> Enum.join() end From 8c7d7cc9365e996c624d4bdb6c8d578687ff117f Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 2 Jul 2019 18:47:53 +0300 Subject: [PATCH 23/29] [#1048] Fixed version parsing in pleroma_ctl. Closes #1036. --- rel/files/bin/pleroma_ctl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rel/files/bin/pleroma_ctl b/rel/files/bin/pleroma_ctl index 9c67b209b..e731d20eb 100755 --- a/rel/files/bin/pleroma_ctl +++ b/rel/files/bin/pleroma_ctl @@ -30,12 +30,15 @@ detect_flavour() { detect_branch() { version="$(cut -d' ' -f2 <"$RELEASE_ROOT"/releases/start_erl.data)" - branch="$(echo "$version" | cut -d'-' -f 4)" + # Expected format: major.minor.patch_version(-number_of_commits_ahead_of_tag-gcommit_hash).branch + branch="$(echo "$version" | cut -d'.' -f 4)" if [ "$branch" = "develop" ]; then echo "develop" elif [ "$branch" = "" ]; then echo "master" else + # Note: branch name in version is of SemVer format and may only contain [0-9a-zA-Z-] symbols — + # if supporting releases for more branches, need to ensure they contain only these symbols. echo "Releases are built only for master and develop branches" >&2 exit 1 fi From 291d95dcc3cd85050421d2ab022e01a6d89c0f3d Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Wed, 3 Jul 2019 01:14:40 +0300 Subject: [PATCH 24/29] Fix migration --- .../20170522160642_case_insensivtivity.exs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/priv/repo/migrations/20170522160642_case_insensivtivity.exs b/priv/repo/migrations/20170522160642_case_insensivtivity.exs index c7565946e..470a545e5 100644 --- a/priv/repo/migrations/20170522160642_case_insensivtivity.exs +++ b/priv/repo/migrations/20170522160642_case_insensivtivity.exs @@ -2,18 +2,24 @@ defmodule Pleroma.Repo.Migrations.CaseInsensivtivity do use Ecto.Migration def up do - execute ("create extension if not exists citext") + execute("create extension if not exists citext") + + drop_if_exists(index(:users, [:email])) + alter table(:users) do - modify :email, :citext - modify :nickname, :citext + modify(:email, :citext) + modify(:nickname, :citext) end + + create_if_not_exists(index(:users, [:email])) end def down do alter table(:users) do - modify :email, :string - modify :nickname, :string + modify(:email, :string) + modify(:nickname, :string) end - execute ("drop extension if exists citext") + + execute("drop extension if exists citext") end end From 5184b0f41aabb79a47e8d3b8c76b88ca04a53b24 Mon Sep 17 00:00:00 2001 From: Eugenij Date: Wed, 3 Jul 2019 10:19:51 +0000 Subject: [PATCH 25/29] Use fallback values for search queries This is to make sure the entire request doesn't return a 500 error if user or status search times out. --- CHANGELOG.md | 1 + .../web/mastodon_api/search_controller.ex | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a0dad453..aefc09177 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Not being able to pin unlisted posts +- Mastodon API: Handling of search timeouts (`/api/v1/search` and `/api/v2/search`) ### Changed - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text diff --git a/lib/pleroma/web/mastodon_api/search_controller.ex b/lib/pleroma/web/mastodon_api/search_controller.ex index 0d1e2355d..efa9cc788 100644 --- a/lib/pleroma/web/mastodon_api/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/search_controller.ex @@ -17,8 +17,8 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do plug(Pleroma.Plugs.RateLimiter, :search when action in [:search, :search2, :account_search]) def search2(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do - accounts = User.search(query, search_options(params, user)) - statuses = Activity.search(user, query) + accounts = with_fallback(fn -> User.search(query, search_options(params, user)) end, []) + statuses = with_fallback(fn -> Activity.search(user, query) end, []) tags_path = Web.base_url() <> "/tag/" tags = @@ -40,8 +40,8 @@ def search2(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do end def search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do - accounts = User.search(query, search_options(params, user)) - statuses = Activity.search(user, query) + accounts = with_fallback(fn -> User.search(query, search_options(params, user)) end, []) + statuses = with_fallback(fn -> Activity.search(user, query) end, []) tags = query @@ -76,4 +76,14 @@ defp search_options(params, user) do for_user: user ] end + + defp with_fallback(f, fallback) do + try do + f.() + rescue + error -> + Logger.error("#{__MODULE__} search error: #{inspect(error)}") + fallback + end + end end From efefee82415d872209cf5884375891e577f54781 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Wed, 3 Jul 2019 14:56:02 +0300 Subject: [PATCH 26/29] Split alters rather than work with indexes --- .../20170522160642_case_insensivtivity.exs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/priv/repo/migrations/20170522160642_case_insensivtivity.exs b/priv/repo/migrations/20170522160642_case_insensivtivity.exs index 470a545e5..9a67727e9 100644 --- a/priv/repo/migrations/20170522160642_case_insensivtivity.exs +++ b/priv/repo/migrations/20170522160642_case_insensivtivity.exs @@ -1,22 +1,28 @@ defmodule Pleroma.Repo.Migrations.CaseInsensivtivity do use Ecto.Migration + # Two-steps alters are intentional. + # When alter of 2 columns is done in a single operation, + # inconsistent failures happen because of index on `email` column. + def up do execute("create extension if not exists citext") - drop_if_exists(index(:users, [:email])) - alter table(:users) do modify(:email, :citext) - modify(:nickname, :citext) end - create_if_not_exists(index(:users, [:email])) + alter table(:users) do + modify(:nickname, :citext) + end end def down do alter table(:users) do modify(:email, :string) + end + + alter table(:users) do modify(:nickname, :string) end From 15eb1235ed54b65d79a6e955be7b4302993bd620 Mon Sep 17 00:00:00 2001 From: Eugenij Date: Wed, 3 Jul 2019 18:35:57 +0000 Subject: [PATCH 27/29] Expose `skip_thread_containment` in nodeinfo --- CHANGELOG.md | 3 +++ lib/pleroma/web/nodeinfo/nodeinfo_controller.ex | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aefc09177..3dbbd8225 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text +### Changed +- NodeInfo: Return `skipThreadContainment` in `metadata` for the `skip_thread_containment` option + ## [1.0.0] - 2019-06-29 ### Security - Mastodon API: Fix display names not being sanitized diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index 32be430b7..869dda5c5 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -162,7 +162,8 @@ def raw_nodeinfo do accountActivationRequired: Config.get([:instance, :account_activation_required], false), invitesEnabled: Config.get([:instance, :invites_enabled], false), features: features, - restrictedNicknames: Config.get([Pleroma.User, :restricted_nicknames]) + restrictedNicknames: Config.get([Pleroma.User, :restricted_nicknames]), + skipThreadContainment: Config.get([:instance, :skip_thread_containment], false) } } end From 977c2d044810d71fa89f9976ef342b6f708a027a Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 5 Jul 2019 06:19:27 +0200 Subject: [PATCH 28/29] tasks/pleroma/instance.ex: Change :upload_dir to :uploads_dir Closes: https://git.pleroma.social/pleroma/pleroma/issues/1058 --- lib/mix/tasks/pleroma/instance.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index a27c4b897..2ae16adc0 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -149,7 +149,7 @@ def run(["gen" | rest]) do uploads_dir = get_option( options, - :upload_dir, + :uploads_dir, "What directory should media uploads go in (when using the local uploader)?", Pleroma.Config.get([Pleroma.Uploaders.Local, :uploads]) ) From 6dadf5d6f41dda8d0f760da25f4394eecf467a80 Mon Sep 17 00:00:00 2001 From: Maksim Date: Fri, 5 Jul 2019 04:22:08 +0000 Subject: [PATCH 29/29] [#1043] fix search accounts. --- lib/pleroma/user.ex | 11 +- lib/pleroma/user/search.ex | 21 ++++ test/support/helpers.ex | 6 + test/user_search_test.exs | 221 +++++++++++++++++++++++++++++++++++++ test/user_test.exs | 183 ------------------------------ 5 files changed, 252 insertions(+), 190 deletions(-) create mode 100644 test/user_search_test.exs diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index f7191762f..09f86aaa2 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -836,15 +836,12 @@ def unblock(blocker, %{ap_id: ap_id}) do def mutes?(nil, _), do: false def mutes?(user, %{ap_id: ap_id}), do: Enum.member?(user.info.mutes, ap_id) - def blocks?(user, %{ap_id: ap_id}) do - blocks = user.info.blocks - domain_blocks = user.info.domain_blocks + def blocks?(%User{info: info} = _user, %{ap_id: ap_id}) do + blocks = info.blocks + domain_blocks = info.domain_blocks %{host: host} = URI.parse(ap_id) - Enum.member?(blocks, ap_id) || - Enum.any?(domain_blocks, fn domain -> - host == domain - end) + Enum.member?(blocks, ap_id) || Enum.any?(domain_blocks, &(&1 == host)) end def subscribed_to?(user, %{ap_id: ap_id}) do diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index ed06c2ab9..7680c2afd 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -43,6 +43,8 @@ def search(query_string, opts \\ []) do defp search_query(query_string, for_user, following) do for_user |> base_query(following) + |> filter_blocked_user(for_user) + |> filter_blocked_domains(for_user) |> search_subqueries(query_string) |> union_subqueries |> distinct_query() @@ -55,6 +57,25 @@ defp search_query(query_string, for_user, following) do defp base_query(_user, false), do: User defp base_query(user, true), do: User.get_followers_query(user) + defp filter_blocked_user(query, %User{info: %{blocks: blocks}}) + when length(blocks) > 0 do + from(q in query, where: not (q.ap_id in ^blocks)) + end + + defp filter_blocked_user(query, _), do: query + + defp filter_blocked_domains(query, %User{info: %{domain_blocks: domain_blocks}}) + when length(domain_blocks) > 0 do + domains = Enum.join(domain_blocks, ",") + + from( + q in query, + where: fragment("substring(ap_id from '.*://([^/]*)') NOT IN (?)", ^domains) + ) + end + + defp filter_blocked_domains(query, _), do: query + defp paginate(query, limit, offset) do from(q in query, limit: ^limit, offset: ^offset) end diff --git a/test/support/helpers.ex b/test/support/helpers.ex index 6e389ce52..1a92be065 100644 --- a/test/support/helpers.ex +++ b/test/support/helpers.ex @@ -9,6 +9,12 @@ defmodule Pleroma.Tests.Helpers do defmacro __using__(_opts) do quote do + def collect_ids(collection) do + collection + |> Enum.map(& &1.id) + |> Enum.sort() + end + def refresh_record(%{id: id, __struct__: model} = _), do: refresh_record(model, %{id: id}) diff --git a/test/user_search_test.exs b/test/user_search_test.exs new file mode 100644 index 000000000..8f8472aae --- /dev/null +++ b/test/user_search_test.exs @@ -0,0 +1,221 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.UserSearchTest do + alias Pleroma.Repo + alias Pleroma.User + use Pleroma.DataCase + + import Pleroma.Factory + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + describe "User.search" do + test "accepts limit parameter" do + Enum.each(0..4, &insert(:user, %{nickname: "john#{&1}"})) + assert length(User.search("john", limit: 3)) == 3 + assert length(User.search("john")) == 5 + end + + test "accepts offset parameter" do + Enum.each(0..4, &insert(:user, %{nickname: "john#{&1}"})) + assert length(User.search("john", limit: 3)) == 3 + assert length(User.search("john", limit: 3, offset: 3)) == 2 + end + + test "finds a user by full or partial nickname" do + user = insert(:user, %{nickname: "john"}) + + Enum.each(["john", "jo", "j"], fn query -> + assert user == + User.search(query) + |> List.first() + |> Map.put(:search_rank, nil) + |> Map.put(:search_type, nil) + end) + end + + test "finds a user by full or partial name" do + user = insert(:user, %{name: "John Doe"}) + + Enum.each(["John Doe", "JOHN", "doe", "j d", "j", "d"], fn query -> + assert user == + User.search(query) + |> List.first() + |> Map.put(:search_rank, nil) + |> Map.put(:search_type, nil) + end) + end + + test "finds users, preferring nickname matches over name matches" do + u1 = insert(:user, %{name: "lain", nickname: "nick1"}) + u2 = insert(:user, %{nickname: "lain", name: "nick1"}) + + assert [u2.id, u1.id] == Enum.map(User.search("lain"), & &1.id) + end + + test "finds users, considering density of matched tokens" do + u1 = insert(:user, %{name: "Bar Bar plus Word Word"}) + u2 = insert(:user, %{name: "Word Word Bar Bar Bar"}) + + assert [u2.id, u1.id] == Enum.map(User.search("bar word"), & &1.id) + end + + test "finds users, ranking by similarity" do + u1 = insert(:user, %{name: "lain"}) + _u2 = insert(:user, %{name: "ean"}) + u3 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social"}) + u4 = insert(:user, %{nickname: "lain@pleroma.soykaf.com"}) + + assert [u4.id, u3.id, u1.id] == Enum.map(User.search("lain@ple", for_user: u1), & &1.id) + end + + test "finds users, handling misspelled requests" do + u1 = insert(:user, %{name: "lain"}) + + assert [u1.id] == Enum.map(User.search("laiin"), & &1.id) + end + + test "finds users, boosting ranks of friends and followers" do + u1 = insert(:user) + u2 = insert(:user, %{name: "Doe"}) + follower = insert(:user, %{name: "Doe"}) + friend = insert(:user, %{name: "Doe"}) + + {:ok, follower} = User.follow(follower, u1) + {:ok, u1} = User.follow(u1, friend) + + assert [friend.id, follower.id, u2.id] -- + Enum.map(User.search("doe", resolve: false, for_user: u1), & &1.id) == [] + end + + test "finds followers of user by partial name" do + u1 = insert(:user) + u2 = insert(:user, %{name: "Jimi"}) + follower_jimi = insert(:user, %{name: "Jimi Hendrix"}) + follower_lizz = insert(:user, %{name: "Lizz Wright"}) + friend = insert(:user, %{name: "Jimi"}) + + {:ok, follower_jimi} = User.follow(follower_jimi, u1) + {:ok, _follower_lizz} = User.follow(follower_lizz, u2) + {:ok, u1} = User.follow(u1, friend) + + assert Enum.map(User.search("jimi", following: true, for_user: u1), & &1.id) == [ + follower_jimi.id + ] + + assert User.search("lizz", following: true, for_user: u1) == [] + end + + test "find local and remote users for authenticated users" do + u1 = insert(:user, %{name: "lain"}) + u2 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false}) + u3 = insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false}) + + results = + "lain" + |> User.search(for_user: u1) + |> Enum.map(& &1.id) + |> Enum.sort() + + assert [u1.id, u2.id, u3.id] == results + end + + test "find only local users for unauthenticated users" do + %{id: id} = insert(:user, %{name: "lain"}) + insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false}) + insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false}) + + assert [%{id: ^id}] = User.search("lain") + end + + test "find only local users for authenticated users when `limit_to_local_content` is `:all`" do + Pleroma.Config.put([:instance, :limit_to_local_content], :all) + + %{id: id} = insert(:user, %{name: "lain"}) + insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false}) + insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false}) + + assert [%{id: ^id}] = User.search("lain") + + Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) + end + + test "find all users for unauthenticated users when `limit_to_local_content` is `false`" do + Pleroma.Config.put([:instance, :limit_to_local_content], false) + + u1 = insert(:user, %{name: "lain"}) + u2 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false}) + u3 = insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false}) + + results = + "lain" + |> User.search() + |> Enum.map(& &1.id) + |> Enum.sort() + + assert [u1.id, u2.id, u3.id] == results + + Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) + end + + test "finds a user whose name is nil" do + _user = insert(:user, %{name: "notamatch", nickname: "testuser@pleroma.amplifie.red"}) + user_two = insert(:user, %{name: nil, nickname: "lain@pleroma.soykaf.com"}) + + assert user_two == + User.search("lain@pleroma.soykaf.com") + |> List.first() + |> Map.put(:search_rank, nil) + |> Map.put(:search_type, nil) + end + + test "does not yield false-positive matches" do + insert(:user, %{name: "John Doe"}) + + Enum.each(["mary", "a", ""], fn query -> + assert [] == User.search(query) + end) + end + + test "works with URIs" do + user = insert(:user) + + results = + User.search("http://mastodon.example.org/users/admin", resolve: true, for_user: user) + + result = results |> List.first() + + user = User.get_cached_by_ap_id("http://mastodon.example.org/users/admin") + + assert length(results) == 1 + assert user == result |> Map.put(:search_rank, nil) |> Map.put(:search_type, nil) + end + + test "excludes a blocked users from search result" do + user = insert(:user, %{nickname: "Bill"}) + + [blocked_user | users] = Enum.map(0..3, &insert(:user, %{nickname: "john#{&1}"})) + + blocked_user2 = + insert( + :user, + %{nickname: "john awful", ap_id: "https://awful-and-rude-instance.com/user/bully"} + ) + + User.block_domain(user, "awful-and-rude-instance.com") + User.block(user, blocked_user) + + account_ids = User.search("john", for_user: refresh_record(user)) |> collect_ids + + assert account_ids == collect_ids(users) + refute Enum.member?(account_ids, blocked_user.id) + refute Enum.member?(account_ids, blocked_user2.id) + assert length(account_ids) == 3 + end + end +end diff --git a/test/user_test.exs b/test/user_test.exs index 198a97fae..fb497843c 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1012,189 +1012,6 @@ test "User.delete() plugs any possible zombie objects" do end end - describe "User.search" do - test "accepts limit parameter" do - Enum.each(0..4, &insert(:user, %{nickname: "john#{&1}"})) - assert length(User.search("john", limit: 3)) == 3 - assert length(User.search("john")) == 5 - end - - test "accepts offset parameter" do - Enum.each(0..4, &insert(:user, %{nickname: "john#{&1}"})) - assert length(User.search("john", limit: 3)) == 3 - assert length(User.search("john", limit: 3, offset: 3)) == 2 - end - - test "finds a user by full or partial nickname" do - user = insert(:user, %{nickname: "john"}) - - Enum.each(["john", "jo", "j"], fn query -> - assert user == - User.search(query) - |> List.first() - |> Map.put(:search_rank, nil) - |> Map.put(:search_type, nil) - end) - end - - test "finds a user by full or partial name" do - user = insert(:user, %{name: "John Doe"}) - - Enum.each(["John Doe", "JOHN", "doe", "j d", "j", "d"], fn query -> - assert user == - User.search(query) - |> List.first() - |> Map.put(:search_rank, nil) - |> Map.put(:search_type, nil) - end) - end - - test "finds users, preferring nickname matches over name matches" do - u1 = insert(:user, %{name: "lain", nickname: "nick1"}) - u2 = insert(:user, %{nickname: "lain", name: "nick1"}) - - assert [u2.id, u1.id] == Enum.map(User.search("lain"), & &1.id) - end - - test "finds users, considering density of matched tokens" do - u1 = insert(:user, %{name: "Bar Bar plus Word Word"}) - u2 = insert(:user, %{name: "Word Word Bar Bar Bar"}) - - assert [u2.id, u1.id] == Enum.map(User.search("bar word"), & &1.id) - end - - test "finds users, ranking by similarity" do - u1 = insert(:user, %{name: "lain"}) - _u2 = insert(:user, %{name: "ean"}) - u3 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social"}) - u4 = insert(:user, %{nickname: "lain@pleroma.soykaf.com"}) - - assert [u4.id, u3.id, u1.id] == Enum.map(User.search("lain@ple", for_user: u1), & &1.id) - end - - test "finds users, handling misspelled requests" do - u1 = insert(:user, %{name: "lain"}) - - assert [u1.id] == Enum.map(User.search("laiin"), & &1.id) - end - - test "finds users, boosting ranks of friends and followers" do - u1 = insert(:user) - u2 = insert(:user, %{name: "Doe"}) - follower = insert(:user, %{name: "Doe"}) - friend = insert(:user, %{name: "Doe"}) - - {:ok, follower} = User.follow(follower, u1) - {:ok, u1} = User.follow(u1, friend) - - assert [friend.id, follower.id, u2.id] -- - Enum.map(User.search("doe", resolve: false, for_user: u1), & &1.id) == [] - end - - test "finds followers of user by partial name" do - u1 = insert(:user) - u2 = insert(:user, %{name: "Jimi"}) - follower_jimi = insert(:user, %{name: "Jimi Hendrix"}) - follower_lizz = insert(:user, %{name: "Lizz Wright"}) - friend = insert(:user, %{name: "Jimi"}) - - {:ok, follower_jimi} = User.follow(follower_jimi, u1) - {:ok, _follower_lizz} = User.follow(follower_lizz, u2) - {:ok, u1} = User.follow(u1, friend) - - assert Enum.map(User.search("jimi", following: true, for_user: u1), & &1.id) == [ - follower_jimi.id - ] - - assert User.search("lizz", following: true, for_user: u1) == [] - end - - test "find local and remote users for authenticated users" do - u1 = insert(:user, %{name: "lain"}) - u2 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false}) - u3 = insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false}) - - results = - "lain" - |> User.search(for_user: u1) - |> Enum.map(& &1.id) - |> Enum.sort() - - assert [u1.id, u2.id, u3.id] == results - end - - test "find only local users for unauthenticated users" do - %{id: id} = insert(:user, %{name: "lain"}) - insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false}) - insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false}) - - assert [%{id: ^id}] = User.search("lain") - end - - test "find only local users for authenticated users when `limit_to_local_content` is `:all`" do - Pleroma.Config.put([:instance, :limit_to_local_content], :all) - - %{id: id} = insert(:user, %{name: "lain"}) - insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false}) - insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false}) - - assert [%{id: ^id}] = User.search("lain") - - Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) - end - - test "find all users for unauthenticated users when `limit_to_local_content` is `false`" do - Pleroma.Config.put([:instance, :limit_to_local_content], false) - - u1 = insert(:user, %{name: "lain"}) - u2 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false}) - u3 = insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false}) - - results = - "lain" - |> User.search() - |> Enum.map(& &1.id) - |> Enum.sort() - - assert [u1.id, u2.id, u3.id] == results - - Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) - end - - test "finds a user whose name is nil" do - _user = insert(:user, %{name: "notamatch", nickname: "testuser@pleroma.amplifie.red"}) - user_two = insert(:user, %{name: nil, nickname: "lain@pleroma.soykaf.com"}) - - assert user_two == - User.search("lain@pleroma.soykaf.com") - |> List.first() - |> Map.put(:search_rank, nil) - |> Map.put(:search_type, nil) - end - - test "does not yield false-positive matches" do - insert(:user, %{name: "John Doe"}) - - Enum.each(["mary", "a", ""], fn query -> - assert [] == User.search(query) - end) - end - - test "works with URIs" do - user = insert(:user) - - results = - User.search("http://mastodon.example.org/users/admin", resolve: true, for_user: user) - - result = results |> List.first() - - user = User.get_cached_by_ap_id("http://mastodon.example.org/users/admin") - - assert length(results) == 1 - assert user == result |> Map.put(:search_rank, nil) |> Map.put(:search_type, nil) - end - end - test "auth_active?/1 works correctly" do Pleroma.Config.put([:instance, :account_activation_required], true)