From e93cc561cd42ff4ca7f3c95cdbf8dfa7fb9f4a74 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sat, 25 Jan 2020 18:42:04 +0300 Subject: [PATCH 01/13] restarting pleroma from outside application --- lib/pleroma/config/loader.ex | 6 +- lib/pleroma/config/transfer_task.ex | 95 +++++++++++++++---- .../web/admin_api/admin_api_controller.ex | 33 +++++-- .../web/admin_api/views/config_view.ex | 10 +- lib/pleroma/web/router.ex | 1 + mix.exs | 10 +- mix.lock | 1 + test/config/transfer_task_test.exs | 67 +++++++++++++ .../admin_api/admin_api_controller_test.exs | 43 ++++++++- 9 files changed, 231 insertions(+), 35 deletions(-) diff --git a/lib/pleroma/config/loader.ex b/lib/pleroma/config/loader.ex index 68b247381..b8787cb49 100644 --- a/lib/pleroma/config/loader.ex +++ b/lib/pleroma/config/loader.ex @@ -3,8 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Config.Loader do - @paths ["config/config.exs", "config/#{Mix.env()}.exs"] - @reject_keys [ Pleroma.Repo, Pleroma.Web.Endpoint, @@ -35,8 +33,8 @@ defp do_merge(conf1, conf2), do: Mix.Config.merge(conf1, conf2) def load_and_merge do all_paths = if Pleroma.Config.get(:release), - do: @paths ++ ["config/releases.exs"], - else: @paths + do: ["config/config.exs", "config/releases.exs"], + else: ["config/config.exs"] all_paths |> Enum.map(&load(&1)) diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index d54f38ee4..6c5ba1f95 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -10,6 +10,30 @@ defmodule Pleroma.Config.TransferTask do require Logger + @type env() :: :test | :benchmark | :dev | :prod + + @reboot_time_keys [ + {:pleroma, :hackney_pools}, + {:pleroma, :chat}, + {:pleroma, Oban}, + {:pleroma, :rate_limit}, + {:pleroma, :markup}, + {:plerome, :streamer} + ] + + @reboot_time_subkeys [ + {:pleroma, Pleroma.Captcha, [:seconds_valid]}, + {:pleroma, Pleroma.Upload, [:proxy_remote]}, + {:pleroma, :instance, [:upload_limit]}, + {:pleroma, :email_notifications, [:digest]}, + {:pleroma, :oauth2, [:clean_expired_tokens]}, + {:pleroma, Pleroma.ActivityExpiration, [:enabled]}, + {:pleroma, Pleroma.ScheduledActivity, [:enabled]}, + {:pleroma, :gopher, [:enabled]} + ] + + @reject [nil, :prometheus] + def start_link(_) do load_and_update_env() if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo) @@ -17,21 +41,34 @@ def start_link(_) do end @spec load_and_update_env([ConfigDB.t()]) :: :ok | false - def load_and_update_env(deleted \\ []) do + def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do with true <- Pleroma.Config.get(:configurable_from_database), true <- Ecto.Adapters.SQL.table_exists?(Repo, "config"), started_applications <- Application.started_applications() do # We need to restart applications for loaded settings take effect + in_db = Repo.all(ConfigDB) with_deleted = in_db ++ deleted - with_deleted - |> Enum.map(&merge_and_update(&1)) - |> Enum.uniq() - # TODO: some problem with prometheus after restart! - |> Enum.reject(&(&1 in [:pleroma, nil, :prometheus])) - |> Enum.each(&restart(started_applications, &1)) + reject_for_restart = if restart_pleroma?, do: @reject, else: [:pleroma | @reject] + + applications = + with_deleted + |> Enum.map(&merge_and_update(&1)) + |> Enum.uniq() + # TODO: some problem with prometheus after restart! + |> Enum.reject(&(&1 in reject_for_restart)) + + # to be ensured that pleroma will be restarted last + applications = + if :pleroma in applications do + List.delete(applications, :pleroma) ++ [:pleroma] + else + applications + end + + Enum.each(applications, &restart(started_applications, &1, Pleroma.Config.get(:env))) :ok end @@ -43,12 +80,25 @@ defp merge_and_update(setting) do group = ConfigDB.from_string(setting.group) default = Pleroma.Config.Holder.config(group, key) - merged_value = merge_value(setting, default, group, key) + value = ConfigDB.from_binary(setting.value) + + merged_value = + if Ecto.get_meta(setting, :state) == :deleted do + default + else + if can_be_merged?(default, value) do + ConfigDB.merge_group(group, key, default, value) + else + value + end + end :ok = update_env(group, key, merged_value) if group != :logger do - group + if group != :pleroma or pleroma_need_restart?(group, key, value) do + group + end else # change logger configuration in runtime, without restart if Keyword.keyword?(merged_value) and @@ -76,22 +126,31 @@ defp merge_and_update(setting) do end end - defp merge_value(%{__meta__: %{state: :deleted}}, default, _group, _key), do: default + @spec pleroma_need_restart?(atom(), atom(), any()) :: boolean() + def pleroma_need_restart?(group, key, value) do + group_and_key_need_reboot?(group, key) or group_and_subkey_need_reboot?(group, key, value) + end - defp merge_value(setting, default, group, key) do - value = ConfigDB.from_binary(setting.value) + defp group_and_key_need_reboot?(group, key) do + Enum.any?(@reboot_time_keys, fn {g, k} -> g == group and k == key end) + end - if can_be_merged?(default, value) do - ConfigDB.merge_group(group, key, default, value) - else - value - end + defp group_and_subkey_need_reboot?(group, key, value) do + Keyword.keyword?(value) and + Enum.any?(@reboot_time_subkeys, fn {g, k, subkeys} -> + g == group and k == key and + Enum.any?(Keyword.keys(value), &(&1 in subkeys)) + end) end defp update_env(group, key, nil), do: Application.delete_env(group, key) defp update_env(group, key, value), do: Application.put_env(group, key, value) - defp restart(started_applications, app) do + defp restart(_, :pleroma, :test), do: Logger.warn("pleroma restarted") + + defp restart(_, :pleroma, _), do: send(Restarter.Pleroma, :after_boot) + + defp restart(started_applications, app, _) do with {^app, _, _} <- List.keyfind(started_applications, app, 0), :ok <- Application.stop(app) do :ok = Application.start(app) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 2314d3274..6f0449418 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -890,17 +890,36 @@ def config_update(conn, %{"configs" => configs}) do Ecto.get_meta(config, :state) == :deleted end) - Pleroma.Config.TransferTask.load_and_update_env(deleted) + Pleroma.Config.TransferTask.load_and_update_env(deleted, false) - Mix.Tasks.Pleroma.Config.run([ - "migrate_from_db", - "--env", - to_string(Pleroma.Config.get(:env)) - ]) + need_reboot? = + Enum.any?(updated, fn config -> + group = ConfigDB.from_string(config.group) + key = ConfigDB.from_string(config.key) + value = ConfigDB.from_binary(config.value) + Pleroma.Config.TransferTask.pleroma_need_restart?(group, key, value) + end) + + response = %{configs: updated} + + response = + if need_reboot?, do: Map.put(response, :need_reboot, need_reboot?), else: response conn |> put_view(ConfigView) - |> render("index.json", %{configs: updated}) + |> render("index.json", response) + end + end + + def restart(conn, _params) do + with :ok <- configurable_from_database(conn) do + if Pleroma.Config.get(:env) == :test do + Logger.warn("pleroma restarted") + else + send(Restarter.Pleroma, {:restart, 50}) + end + + json(conn, %{}) end end diff --git a/lib/pleroma/web/admin_api/views/config_view.ex b/lib/pleroma/web/admin_api/views/config_view.ex index 23d97e847..bbb53efcd 100644 --- a/lib/pleroma/web/admin_api/views/config_view.ex +++ b/lib/pleroma/web/admin_api/views/config_view.ex @@ -5,10 +5,16 @@ defmodule Pleroma.Web.AdminAPI.ConfigView do use Pleroma.Web, :view - def render("index.json", %{configs: configs}) do - %{ + def render("index.json", %{configs: configs} = params) do + map = %{ configs: render_many(configs, __MODULE__, "show.json", as: :config) } + + if params[:need_reboot] do + Map.put(map, :need_reboot, true) + else + map + end end def render("show.json", %{config: config}) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index ef6e5a565..43fee8a0f 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -197,6 +197,7 @@ defmodule Pleroma.Web.Router do post("/config", AdminAPIController, :config_update) get("/config/descriptions", AdminAPIController, :config_descriptions) get("/config/migrate_from_db", AdminAPIController, :migrate_from_db) + get("/restart", AdminAPIController, :restart) get("/moderation_log", AdminAPIController, :list_log) diff --git a/mix.exs b/mix.exs index 0aa7c862f..1d0b59e56 100644 --- a/mix.exs +++ b/mix.exs @@ -8,7 +8,7 @@ def project do elixir: "~> 1.8", elixirc_paths: elixirc_paths(Mix.env()), compilers: [:phoenix, :gettext] ++ Mix.compilers(), - elixirc_options: [warnings_as_errors: true], + elixirc_options: [warnings_as_errors: warnings_as_errors(Mix.env())], xref: [exclude: [:eldap]], start_permanent: Mix.env() == :prod, aliases: aliases(), @@ -73,6 +73,11 @@ defp elixirc_paths(:benchmark), do: ["lib", "benchmarks"] defp elixirc_paths(:test), do: ["lib", "test/support"] defp elixirc_paths(_), do: ["lib"] + defp warnings_as_errors(:prod), do: false + # Uncomment this if you need testing configurable_from_database logic + # defp warnings_as_errors(:dev), do: false + defp warnings_as_errors(_), do: true + # Specifies OAuth dependencies. defp oauth_deps do oauth_strategy_packages = @@ -166,7 +171,8 @@ defp deps do {:captcha, git: "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"}, - {:mox, "~> 0.5", only: :test} + {:mox, "~> 0.5", only: :test}, + {:restarter, git: "https://git.pleroma.social/alex.s/restarter"} ] ++ oauth_deps() end diff --git a/mix.lock b/mix.lock index c1fe223c0..538337cf3 100644 --- a/mix.lock +++ b/mix.lock @@ -94,6 +94,7 @@ "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]}, "remote_ip": {:git, "https://git.pleroma.social/pleroma/remote_ip.git", "825dc00aaba5a1b7c4202a532b696b595dd3bcb3", [ref: "825dc00aaba5a1b7c4202a532b696b595dd3bcb3"]}, + "restarter": {:git, "https://git.pleroma.social/alex.s/restarter", "1932655b80a1409405d897911c06ebee4ac8c2d8", []}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.5", "6eaf7ad16cb568bb01753dbbd7a95ff8b91c7979482b95f38443fe2c8852a79b", [:make, :mix, :rebar3], [], "hexpm"}, "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm"}, "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"}, diff --git a/test/config/transfer_task_test.exs b/test/config/transfer_task_test.exs index 53e8703fd..0d328f0c3 100644 --- a/test/config/transfer_task_test.exs +++ b/test/config/transfer_task_test.exs @@ -5,6 +5,8 @@ defmodule Pleroma.Config.TransferTaskTest do use Pleroma.DataCase + import ExUnit.CaptureLog + alias Pleroma.Config.TransferTask alias Pleroma.ConfigDB @@ -105,4 +107,69 @@ test "transfer config values with full subkey update" do Application.put_env(:pleroma, :assets, assets) end) end + + describe "pleroma restart" do + test "don't restart if no reboot time settings were changed" do + emoji = Application.get_env(:pleroma, :emoji) + on_exit(fn -> Application.put_env(:pleroma, :emoji, emoji) end) + + ConfigDB.create(%{ + group: ":pleroma", + key: ":emoji", + value: [groups: [a: 1, b: 2]] + }) + + assert capture_log(fn -> TransferTask.start_link([]) end) =~ "" + end + + test "restart pleroma on reboot time key" do + chat = Application.get_env(:pleroma, :chat) + on_exit(fn -> Application.put_env(:pleroma, :chat, chat) end) + + ConfigDB.create(%{ + group: ":pleroma", + key: ":chat", + value: [enabled: false] + }) + + assert capture_log(fn -> TransferTask.start_link([]) end) =~ "pleroma restarted" + end + + test "restart pleroma on reboot time subkey" do + captcha = Application.get_env(:pleroma, Pleroma.Captcha) + on_exit(fn -> Application.put_env(:pleroma, Pleroma.Captcha, captcha) end) + + ConfigDB.create(%{ + group: ":pleroma", + key: "Pleroma.Captcha", + value: [seconds_valid: 60] + }) + + assert capture_log(fn -> TransferTask.start_link([]) end) =~ "pleroma restarted" + end + + test "don't restart pleroma on reboot time key and subkey if there is false flag" do + chat = Application.get_env(:pleroma, :chat) + captcha = Application.get_env(:pleroma, Pleroma.Captcha) + + on_exit(fn -> + Application.put_env(:pleroma, :chat, chat) + Application.put_env(:pleroma, Pleroma.Captcha, captcha) + end) + + ConfigDB.create(%{ + group: ":pleroma", + key: ":chat", + value: [enabled: false] + }) + + ConfigDB.create(%{ + group: ":pleroma", + key: "Pleroma.Captcha", + value: [seconds_valid: 60] + }) + + assert capture_log(fn -> TransferTask.load_and_update_env([], false) end) =~ "" + end + end end diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 5c767219a..81e346fb8 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -2043,7 +2043,6 @@ test "POST /api/pleroma/admin/config error", %{conn: conn} do Application.delete_env(:pleroma, Pleroma.Captcha.NotReal) Application.put_env(:pleroma, :http, http) Application.put_env(:tesla, :adapter, Tesla.Mock) - :ok = File.rm("config/test.exported_from_db.secret.exs") end) end @@ -2170,7 +2169,7 @@ test "create new config setting in db", %{conn: conn} do assert Application.get_env(:idna, :key5) == {"string", Pleroma.Captcha.NotReal, []} end - test "save config setting without key", %{conn: conn} do + test "save configs setting without explicit key", %{conn: conn} do level = Application.get_env(:quack, :level) meta = Application.get_env(:quack, :meta) webhook_url = Application.get_env(:quack, :webhook_url) @@ -2256,6 +2255,34 @@ test "saving config with partial update", %{conn: conn} do } end + test "saving config which need pleroma reboot", %{conn: conn} do + chat = Pleroma.Config.get(:chat) + on_exit(fn -> Pleroma.Config.put(:chat, chat) end) + + conn = + post( + conn, + "/api/pleroma/admin/config", + %{ + configs: [ + %{group: ":pleroma", key: ":chat", value: [%{"tuple" => [":enabled", true]}]} + ] + } + ) + + assert json_response(conn, 200) == %{ + "configs" => [ + %{ + "db" => [":enabled"], + "group" => ":pleroma", + "key" => ":chat", + "value" => [%{"tuple" => [":enabled", true]}] + } + ], + "need_reboot" => true + } + end + test "saving config with nested merge", %{conn: conn} do config = insert(:config, key: ":key1", value: :erlang.term_to_binary(key1: 1, key2: [k1: 1, k2: 2])) @@ -3001,6 +3028,18 @@ test "returns error if configuration from database is off", %{conn: conn} do end end + describe "GET /api/pleroma/admin/restart" do + clear_config(:configurable_from_database) do + Pleroma.Config.put(:configurable_from_database, true) + end + + test "pleroma restarts", %{conn: conn} do + ExUnit.CaptureLog.capture_log(fn -> + assert conn |> get("/api/pleroma/admin/restart") |> json_response(200) == %{} + end) =~ "pleroma restarted" + end + end + describe "GET /api/pleroma/admin/users/:nickname/statuses" do setup do user = insert(:user) From ac97d01fb6c3eae653ee626e21a62f74362e07cc Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sat, 25 Jan 2020 19:21:21 +0300 Subject: [PATCH 02/13] right test --- test/config/transfer_task_test.exs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/config/transfer_task_test.exs b/test/config/transfer_task_test.exs index 0d328f0c3..61ab1440d 100644 --- a/test/config/transfer_task_test.exs +++ b/test/config/transfer_task_test.exs @@ -119,7 +119,10 @@ test "don't restart if no reboot time settings were changed" do value: [groups: [a: 1, b: 2]] }) - assert capture_log(fn -> TransferTask.start_link([]) end) =~ "" + refute String.contains?( + capture_log(fn -> TransferTask.start_link([]) end), + "pleroma restarted" + ) end test "restart pleroma on reboot time key" do From 7c6e5c541de808957be8a1948fa7a20eba8734df Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 27 Jan 2020 19:48:20 +0300 Subject: [PATCH 03/13] docs update --- docs/API/admin_api.md | 13 +++++++++++++ docs/admin/config.md | 19 ++++++++++++------- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index 07aa7ec3f..2c0c5f46b 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -665,6 +665,19 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - 404 Not Found `"Not found"` - On success: 200 OK `{}` +## `GET /api/pleroma/admin/restart` + +### Restarts pleroma application + +- Params: none +- Response: + - On failure: + - 400 Bad Request `"To use this endpoint you need to enable configuration from database."` + +```json +{} +``` + ## `GET /api/pleroma/admin/config/migrate_from_db` ### Run mix task pleroma.config migrate_from_db diff --git a/docs/admin/config.md b/docs/admin/config.md index 35e43b6a9..b39a73961 100644 --- a/docs/admin/config.md +++ b/docs/admin/config.md @@ -6,11 +6,7 @@ config :pleroma, configurable_from_database: true ``` ## How it works -Settings are stored in database and are applied in `runtime` after each change. Most of the settings take effect immediately, except some, which need instance reboot. These settings are needed in `compile time`, that's why settings are duplicated to the file. - -File with duplicated settings is located in `config/{env}.exported_from_db.exs` if pleroma is runned from source. For prod env it will be `config/prod.exported_from_db.exs`. - -For releases: `/etc/pleroma/prod.exported_from_db.secret.exs` or `PLEROMA_CONFIG_PATH/prod.exported_from_db.exs`. +Settings are stored in database and are applied in `runtime` after each change. Most of the settings take effect immediately, except some, which need instance reboot. ## How to set it up You need to migrate your existing settings to the database. This task will migrate only added by user settings. @@ -25,7 +21,7 @@ You can do this with mix task (all config files will remain untouched): mix pleroma.config migrate_to_db ``` -Now you can change settings in admin interface. After each save, settings from database are duplicated to the `config/{env}.exported_from_db.exs` file. +Now you can change settings in admin interface. If `reboot time` settings were changed, pleroma must be rebooted. **ATTENTION** @@ -35,10 +31,19 @@ Now you can change settings in admin interface. After each save, settings from d - all settings inside these keys: - `:hackney_pools` - `:chat` + - `Oban` + - `:rate_limit` + - `:markup` + - `:streamer` - partially settings inside these keys: - `:seconds_valid` in `Pleroma.Captcha` - `:proxy_remote` in `Pleroma.Upload` - `:upload_limit` in `:instance` + - `:digest` in `:email_notifications` + - `:clean_expired_tokens` in `:oauth2` + - `:enabled` in `Pleroma.ActivityExpiration` + - `:enabled` in `Pleroma.ScheduledActivity` + - `:enabled` in `:gopher` ## How to dump settings from database to file @@ -59,7 +64,7 @@ mix pleroma.config migrate_from_db [-d] ```sql TRUNCATE TABLE config; ``` -2. Delete `config/{env}.exported_from_db.exs`. +2. If migrate_from_db task was runned, backup and delete `config/{env}.exported_from_db.exs`. For `prod` env: ```bash From 91ea3ed82cc29f02ae8eec94e4e4ced325a7e008 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 28 Jan 2020 15:19:05 +0300 Subject: [PATCH 04/13] moving restarter application into pleroma repo --- mix.exs | 2 +- restarter/lib/pleroma.ex | 28 ++++++++++++++++++++++++++++ restarter/lib/restarter.ex | 8 ++++++++ restarter/mix .exs | 21 +++++++++++++++++++++ 4 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 restarter/lib/pleroma.ex create mode 100644 restarter/lib/restarter.ex create mode 100644 restarter/mix .exs diff --git a/mix.exs b/mix.exs index 1d0b59e56..2608f986e 100644 --- a/mix.exs +++ b/mix.exs @@ -172,7 +172,7 @@ defp deps do git: "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"}, {:mox, "~> 0.5", only: :test}, - {:restarter, git: "https://git.pleroma.social/alex.s/restarter"} + {:restarter, path: "../restarter"} ] ++ oauth_deps() end diff --git a/restarter/lib/pleroma.ex b/restarter/lib/pleroma.ex new file mode 100644 index 000000000..da714654c --- /dev/null +++ b/restarter/lib/pleroma.ex @@ -0,0 +1,28 @@ +defmodule Restarter.Pleroma do + use GenServer + + def start_link(_) do + GenServer.start_link(__MODULE__, [], name: __MODULE__) + end + + def init(_), do: {:ok, %{}} + + def handle_info(:after_boot, %{after_boot: true} = state), do: {:noreply, state} + + def handle_info(:after_boot, state) do + restart(:pleroma) + {:noreply, Map.put(state, :after_boot, true)} + end + + def handle_info({:restart, delay}, state) do + Process.sleep(delay) + restart(:pleroma) + {:noreply, state} + end + + defp restart(app) do + :ok = Application.ensure_started(app) + :ok = Application.stop(app) + :ok = Application.start(app) + end +end diff --git a/restarter/lib/restarter.ex b/restarter/lib/restarter.ex new file mode 100644 index 000000000..eadd86f89 --- /dev/null +++ b/restarter/lib/restarter.ex @@ -0,0 +1,8 @@ +defmodule Restarter do + use Application + + def start(_, _) do + opts = [strategy: :one_for_one, name: Restarter.Supervisor] + Supervisor.start_link([Restarter.Pleroma], opts) + end +end diff --git a/restarter/mix .exs b/restarter/mix .exs new file mode 100644 index 000000000..b0908aece --- /dev/null +++ b/restarter/mix .exs @@ -0,0 +1,21 @@ +defmodule Restarter.MixProject do + use Mix.Project + + def project do + [ + app: :restarter, + version: "0.1.0", + elixir: "~> 1.8", + start_permanent: Mix.env() == :prod, + deps: deps() + ] + end + + def application do + [ + mod: {Restarter, []} + ] + end + + defp deps, do: [] +end From aa3ba635131fb05b09ab487bdf68cbe5dda98257 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 28 Jan 2020 15:41:45 +0300 Subject: [PATCH 05/13] like this --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 2608f986e..ad23c7414 100644 --- a/mix.exs +++ b/mix.exs @@ -172,7 +172,7 @@ defp deps do git: "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"}, {:mox, "~> 0.5", only: :test}, - {:restarter, path: "../restarter"} + {:restarter, in_umbrella: true} ] ++ oauth_deps() end From 251bbd794a696655bee5df7a31727db7dfa60375 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 28 Jan 2020 13:29:44 +0000 Subject: [PATCH 06/13] Apply suggestion to mix.exs --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index ad23c7414..8cbea6f75 100644 --- a/mix.exs +++ b/mix.exs @@ -172,7 +172,7 @@ defp deps do git: "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"}, {:mox, "~> 0.5", only: :test}, - {:restarter, in_umbrella: true} + {:restarter, path: "./restarter"} ] ++ oauth_deps() end From da334ef9343b3ff96bf5802bc7e8864566e612dc Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 28 Jan 2020 13:29:48 +0000 Subject: [PATCH 07/13] Apply suggestion to mix.lock --- mix.lock | 1 - 1 file changed, 1 deletion(-) diff --git a/mix.lock b/mix.lock index 538337cf3..c1fe223c0 100644 --- a/mix.lock +++ b/mix.lock @@ -94,7 +94,6 @@ "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]}, "remote_ip": {:git, "https://git.pleroma.social/pleroma/remote_ip.git", "825dc00aaba5a1b7c4202a532b696b595dd3bcb3", [ref: "825dc00aaba5a1b7c4202a532b696b595dd3bcb3"]}, - "restarter": {:git, "https://git.pleroma.social/alex.s/restarter", "1932655b80a1409405d897911c06ebee4ac8c2d8", []}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.5", "6eaf7ad16cb568bb01753dbbd7a95ff8b91c7979482b95f38443fe2c8852a79b", [:make, :mix, :rebar3], [], "hexpm"}, "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm"}, "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"}, From 93b9f226e0163519b0bbb111191abef45893f23e Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 28 Jan 2020 16:32:33 +0300 Subject: [PATCH 08/13] path fix --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 8cbea6f75..2608f986e 100644 --- a/mix.exs +++ b/mix.exs @@ -172,7 +172,7 @@ defp deps do git: "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"}, {:mox, "~> 0.5", only: :test}, - {:restarter, path: "./restarter"} + {:restarter, path: "../restarter"} ] ++ oauth_deps() end From 33bd8fbffea79b8ca510a098ad4654b8f01324d6 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 28 Jan 2020 18:02:11 +0300 Subject: [PATCH 09/13] filename and test fixes --- mix.exs | 2 +- restarter/{mix .exs => mix.exs} | 0 test/config/transfer_task_test.exs | 5 ++++- 3 files changed, 5 insertions(+), 2 deletions(-) rename restarter/{mix .exs => mix.exs} (100%) diff --git a/mix.exs b/mix.exs index 2608f986e..8cbea6f75 100644 --- a/mix.exs +++ b/mix.exs @@ -172,7 +172,7 @@ defp deps do git: "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"}, {:mox, "~> 0.5", only: :test}, - {:restarter, path: "../restarter"} + {:restarter, path: "./restarter"} ] ++ oauth_deps() end diff --git a/restarter/mix .exs b/restarter/mix.exs similarity index 100% rename from restarter/mix .exs rename to restarter/mix.exs diff --git a/test/config/transfer_task_test.exs b/test/config/transfer_task_test.exs index 61ab1440d..ebdc951cf 100644 --- a/test/config/transfer_task_test.exs +++ b/test/config/transfer_task_test.exs @@ -172,7 +172,10 @@ test "don't restart pleroma on reboot time key and subkey if there is false flag value: [seconds_valid: 60] }) - assert capture_log(fn -> TransferTask.load_and_update_env([], false) end) =~ "" + refute String.contains?( + capture_log(fn -> TransferTask.load_and_update_env([], false) end), + "pleroma restarted" + ) end end end From 50f5a920219d6637582a1998fd33ec4552e02e9c Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 2 Feb 2020 14:55:06 +0300 Subject: [PATCH 10/13] fix not being able to pin polls --- lib/pleroma/web/common_api/common_api.ex | 3 ++- test/web/common_api/common_api_test.exs | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index c05a6c544..2a348dcf6 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -315,8 +315,9 @@ def pin(id_or_ap_id, %{ap_id: user_ap_id} = user) do with %Activity{ actor: ^user_ap_id, data: %{"type" => "Create"}, - object: %Object{data: %{"type" => "Note"}} + object: %Object{data: %{"type" => object_type}} } = activity <- get_by_id_or_ap_id(id_or_ap_id), + true <- object_type in ["Note", "Article", "Question"], true <- Visibility.is_public?(activity), {:ok, _user} <- User.add_pinnned_activity(user, activity) do {:ok, activity} diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 8fa0c6faa..214cbdd7c 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -324,6 +324,21 @@ test "pin status", %{user: user, activity: activity} do assert %User{pinned_activities: [^id]} = user end + test "pin poll", %{user: user} do + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "How is fediverse today?", + "poll" => %{"options" => ["Absolutely outstanding", "Not good"], "expires_in" => 20} + }) + + assert {:ok, ^activity} = CommonAPI.pin(activity.id, user) + + id = activity.id + user = refresh_record(user) + + assert %User{pinned_activities: [^id]} = user + end + test "unlisted statuses can be pinned", %{user: user} do {:ok, activity} = CommonAPI.post(user, %{"status" => "HI!!!", "visibility" => "unlisted"}) assert {:ok, ^activity} = CommonAPI.pin(activity.id, user) From 3909b5b7b34bbf879fd7809eef6ec0ae286c2ef2 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 5 Feb 2020 21:13:56 +0400 Subject: [PATCH 11/13] Remove AutoLinker `scheme` option from the config --- config/config.exs | 1 - config/description.exs | 5 ----- lib/pleroma/formatter.ex | 3 ++- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/config/config.exs b/config/config.exs index c57ef1bf7..d01dd8d74 100644 --- a/config/config.exs +++ b/config/config.exs @@ -508,7 +508,6 @@ config :auto_linker, opts: [ - scheme: true, extra: true, # TODO: Set to :no_scheme when it works properly validate_tld: true, diff --git a/config/description.exs b/config/description.exs index 5f3c58b08..a0cb03e48 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2171,11 +2171,6 @@ type: :boolean, description: "Set to `false` to remove target='_blank' attribute" }, - %{ - key: :scheme, - type: :boolean, - description: "Set to `true` to link urls with schema http://google.com" - }, %{ key: :truncate, type: [:integer, false], diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 19b9af46c..90895374d 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -13,7 +13,8 @@ defmodule Pleroma.Formatter do @auto_linker_config hashtag: true, hashtag_handler: &Pleroma.Formatter.hashtag_handler/4, mention: true, - mention_handler: &Pleroma.Formatter.mention_handler/4 + mention_handler: &Pleroma.Formatter.mention_handler/4, + scheme: true def escape_mention_handler("@" <> nickname = mention, buffer, _, _) do case User.get_cached_by_nickname(nickname) do From 5db6ac8ee405d89943a3669da4ea154ce004860f Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 5 Feb 2020 20:36:21 +0300 Subject: [PATCH 12/13] removing migrate_from_db endpoint from admin api --- CHANGELOG.md | 1 + docs/API/admin_api.md | 15 ------- .../web/admin_api/admin_api_controller.ex | 15 +------ lib/pleroma/web/router.ex | 1 - .../admin_api/admin_api_controller_test.exs | 44 ------------------- 5 files changed, 2 insertions(+), 74 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 713ae4361..b146ace46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **Breaking**: OStatus protocol support - **Breaking**: MDII uploader - **Breaking**: Using third party engines for user recommendation +- **Breaking**: AdminAPI: migrate_from_db endpoint. ### Changed - **Breaking:** Pleroma won't start if it detects unapplied migrations diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index 2c0c5f46b..e445583cb 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -678,21 +678,6 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret {} ``` -## `GET /api/pleroma/admin/config/migrate_from_db` - -### Run mix task pleroma.config migrate_from_db - -Copies all settings from database to `config/{env}.exported_from_db.secret.exs` with deletion from the table. Where `{env}` is the environment in which `pleroma` is running. - -- Params: none -- Response: - - On failure: - - 400 Bad Request `"To use this endpoint you need to enable configuration from database."` - -```json -{} -``` - ## `GET /api/pleroma/admin/config` ### Get list of merged default settings with saved in database. diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 6f0449418..293f1befc 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -97,7 +97,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do plug( OAuthScopesPlug, %{scopes: ["read"], admin: true} - when action in [:config_show, :migrate_from_db, :list_log] + when action in [:config_show, :list_log] ) plug( @@ -793,19 +793,6 @@ def config_descriptions(conn, _params) do |> Plug.Conn.send_resp(200, @descriptions_json) end - def migrate_from_db(conn, _params) do - with :ok <- configurable_from_database(conn) do - Mix.Tasks.Pleroma.Config.run([ - "migrate_from_db", - "--env", - to_string(Pleroma.Config.get(:env)), - "-d" - ]) - - json(conn, %{}) - end - end - def config_show(conn, %{"only_db" => true}) do with :ok <- configurable_from_database(conn) do configs = Pleroma.Repo.all(ConfigDB) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 078bf138c..e86bc3cc3 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -196,7 +196,6 @@ defmodule Pleroma.Web.Router do get("/config", AdminAPIController, :config_show) post("/config", AdminAPIController, :config_update) get("/config/descriptions", AdminAPIController, :config_descriptions) - get("/config/migrate_from_db", AdminAPIController, :migrate_from_db) get("/restart", AdminAPIController, :restart) get("/moderation_log", AdminAPIController, :list_log) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 81e346fb8..87f1366a4 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -2984,50 +2984,6 @@ test "proxy tuple ip", %{conn: conn} do end end - describe "config mix tasks run" do - setup do - Mix.shell(Mix.Shell.Quiet) - - on_exit(fn -> - Mix.shell(Mix.Shell.IO) - end) - - :ok - end - - clear_config(:configurable_from_database) do - Pleroma.Config.put(:configurable_from_database, true) - end - - clear_config([:feed, :post_title]) do - Pleroma.Config.put([:feed, :post_title], %{max_length: 100, omission: "…"}) - end - - test "transfer settings to DB and to file", %{conn: conn} do - assert Repo.all(Pleroma.ConfigDB) == [] - Mix.Tasks.Pleroma.Config.migrate_to_db("test/fixtures/config/temp.secret.exs") - assert Repo.aggregate(Pleroma.ConfigDB, :count, :id) > 0 - - conn = get(conn, "/api/pleroma/admin/config/migrate_from_db") - - assert json_response(conn, 200) == %{} - assert Repo.all(Pleroma.ConfigDB) == [] - end - - test "returns error if configuration from database is off", %{conn: conn} do - initial = Pleroma.Config.get(:configurable_from_database) - on_exit(fn -> Pleroma.Config.put(:configurable_from_database, initial) end) - Pleroma.Config.put(:configurable_from_database, false) - - conn = get(conn, "/api/pleroma/admin/config/migrate_from_db") - - assert json_response(conn, 400) == - "To use this endpoint you need to enable configuration from database." - - assert Repo.all(Pleroma.ConfigDB) == [] - end - end - describe "GET /api/pleroma/admin/restart" do clear_config(:configurable_from_database) do Pleroma.Config.put(:configurable_from_database, true) From 2a3e06eb709966204c1723258b82644a7ac8d11e Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 5 Feb 2020 18:28:05 +0000 Subject: [PATCH 13/13] Apply suggestion to CHANGELOG.md --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b146ace46..1b6ba53d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **Breaking**: OStatus protocol support - **Breaking**: MDII uploader - **Breaking**: Using third party engines for user recommendation -- **Breaking**: AdminAPI: migrate_from_db endpoint. +
+ API Changes +- **Breaking**: AdminAPI: migrate_from_db endpoint +
### Changed - **Breaking:** Pleroma won't start if it detects unapplied migrations