From 6610a1d5fbc8e92c3840d7c1b7de541643199546 Mon Sep 17 00:00:00 2001 From: Francis Dinh Date: Thu, 22 Dec 2022 19:18:54 -0500 Subject: [PATCH 01/22] docs/installation: update comment to reflect flavour change The comment still says the flavour is `amd64-musl` when it was updated to just `amd64` in 64ccdadad318c4a75fea4875009fb05895f6b93a. --- docs/docs/installation/otp_en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/installation/otp_en.md b/docs/docs/installation/otp_en.md index 1b5c1270e..ddbb66d96 100644 --- a/docs/docs/installation/otp_en.md +++ b/docs/docs/installation/otp_en.md @@ -118,7 +118,7 @@ Restart PostgreSQL to apply configuration changes: adduser --system --shell /bin/false --home /opt/akkoma akkoma # Set the flavour environment variable to the string you got in Detecting flavour section. -# For example if the flavour is `amd64-musl` the command will be +# For example if the flavour is `amd64` the command will be export FLAVOUR="amd64" # Clone the release build into a temporary directory and unpack it From 03a00d005a74c1f74f403ca31dff0b3eee20d386 Mon Sep 17 00:00:00 2001 From: Francis Dinh Date: Thu, 22 Dec 2022 19:27:16 -0500 Subject: [PATCH 02/22] remove comment about old openssl versions in nginx config I doubt many people are actually still using OpenSSL 1.0.2 or older, since that version was first released in 2015, and last updated in 2019. --- installation/nginx/akkoma.nginx | 2 -- 1 file changed, 2 deletions(-) diff --git a/installation/nginx/akkoma.nginx b/installation/nginx/akkoma.nginx index 772716677..18d92f30f 100644 --- a/installation/nginx/akkoma.nginx +++ b/installation/nginx/akkoma.nginx @@ -54,8 +54,6 @@ server { ssl_protocols TLSv1.2 TLSv1.3; ssl_ciphers "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4"; ssl_prefer_server_ciphers off; - # In case of an old server with an OpenSSL version of 1.0.2 or below, - # leave only prime256v1 or comment out the following line. ssl_ecdh_curve X25519:prime256v1:secp384r1:secp521r1; ssl_stapling on; ssl_stapling_verify on; From 5a6fa6717b26e240eef3a446d497e05ade1bbd33 Mon Sep 17 00:00:00 2001 From: Norm Date: Fri, 23 Dec 2022 18:03:14 +0000 Subject: [PATCH 03/22] Don't treat js/css as binary in git anymore Since Akkoma doesn't include precompiled frontends in the main repo anymore, it doesn't make sense to keep treating the few js/css files remaining as binary files. --- .gitattributes | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.gitattributes b/.gitattributes index ac67c53c2..febafe62f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,10 +1,4 @@ *.ex diff=elixir *.exs diff=elixir -# Most of js/css files included in the repo are minified bundles, -# and we don't want to search/diff those as text files. -*.js binary -*.js.map binary -*.css binary - *.css diff=css From e392662d76779946e12a1ce02dc85394cea9ac37 Mon Sep 17 00:00:00 2001 From: acuteaura Date: Sun, 25 Dec 2022 15:32:57 +0000 Subject: [PATCH 04/22] docs: fedora install errata --- docs/docs/installation/fedora_based_en.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/docs/installation/fedora_based_en.md b/docs/docs/installation/fedora_based_en.md index d8c7b3e74..0566d6507 100644 --- a/docs/docs/installation/fedora_based_en.md +++ b/docs/docs/installation/fedora_based_en.md @@ -30,11 +30,10 @@ sudo dnf install git gcc g++ make cmake file-devel postgresql-server postgresql- * Enable and initialize Postgres: ```shell -sudo systemctl enable postgresql.service sudo postgresql-setup --initdb --unit postgresql # Allow password auth for postgres sudo sed -E -i 's|(host +all +all +127.0.0.1/32 +)ident|\1md5|' /var/lib/pgsql/data/pg_hba.conf -sudo systemctl start postgresql.service +sudo systemctl enable --now postgresql.service ``` ### Install Elixir and Erlang @@ -59,7 +58,7 @@ sudo dnf install ffmpeg * Install ImageMagick and ExifTool for image manipulation: ```shell -sudo dnf install Imagemagick perl-Image-ExifTool +sudo dnf install ImageMagick perl-Image-ExifTool ``` From 11ec4e1b8f596f67713e94171832fb01e0bc15db Mon Sep 17 00:00:00 2001 From: Stefan Date: Fri, 16 Dec 2022 17:23:31 +0100 Subject: [PATCH 05/22] clean-up docs to avoid mismatches in BE and FE. Clearly state that stable-versions are installed --- docs/docs/administration/CLI_tasks/frontend.md | 6 +++--- docs/docs/administration/updating.md | 4 ++-- docs/docs/installation/alpine_linux_en.md | 6 +++--- docs/docs/installation/arch_linux_en.md | 6 +++--- docs/docs/installation/debian_based_en.md | 6 +++--- docs/docs/installation/fedora_based_en.md | 6 +++--- docs/docs/installation/otp_redhat_en.md | 2 +- docs/docs/installation/verifying_otp_releases.md | 2 +- 8 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/docs/administration/CLI_tasks/frontend.md b/docs/docs/administration/CLI_tasks/frontend.md index 5e87f1227..382ac268e 100644 --- a/docs/docs/administration/CLI_tasks/frontend.md +++ b/docs/docs/administration/CLI_tasks/frontend.md @@ -24,20 +24,20 @@ Currently, known `` values are: You can still install frontends that are not configured, see below. -## Example installations for a known frontend +## Example installations for a known frontend (Stable-Version) For a frontend configured under the `available` key, it's enough to install it by name. === "OTP" ```sh - ./bin/pleroma_ctl frontend install pleroma-fe + ./bin/pleroma_ctl frontend install pleroma-fe --ref stable ``` === "From Source" ```sh - mix pleroma.frontend install pleroma-fe + mix pleroma.frontend install pleroma-fe --ref stable ``` This will download the latest build for the pre-configured `ref` and install it. It can then be configured as the one of the served frontends in the config file (see `primary` or `admin`). diff --git a/docs/docs/administration/updating.md b/docs/docs/administration/updating.md index 52979a1f5..9f11e40b7 100644 --- a/docs/docs/administration/updating.md +++ b/docs/docs/administration/updating.md @@ -1,6 +1,6 @@ # Updating your instance -You should **always check the [release notes/changelog](https://akkoma.dev/AkkomaGang/akkoma/src/branch/develop/CHANGELOG.md)** in case there are config deprecations, special update steps, etc. +You should **always check the [release notes/changelog](https://akkoma.dev/AkkomaGang/akkoma/src/branch/stable/CHANGELOG.md)** in case there are config deprecations, special update steps, etc. Besides that, doing the following is generally enough: ## Switch to the akkoma user @@ -57,7 +57,7 @@ sudo systemctl stop akkoma # Run database migrations mix ecto.migrate -# Update frontend(s). See Frontend Configration doc for more information. +# Update Pleroma-FE frontend to latest stable. For other Frontends see Frontend Configration doc for more information. mix pleroma.frontend install pleroma-fe --ref stable # Start akkoma (replace with your system service manager's equivalent if different) diff --git a/docs/docs/installation/alpine_linux_en.md b/docs/docs/installation/alpine_linux_en.md index aae8f9626..bdfb96d77 100644 --- a/docs/docs/installation/alpine_linux_en.md +++ b/docs/docs/installation/alpine_linux_en.md @@ -84,12 +84,12 @@ doas adduser -S -s /bin/false -h /opt/akkoma -H -G akkoma akkoma **Note**: To execute a single command as the Akkoma system user, use `doas -u akkoma command`. You can also switch to a shell by using `doas -su akkoma`. If you don’t have and want `doas` on your system, you can use `su` as root user (UID 0) for a single command by using `su -l akkoma -s $SHELL -c 'command'` and `su -l akkoma -s $SHELL` for starting a shell. -* Git clone the AkkomaBE repository and make the Akkoma user the owner of the directory: +* Git clone the AkkomaBE repository from stable-branch and make the Akkoma user the owner of the directory: ```shell doas mkdir -p /opt/akkoma doas chown -R akkoma:akkoma /opt/akkoma -doas -u akkoma git clone https://akkoma.dev/AkkomaGang/akkoma.git /opt/akkoma +doas -u akkoma git clone https://akkoma.dev/AkkomaGang/akkoma.git -b stable /opt/akkoma ``` * Change to the new directory: @@ -109,7 +109,7 @@ doas -u akkoma mix deps.get * This may take some time, because parts of akkoma get compiled first. * After that it will ask you a few questions about your instance and generates a configuration file in `config/generated_config.exs`. -* Check the configuration and if all looks right, rename it, so Akkoma will load it (`prod.secret.exs` for productive instance, `dev.secret.exs` for development instances): +* Check the configuration and if all looks right, rename it, so Akkoma will load it (`prod.secret.exs` for productive instances): ```shell doas -u akkoma mv config/{generated_config.exs,prod.secret.exs} diff --git a/docs/docs/installation/arch_linux_en.md b/docs/docs/installation/arch_linux_en.md index 639c9c798..300a5d80f 100644 --- a/docs/docs/installation/arch_linux_en.md +++ b/docs/docs/installation/arch_linux_en.md @@ -75,12 +75,12 @@ sudo useradd -r -s /bin/false -m -d /var/lib/akkoma -U akkoma **Note**: To execute a single command as the Akkoma system user, use `sudo -Hu akkoma command`. You can also switch to a shell by using `sudo -Hu akkoma $SHELL`. If you don’t have and want `sudo` on your system, you can use `su` as root user (UID 0) for a single command by using `su -l akkoma -s $SHELL -c 'command'` and `su -l akkoma -s $SHELL` for starting a shell. -* Git clone the AkkomaBE repository and make the Akkoma user the owner of the directory: +* Git clone the AkkomaBE repository from stable-branch and make the Akkoma user the owner of the directory: ```shell sudo mkdir -p /opt/akkoma sudo chown -R akkoma:akkoma /opt/akkoma -sudo -Hu akkoma git clone https://akkoma.dev/AkkomaGang/akkoma.git /opt/akkoma +sudo -Hu akkoma git clone https://akkoma.dev/AkkomaGang/akkoma.git -b stable /opt/akkoma ``` * Change to the new directory: @@ -100,7 +100,7 @@ sudo -Hu akkoma mix deps.get * This may take some time, because parts of akkoma get compiled first. * After that it will ask you a few questions about your instance and generates a configuration file in `config/generated_config.exs`. -* Check the configuration and if all looks right, rename it, so Akkoma will load it (`prod.secret.exs` for productive instance, `dev.secret.exs` for development instances): +* Check the configuration and if all looks right, rename it, so Akkoma will load it (`prod.secret.exs` for productive instances): ```shell sudo -Hu akkoma mv config/{generated_config.exs,prod.secret.exs} diff --git a/docs/docs/installation/debian_based_en.md b/docs/docs/installation/debian_based_en.md index 139c789bc..265658fef 100644 --- a/docs/docs/installation/debian_based_en.md +++ b/docs/docs/installation/debian_based_en.md @@ -49,12 +49,12 @@ sudo useradd -r -s /bin/false -m -d /var/lib/akkoma -U akkoma **Note**: To execute a single command as the Akkoma system user, use `sudo -Hu akkoma command`. You can also switch to a shell by using `sudo -Hu akkoma $SHELL`. If you don’t have and want `sudo` on your system, you can use `su` as root user (UID 0) for a single command by using `su -l akkoma -s $SHELL -c 'command'` and `su -l akkoma -s $SHELL` for starting a shell. -* Git clone the AkkomaBE repository and make the Akkoma user the owner of the directory: +* Git clone the AkkomaBE repository from stable-branch and make the Akkoma user the owner of the directory: ```shell sudo mkdir -p /opt/akkoma sudo chown -R akkoma:akkoma /opt/akkoma -sudo -Hu akkoma git clone https://akkoma.dev/AkkomaGang/akkoma.git /opt/akkoma +sudo -Hu akkoma git clone https://akkoma.dev/AkkomaGang/akkoma.git -b stable /opt/akkoma ``` * Change to the new directory: @@ -74,7 +74,7 @@ sudo -Hu akkoma mix deps.get * This may take some time, because parts of akkoma get compiled first. * After that it will ask you a few questions about your instance and generates a configuration file in `config/generated_config.exs`. -* Check the configuration and if all looks right, rename it, so Akkoma will load it (`prod.secret.exs` for productive instance, `dev.secret.exs` for development instances): +* Check the configuration and if all looks right, rename it, so Akkoma will load it (`prod.secret.exs` for productive instances): ```shell sudo -Hu akkoma mv config/{generated_config.exs,prod.secret.exs} diff --git a/docs/docs/installation/fedora_based_en.md b/docs/docs/installation/fedora_based_en.md index d8c7b3e74..a24a870a4 100644 --- a/docs/docs/installation/fedora_based_en.md +++ b/docs/docs/installation/fedora_based_en.md @@ -74,12 +74,12 @@ sudo useradd -r -s /bin/false -m -d /var/lib/akkoma -U akkoma **Note**: To execute a single command as the Akkoma system user, use `sudo -Hu akkoma command`. You can also switch to a shell by using `sudo -Hu akkoma $SHELL`. If you don’t have and want `sudo` on your system, you can use `su` as root user (UID 0) for a single command by using `su -l akkoma -s $SHELL -c 'command'` and `su -l akkoma -s $SHELL` for starting a shell. -* Git clone the AkkomaBE repository and make the Akkoma user the owner of the directory: +* Git clone the AkkomaBE repository from stable-branch and make the Akkoma user the owner of the directory: ```shell sudo mkdir -p /opt/akkoma sudo chown -R akkoma:akkoma /opt/akkoma -sudo -Hu akkoma git clone https://akkoma.dev/AkkomaGang/akkoma.git /opt/akkoma +sudo -Hu akkoma git clone https://akkoma.dev/AkkomaGang/akkoma.git -b stable /opt/akkoma ``` * Change to the new directory: @@ -99,7 +99,7 @@ sudo -Hu akkoma mix deps.get * This may take some time, because parts of akkoma get compiled first. * After that it will ask you a few questions about your instance and generates a configuration file in `config/generated_config.exs`. -* Check the configuration and if all looks right, rename it, so Akkoma will load it (`prod.secret.exs` for productive instance, `dev.secret.exs` for development instances): +* Check the configuration and if all looks right, rename it, so Akkoma will load it (`prod.secret.exs` for productive instances): ```shell sudo -Hu akkoma mv config/{generated_config.exs,prod.secret.exs} diff --git a/docs/docs/installation/otp_redhat_en.md b/docs/docs/installation/otp_redhat_en.md index ec6c30bcf..1490d3139 100644 --- a/docs/docs/installation/otp_redhat_en.md +++ b/docs/docs/installation/otp_redhat_en.md @@ -37,7 +37,7 @@ sudo dnf install git gcc g++ erlang elixir erlang-os_mon erlang-eldap erlang-xme ```shell cd ~ -git clone https://akkoma.dev/AkkomaGang/akkoma.git +git clone https://akkoma.dev/AkkomaGang/akkoma.git -b stable ``` * Change to the new directory: diff --git a/docs/docs/installation/verifying_otp_releases.md b/docs/docs/installation/verifying_otp_releases.md index 5f1ac6949..6e3c6f8ca 100644 --- a/docs/docs/installation/verifying_otp_releases.md +++ b/docs/docs/installation/verifying_otp_releases.md @@ -12,7 +12,7 @@ Release URLs will always be of the form https://akkoma-updates.s3-website.fr-par.scw.cloud/{branch}/akkoma-{flavour}.zip ``` -Where branch is usually `stable` or `develop`, and `flavour` is +Where branch is usually `stable` and `flavour` is the one [that you detect on install](../otp_en/#detecting-flavour). So, for an AMD64 stable install, your update URL will be From e66bcb64a4691383c9253c99b9863a0cc6856415 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Thu, 29 Dec 2022 15:42:25 +0000 Subject: [PATCH 06/22] Check out the latest tag on update --- docs/docs/administration/updating.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/docs/administration/updating.md b/docs/docs/administration/updating.md index 9f11e40b7..d0c955e1a 100644 --- a/docs/docs/administration/updating.md +++ b/docs/docs/administration/updating.md @@ -41,8 +41,10 @@ you _may_ need to specify `--flavour`, in the same way as Run as the `akkoma` user: ```sh -# Pull in new changes -git pull +# fetch changes +git fetch +# check out the latest tag +git checkout $(git tag -l | grep -v 'rc[0-9]*$' | sort -V | tail -n 1) # Run with production configuration export MIX_ENV=prod From d1bf8aa9edfd53d46d3e40c92acbc57784910979 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Thu, 29 Dec 2022 19:56:35 +0000 Subject: [PATCH 07/22] Add dump_to_file and load_from_file tasks --- lib/mix/tasks/pleroma/config.ex | 37 +++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index c259a6cbd..e70ca709a 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -79,6 +79,43 @@ def run(["dump", group]) do end) end + def run(["dump_to_file", group, key]) do + check_configdb(fn -> + start_pleroma() + + group = maybe_atomize(group) + key = maybe_atomize(key) + + config = ConfigDB.get_by_group_and_key(group, key) + json = %{ + group: ConfigDB.to_json_types(config.group), + key: ConfigDB.to_json_types(config.key), + value: ConfigDB.to_json_types(config.value), + } + |> Jason.encode!() + |> Jason.Formatter.pretty_print() + + File.write("#{group}_#{key}.json", json) + shell_info("Wrote #{group}_#{key}.json") + end) + end + + def run(["load_from_file", fname]) do + check_configdb(fn -> + start_pleroma() + + json = File.read!(fname) + config = Jason.decode!(json) + group = ConfigDB.to_elixir_types(config["group"]) + key = ConfigDB.to_elixir_types(config["key"]) + value = ConfigDB.to_elixir_types(config["value"]) + params = %{group: group, key: key, value: value} + + ConfigDB.update_or_create(params) + shell_info("Loaded #{config["group"]}, #{config["key"]}") + end) + end + def run(["groups"]) do check_configdb(fn -> start_pleroma() From 5a405bdadfbad41f08b609925300255b856c2f79 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Thu, 29 Dec 2022 20:00:04 +0000 Subject: [PATCH 08/22] document dump_to_file and load_from_file --- docs/docs/administration/CLI_tasks/config.md | 39 ++++++++++++++++++++ lib/mix/tasks/pleroma/config.ex | 4 +- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/docs/docs/administration/CLI_tasks/config.md b/docs/docs/administration/CLI_tasks/config.md index a0199d06f..490d6315f 100644 --- a/docs/docs/administration/CLI_tasks/config.md +++ b/docs/docs/administration/CLI_tasks/config.md @@ -155,3 +155,42 @@ This forcibly removes all saved values in the database. ```sh mix pleroma.config [--force] reset ``` + +## Dumping specific configuration values to JSON + +If you want to bulk-modify configuration values (for example, for MRF modifications), +it may be easier to dump the values to JSON and then modify them in a text editor. + +=== "OTP" + + ```sh + ./bin/pleroma_ctl config dump_to_file group key path + # For example, to dump the MRF simple configuration: + ./bin/pleroma_ctl config dump_to_file pleroma mrf_simple /tmp/mrf_simple.json + ``` + +=== "From Source" + + ```sh + mix pleroma.config dump_to_file group key path + # For example, to dump the MRF simple configuration: + mix pleroma.config dump_to_file pleroma mrf_simple /tmp/mrf_simple.json + ``` + +## Loading specific configuration values from JSON + +=== "OTP" + + ```sh + ./bin/pleroma_ctl config load_from_file path + # For example, to load the MRF simple configuration: + ./bin/pleroma_ctl config load_from_file /tmp/mrf_simple.json + ``` + +=== "From Source" + + ```sh + mix pleroma.config load_from_file path + # For example, to load the MRF simple configuration: + mix pleroma.config load_from_file /tmp/mrf_simple.json + ``` diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index e70ca709a..dfde6520d 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -79,7 +79,7 @@ def run(["dump", group]) do end) end - def run(["dump_to_file", group, key]) do + def run(["dump_to_file", group, key, fname]) do check_configdb(fn -> start_pleroma() @@ -95,7 +95,7 @@ def run(["dump_to_file", group, key]) do |> Jason.encode!() |> Jason.Formatter.pretty_print() - File.write("#{group}_#{key}.json", json) + File.write(fname, json) shell_info("Wrote #{group}_#{key}.json") end) end From 1121deb0784ab0def083e1aa5c744fda62bcee5c Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Thu, 29 Dec 2022 20:24:04 +0000 Subject: [PATCH 09/22] Document instance reboots --- docs/docs/administration/CLI_tasks/config.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/docs/administration/CLI_tasks/config.md b/docs/docs/administration/CLI_tasks/config.md index 490d6315f..31e5af401 100644 --- a/docs/docs/administration/CLI_tasks/config.md +++ b/docs/docs/administration/CLI_tasks/config.md @@ -179,6 +179,11 @@ it may be easier to dump the values to JSON and then modify them in a text edito ## Loading specific configuration values from JSON +**Note:** This will overwrite any existing value in the database, and can +cause crashes if you do not have exactly the correct formatting. + +Once you have modified the JSON file, you can load it back into the database. + === "OTP" ```sh @@ -194,3 +199,7 @@ it may be easier to dump the values to JSON and then modify them in a text edito # For example, to load the MRF simple configuration: mix pleroma.config load_from_file /tmp/mrf_simple.json ``` + +**NOTE** an instance reboot is needed for many changes to take effect, +you may want to visit `/api/v1/pleroma/admin/restart` on your instance +to soft-restart the instance. From a5e98083f253c268bb1468bfaf358038e0e60147 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Thu, 29 Dec 2022 20:56:06 +0000 Subject: [PATCH 10/22] Add link verification in profile fields (#405) Co-authored-by: FloatingGhost Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/405 --- CHANGELOG.md | 2 + lib/pleroma/application.ex | 3 +- lib/pleroma/user.ex | 84 +++++++++++++++++-- lib/pleroma/web/rel_me.ex | 3 +- .../mastodon_api/update_credentials_test.exs | 63 ++++++++++++++ 5 files changed, 144 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d556b39c3..ea6a25e4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Prometheus metrics exporting from `/api/v1/akkoma/metrics` - Ability to alter http pool size - Translation of statuses via ArgosTranslate +- Ability to "verify" links in profile fields via rel=me +- Mix tasks to dump/load config to/from json for bulk editing ### Removed - Non-finch HTTP adapters diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 26b500dc8..0273972be 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -159,7 +159,8 @@ defp cachex_children do build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000), build_cachex("translations", default_ttl: :timer.hours(24 * 30), limit: 2500), build_cachex("instances", default_ttl: :timer.hours(24), ttl_interval: 1000, limit: 2500), - build_cachex("request_signatures", default_ttl: :timer.hours(24 * 30), limit: 3000) + build_cachex("request_signatures", default_ttl: :timer.hours(24 * 30), limit: 3000), + build_cachex("rel_me", default_ttl: :timer.hours(24 * 30), limit: 300) ] end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index d7c1511ce..2d4bd097d 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -479,7 +479,7 @@ def remote_user_changeset(struct \\ %User{local: false}, params) do |> validate_format(:nickname, @email_regex) |> validate_length(:bio, max: bio_limit) |> validate_length(:name, max: name_limit) - |> validate_fields(true) + |> validate_fields(true, struct) |> validate_non_local() end @@ -549,7 +549,7 @@ def update_changeset(struct, params \\ %{}) do :pleroma_settings_store, &{:ok, Map.merge(struct.pleroma_settings_store, &1)} ) - |> validate_fields(false) + |> validate_fields(false, struct) end defp put_fields(changeset) do @@ -2359,7 +2359,8 @@ def update_background(user, background) do |> update_and_set_cache() end - def validate_fields(changeset, remote? \\ false) do + @spec validate_fields(Ecto.Changeset.t(), Boolean.t(), User.t()) :: Ecto.Changeset.t() + def validate_fields(changeset, remote? \\ false, struct) do limit_name = if remote?, do: :max_remote_account_fields, else: :max_account_fields limit = Config.get([:instance, limit_name], 0) @@ -2372,6 +2373,7 @@ def validate_fields(changeset, remote? \\ false) do [fields: "invalid"] end end) + |> maybe_validate_rel_me_field(struct) end defp valid_field?(%{"name" => name, "value" => value}) do @@ -2384,6 +2386,75 @@ defp valid_field?(%{"name" => name, "value" => value}) do defp valid_field?(_), do: false + defp is_url(nil), do: nil + + defp is_url(uri) do + case URI.parse(uri) do + %URI{host: nil} -> false + %URI{scheme: nil} -> false + _ -> true + end + end + + @spec maybe_validate_rel_me_field(Changeset.t(), User.t()) :: Changeset.t() + defp maybe_validate_rel_me_field(changeset, %User{ap_id: _ap_id} = struct) do + fields = get_change(changeset, :fields) + raw_fields = get_change(changeset, :raw_fields) + + if is_nil(fields) do + changeset + else + validate_rel_me_field(changeset, fields, raw_fields, struct) + end + end + + defp maybe_validate_rel_me_field(changeset, _), do: changeset + + @spec validate_rel_me_field(Changeset.t(), [Map.t()], [Map.t()], User.t()) :: Changeset.t() + defp validate_rel_me_field(changeset, fields, raw_fields, %User{ + nickname: nickname, + ap_id: ap_id + }) do + fields = + fields + |> Enum.with_index() + |> Enum.map(fn {%{"name" => name, "value" => value}, index} -> + raw_value = + if is_nil(raw_fields) do + nil + else + Enum.at(raw_fields, index)["value"] + end + + if is_url(raw_value) do + frontend_url = + Pleroma.Web.Router.Helpers.redirect_url( + Pleroma.Web.Endpoint, + :redirector_with_meta, + nickname + ) + + possible_urls = [ap_id, frontend_url] + + with "me" <- RelMe.maybe_put_rel_me(raw_value, possible_urls) do + %{ + "name" => name, + "value" => value, + "verified_at" => DateTime.to_iso8601(DateTime.utc_now()) + } + else + e -> + Logger.error("Could not check for rel=me, #{inspect(e)}") + %{"name" => name, "value" => value} + end + else + %{"name" => name, "value" => value} + end + end) + + put_change(changeset, :fields, fields) + end + defp truncate_field(%{"name" => name, "value" => value}) do {name, _chopped} = String.split_at(name, Config.get([:instance, :account_field_name_length], 255)) @@ -2551,11 +2622,8 @@ def sanitize_html(%User{} = user) do # - display name def sanitize_html(%User{} = user, filter) do fields = - Enum.map(user.fields, fn %{"name" => name, "value" => value} -> - %{ - "name" => name, - "value" => HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly) - } + Enum.map(user.fields, fn %{"value" => value} = field -> + Map.put(field, "value", HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly)) end) user diff --git a/lib/pleroma/web/rel_me.ex b/lib/pleroma/web/rel_me.ex index 1826031dd..3a1812f7a 100644 --- a/lib/pleroma/web/rel_me.ex +++ b/lib/pleroma/web/rel_me.ex @@ -38,12 +38,11 @@ defp parse_url(url) do def maybe_put_rel_me("http" <> _ = target_page, profile_urls) when is_list(profile_urls) do {:ok, rel_me_hrefs} = parse(target_page) - true = Enum.any?(rel_me_hrefs, fn x -> x in profile_urls end) "me" rescue - _ -> nil + e -> nil end def maybe_put_rel_me(_, _) do diff --git a/test/pleroma/web/mastodon_api/update_credentials_test.exs b/test/pleroma/web/mastodon_api/update_credentials_test.exs index 2ba909dad..e9b8825bf 100644 --- a/test/pleroma/web/mastodon_api/update_credentials_test.exs +++ b/test/pleroma/web/mastodon_api/update_credentials_test.exs @@ -465,6 +465,69 @@ test "update fields", %{conn: conn} do ] end + test "update fields with a link to content with rel=me, with ap id", %{user: user, conn: conn} do + Tesla.Mock.mock(fn + %{url: "http://example.com/rel_me/ap_id"} -> + %Tesla.Env{ + status: 200, + body: ~s[] + } + end) + + field = %{name: "Website", value: "http://example.com/rel_me/ap_id"} + + account_data = + conn + |> patch("/api/v1/accounts/update_credentials", %{fields_attributes: [field]}) + |> json_response_and_validate_schema(200) + + assert [ + %{ + "name" => "Website", + "value" => + ~s[http://example.com/rel_me/ap_id], + "verified_at" => verified_at + } + ] = account_data["fields"] + + {:ok, verified_at, _} = DateTime.from_iso8601(verified_at) + assert DateTime.diff(DateTime.utc_now(), verified_at) < 10 + end + + test "update fields with a link to content with rel=me, with frontend path", %{ + user: user, + conn: conn + } do + fe_url = "#{Pleroma.Web.Endpoint.url()}/#{user.nickname}" + + Tesla.Mock.mock(fn + %{url: "http://example.com/rel_me/fe_path"} -> + %Tesla.Env{ + status: 200, + body: ~s[] + } + end) + + field = %{name: "Website", value: "http://example.com/rel_me/fe_path"} + + account_data = + conn + |> patch("/api/v1/accounts/update_credentials", %{fields_attributes: [field]}) + |> json_response_and_validate_schema(200) + + assert [ + %{ + "name" => "Website", + "value" => + ~s[http://example.com/rel_me/fe_path], + "verified_at" => verified_at + } + ] = account_data["fields"] + + {:ok, verified_at, _} = DateTime.from_iso8601(verified_at) + assert DateTime.diff(DateTime.utc_now(), verified_at) < 10 + end + test "emojis in fields labels", %{conn: conn} do fields = [ %{name: ":firefox:", value: "is best 2hu"}, From 9be6caf125f93ce8547a5f808681253131c32148 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Fri, 30 Dec 2022 02:46:58 +0000 Subject: [PATCH 11/22] argon2 password hashing (#406) Co-authored-by: FloatingGhost Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/406 --- CHANGELOG.md | 2 + lib/mix/tasks/pleroma/config.ex | 16 +++-- lib/pleroma/password.ex | 54 +++++++++++++++ lib/pleroma/user.ex | 2 +- lib/pleroma/web/auth/pleroma_authenticator.ex | 5 +- lib/pleroma/web/auth/totp_authenticator.ex | 3 +- lib/pleroma/web/common_api/utils.ex | 3 +- .../web/mongoose_im/mongoose_im_controller.ex | 3 +- lib/pleroma/web/plugs/authentication_plug.ex | 38 ++--------- mix.exs | 4 +- mix.lock | 2 +- test/pleroma/mfa_test.exs | 4 +- test/pleroma/password_test.exs | 65 +++++++++++++++++++ test/pleroma/web/auth/basic_auth_test.exs | 2 +- .../web/auth/pleroma_authenticator_test.exs | 4 +- .../web/auth/totp_authenticator_test.exs | 2 +- .../web/mongoose_im_controller_test.exs | 4 +- .../web/o_auth/ldap_authorization_test.exs | 4 +- .../web/o_auth/mfa_controller_test.exs | 4 +- .../web/o_auth/o_auth_controller_test.exs | 18 ++--- .../web/plugs/authentication_plug_test.exs | 36 +++++----- .../twitter_api/password_controller_test.exs | 2 +- .../web/twitter_api/util_controller_test.exs | 2 +- test/support/builders/user_builder.ex | 2 +- test/support/factory.ex | 6 +- 25 files changed, 188 insertions(+), 99 deletions(-) create mode 100644 lib/pleroma/password.ex create mode 100644 test/pleroma/password_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index ea6a25e4b..3106854ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Prometheus metrics exporting from `/api/v1/akkoma/metrics` - Ability to alter http pool size - Translation of statuses via ArgosTranslate +- Argon2 password hashing - Ability to "verify" links in profile fields via rel=me - Mix tasks to dump/load config to/from json for bulk editing @@ -17,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Non-finch HTTP adapters - Legacy redirect from /api/pleroma/admin to /api/v1/pleroma/admin - Legacy redirects from /api/pleroma to /api/v1/pleroma +- :crypt dependency ### Changed - Return HTTP error 413 when uploading an avatar or banner that's above the configured upload limit instead of a 500. diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index dfde6520d..8661d8d7c 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -87,13 +87,15 @@ def run(["dump_to_file", group, key, fname]) do key = maybe_atomize(key) config = ConfigDB.get_by_group_and_key(group, key) - json = %{ - group: ConfigDB.to_json_types(config.group), - key: ConfigDB.to_json_types(config.key), - value: ConfigDB.to_json_types(config.value), - } - |> Jason.encode!() - |> Jason.Formatter.pretty_print() + + json = + %{ + group: ConfigDB.to_json_types(config.group), + key: ConfigDB.to_json_types(config.key), + value: ConfigDB.to_json_types(config.value) + } + |> Jason.encode!() + |> Jason.Formatter.pretty_print() File.write(fname, json) shell_info("Wrote #{group}_#{key}.json") diff --git a/lib/pleroma/password.ex b/lib/pleroma/password.ex new file mode 100644 index 000000000..f8cba61a3 --- /dev/null +++ b/lib/pleroma/password.ex @@ -0,0 +1,54 @@ +defmodule Pleroma.Password do + @moduledoc """ + This module handles password hashing and verification. + It will delegate to the appropriate module based on the password hash. + It also handles upgrading of password hashes. + """ + + alias Pleroma.User + alias Pleroma.Password.Pbkdf2 + require Logger + + @hashing_module Argon2 + + @spec hash_pwd_salt(String.t()) :: String.t() + defdelegate hash_pwd_salt(pass), to: @hashing_module + + @spec checkpw(String.t(), String.t()) :: boolean() + def checkpw(password, "$2" <> _ = password_hash) do + # Handle bcrypt passwords for Mastodon migration + Bcrypt.verify_pass(password, password_hash) + end + + def checkpw(password, "$pbkdf2" <> _ = password_hash) do + Pbkdf2.verify_pass(password, password_hash) + end + + def checkpw(password, "$argon2" <> _ = password_hash) do + Argon2.verify_pass(password, password_hash) + end + + def checkpw(_password, _password_hash) do + Logger.error("Password hash not recognized") + false + end + + @spec maybe_update_password(User.t(), String.t()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} + def maybe_update_password(%User{password_hash: "$2" <> _} = user, password) do + do_update_password(user, password) + end + + def maybe_update_password(%User{password_hash: "$6" <> _} = user, password) do + do_update_password(user, password) + end + + def maybe_update_password(%User{password_hash: "$pbkdf2" <> _} = user, password) do + do_update_password(user, password) + end + + def maybe_update_password(user, _), do: {:ok, user} + + defp do_update_password(user, password) do + User.reset_password(user, %{password: password, password_confirmation: password}) + end +end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 2d4bd097d..1ddbd36a8 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2277,7 +2277,7 @@ def get_ap_ids_by_nicknames(nicknames) do defp put_password_hash( %Ecto.Changeset{valid?: true, changes: %{password: password}} = changeset ) do - change(changeset, password_hash: Pleroma.Password.Pbkdf2.hash_pwd_salt(password)) + change(changeset, password_hash: Pleroma.Password.hash_pwd_salt(password)) end defp put_password_hash(changeset), do: changeset diff --git a/lib/pleroma/web/auth/pleroma_authenticator.ex b/lib/pleroma/web/auth/pleroma_authenticator.ex index bb377d686..01b54037c 100644 --- a/lib/pleroma/web/auth/pleroma_authenticator.ex +++ b/lib/pleroma/web/auth/pleroma_authenticator.ex @@ -6,7 +6,6 @@ defmodule Pleroma.Web.Auth.PleromaAuthenticator do alias Pleroma.Registration alias Pleroma.Repo alias Pleroma.User - alias Pleroma.Web.Plugs.AuthenticationPlug import Pleroma.Web.Auth.Helpers, only: [fetch_credentials: 1, fetch_user: 1] @@ -15,8 +14,8 @@ defmodule Pleroma.Web.Auth.PleromaAuthenticator do def get_user(%Plug.Conn{} = conn) do with {:ok, {name, password}} <- fetch_credentials(conn), {_, %User{} = user} <- {:user, fetch_user(name)}, - {_, true} <- {:checkpw, AuthenticationPlug.checkpw(password, user.password_hash)}, - {:ok, user} <- AuthenticationPlug.maybe_update_password(user, password) do + {_, true} <- {:checkpw, Pleroma.Password.checkpw(password, user.password_hash)}, + {:ok, user} <- Pleroma.Password.maybe_update_password(user, password) do {:ok, user} else {:error, _reason} = error -> error diff --git a/lib/pleroma/web/auth/totp_authenticator.ex b/lib/pleroma/web/auth/totp_authenticator.ex index 5947cd8c9..e6f839e6e 100644 --- a/lib/pleroma/web/auth/totp_authenticator.ex +++ b/lib/pleroma/web/auth/totp_authenticator.ex @@ -6,7 +6,6 @@ defmodule Pleroma.Web.Auth.TOTPAuthenticator do alias Pleroma.MFA alias Pleroma.MFA.TOTP alias Pleroma.User - alias Pleroma.Web.Plugs.AuthenticationPlug @doc "Verify code or check backup code." @spec verify(String.t(), User.t()) :: @@ -31,7 +30,7 @@ def verify_recovery_code( code ) when is_list(codes) and is_binary(code) do - hash_code = Enum.find(codes, fn hash -> AuthenticationPlug.checkpw(code, hash) end) + hash_code = Enum.find(codes, fn hash -> Pleroma.Password.checkpw(code, hash) end) if hash_code do MFA.invalidate_backup_code(user, hash_code) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index bf03b0a82..22594be46 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -17,7 +17,6 @@ defmodule Pleroma.Web.CommonAPI.Utils do alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.CommonAPI.ActivityDraft alias Pleroma.Web.MediaProxy - alias Pleroma.Web.Plugs.AuthenticationPlug alias Pleroma.Web.Utils.Params require Logger @@ -356,7 +355,7 @@ defp shortname(name) do @spec confirm_current_password(User.t(), String.t()) :: {:ok, User.t()} | {:error, String.t()} def confirm_current_password(user, password) do with %User{local: true} = db_user <- User.get_cached_by_id(user.id), - true <- AuthenticationPlug.checkpw(password, db_user.password_hash) do + true <- Pleroma.Password.checkpw(password, db_user.password_hash) do {:ok, db_user} else _ -> {:error, dgettext("errors", "Invalid password.")} diff --git a/lib/pleroma/web/mongoose_im/mongoose_im_controller.ex b/lib/pleroma/web/mongoose_im/mongoose_im_controller.ex index 6ace3e0b5..85b75190b 100644 --- a/lib/pleroma/web/mongoose_im/mongoose_im_controller.ex +++ b/lib/pleroma/web/mongoose_im/mongoose_im_controller.ex @@ -7,7 +7,6 @@ defmodule Pleroma.Web.MongooseIM.MongooseIMController do alias Pleroma.Repo alias Pleroma.User - alias Pleroma.Web.Plugs.AuthenticationPlug alias Pleroma.Web.Plugs.RateLimiter plug(RateLimiter, [name: :authentication] when action in [:user_exists, :check_password]) @@ -28,7 +27,7 @@ def user_exists(conn, %{"user" => username}) do def check_password(conn, %{"user" => username, "pass" => password}) do with %User{password_hash: password_hash, is_active: true} <- Repo.get_by(User, nickname: username, local: true), - true <- AuthenticationPlug.checkpw(password, password_hash) do + true <- Pleroma.Password.checkpw(password, password_hash) do conn |> json(true) else diff --git a/lib/pleroma/web/plugs/authentication_plug.ex b/lib/pleroma/web/plugs/authentication_plug.ex index 8d58169cf..894a1067e 100644 --- a/lib/pleroma/web/plugs/authentication_plug.ex +++ b/lib/pleroma/web/plugs/authentication_plug.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.Plugs.AuthenticationPlug do alias Pleroma.Helpers.AuthHelper alias Pleroma.User + alias Pleroma.Password import Plug.Conn @@ -25,8 +26,8 @@ def call( } = conn, _ ) do - if checkpw(password, password_hash) do - {:ok, auth_user} = maybe_update_password(auth_user, password) + if Password.checkpw(password, password_hash) do + {:ok, auth_user} = Password.maybe_update_password(auth_user, password) conn |> assign(:user, auth_user) @@ -38,35 +39,6 @@ def call( def call(conn, _), do: conn - def checkpw(password, "$6" <> _ = password_hash) do - :crypt.crypt(password, password_hash) == password_hash - end - - def checkpw(password, "$2" <> _ = password_hash) do - # Handle bcrypt passwords for Mastodon migration - Bcrypt.verify_pass(password, password_hash) - end - - def checkpw(password, "$pbkdf2" <> _ = password_hash) do - Pleroma.Password.Pbkdf2.verify_pass(password, password_hash) - end - - def checkpw(_password, _password_hash) do - Logger.error("Password hash not recognized") - false - end - - def maybe_update_password(%User{password_hash: "$2" <> _} = user, password) do - do_update_password(user, password) - end - - def maybe_update_password(%User{password_hash: "$6" <> _} = user, password) do - do_update_password(user, password) - end - - def maybe_update_password(user, _), do: {:ok, user} - - defp do_update_password(user, password) do - User.reset_password(user, %{password: password, password_confirmation: password}) - end + @spec checkpw(String.t(), String.t()) :: boolean + defdelegate checkpw(password, hash), to: Password end diff --git a/mix.exs b/mix.exs index 4898591b1..2a62dedee 100644 --- a/mix.exs +++ b/mix.exs @@ -143,9 +143,7 @@ defp deps do {:sweet_xml, "~> 0.7.2"}, {:earmark, "~> 1.4.15"}, {:bbcode_pleroma, "~> 0.2.0"}, - {:crypt, - git: "https://github.com/msantos/crypt.git", - ref: "f75cd55325e33cbea198fb41fe41871392f8fb76"}, + {:argon2_elixir, "~> 3.0.0"}, {:cors_plug, "~> 2.0"}, {:web_push_encryption, "~> 0.3.1"}, {:swoosh, "~> 1.0"}, diff --git a/mix.lock b/mix.lock index 4fa4c05ec..dbe0ea5e3 100644 --- a/mix.lock +++ b/mix.lock @@ -1,4 +1,5 @@ %{ + "argon2_elixir": {:hex, :argon2_elixir, "3.0.0", "fd4405f593e77b525a5c667282172dd32772d7c4fa58cdecdaae79d2713b6c5f", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "8b753b270af557d51ba13fcdebc0f0ab27a2a6792df72fd5a6cf9cfaffcedc57"}, "base62": {:hex, :base62, "1.2.2", "85c6627eb609317b70f555294045895ffaaeb1758666ab9ef9ca38865b11e629", [:mix], [{:custom_base, "~> 0.2.1", [hex: :custom_base, repo: "hexpm", optional: false]}], "hexpm", "d41336bda8eaa5be197f1e4592400513ee60518e5b9f4dcf38f4b4dae6f377bb"}, "bbcode_pleroma": {:hex, :bbcode_pleroma, "0.2.0", "d36f5bca6e2f62261c45be30fa9b92725c0655ad45c99025cb1c3e28e25803ef", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "19851074419a5fedb4ef49e1f01b30df504bb5dbb6d6adfc135238063bebd1c3"}, "bcrypt_elixir": {:hex, :bcrypt_elixir, "2.3.1", "5114d780459a04f2b4aeef52307de23de961b69e13a5cd98a911e39fda13f420", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "42182d5f46764def15bf9af83739e3bf4ad22661b1c34fc3e88558efced07279"}, @@ -18,7 +19,6 @@ "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.3.1", "ebd1a1d7aff97f27c66654e78ece187abdc646992714164380d8a041eda16754", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3a6efd3366130eab84ca372cbd4a7d3c3a97bdfcfb4911233b035d117063f0af"}, "cowlib": {:hex, :cowlib, "2.11.0", "0b9ff9c346629256c42ebe1eeb769a83c6cb771a6ee5960bd110ab0b9b872063", [:make, :rebar3], [], "hexpm", "2b3e9da0b21c4565751a6d4901c20d1b4cc25cbb7fd50d91d2ab6dd287bc86a9"}, "credo": {:git, "https://github.com/rrrene/credo.git", "1c1b99ea41a457761383d81aaf6a606913996fe7", [ref: "1c1b99ea41a457761383d81aaf6a606913996fe7"]}, - "crypt": {:git, "https://github.com/msantos/crypt.git", "f75cd55325e33cbea198fb41fe41871392f8fb76", [ref: "f75cd55325e33cbea198fb41fe41871392f8fb76"]}, "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"}, "db_connection": {:hex, :db_connection, "2.4.3", "3b9aac9f27347ec65b271847e6baeb4443d8474289bd18c1d6f4de655b70c94d", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c127c15b0fa6cfb32eed07465e05da6c815b032508d4ed7c116122871df73c12"}, "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, diff --git a/test/pleroma/mfa_test.exs b/test/pleroma/mfa_test.exs index 76ba1a99d..66660d97e 100644 --- a/test/pleroma/mfa_test.exs +++ b/test/pleroma/mfa_test.exs @@ -30,8 +30,8 @@ test "returns backup codes" do {:ok, [code1, code2]} = MFA.generate_backup_codes(user) updated_user = refresh_record(user) [hash1, hash2] = updated_user.multi_factor_authentication_settings.backup_codes - assert Pleroma.Password.Pbkdf2.verify_pass(code1, hash1) - assert Pleroma.Password.Pbkdf2.verify_pass(code2, hash2) + assert Pleroma.Password.checkpw(code1, hash1) + assert Pleroma.Password.checkpw(code2, hash2) end end diff --git a/test/pleroma/password_test.exs b/test/pleroma/password_test.exs new file mode 100644 index 000000000..951fc810a --- /dev/null +++ b/test/pleroma/password_test.exs @@ -0,0 +1,65 @@ +defmodule Pleroma.PasswordTest do + use Pleroma.DataCase, async: true + import Pleroma.Factory + import ExUnit.CaptureLog + + alias Pleroma.Password + + describe "hash_pwd_salt/1" do + test "returns a hash" do + assert "$argon2id" <> _ = Password.hash_pwd_salt("test") + end + end + + describe "maybe_update_password/2" do + test "with a bcrypt hash, it updates to an argon2 hash" do + user = insert(:user, password_hash: Bcrypt.hash_pwd_salt("123")) + assert "$2" <> _ = user.password_hash + + {:ok, user} = Password.maybe_update_password(user, "123") + assert "$argon2" <> _ = user.password_hash + end + + test "with a pbkdf2 hash, it updates to an argon2 hash" do + user = insert(:user, password_hash: Pleroma.Password.Pbkdf2.hash_pwd_salt("123")) + assert "$pbkdf2" <> _ = user.password_hash + + {:ok, user} = Password.maybe_update_password(user, "123") + assert "$argon2" <> _ = user.password_hash + end + end + + describe "checkpw/2" do + test "check pbkdf2 hash" do + hash = + "$pbkdf2-sha512$160000$loXqbp8GYls43F0i6lEfIw$AY.Ep.2pGe57j2hAPY635sI/6w7l9Q9u9Bp02PkPmF3OrClDtJAI8bCiivPr53OKMF7ph6iHhN68Rom5nEfC2A" + + assert Password.checkpw("test-password", hash) + refute Password.checkpw("test-password1", hash) + end + + test "check bcrypt hash" do + hash = "$2a$10$uyhC/R/zoE1ndwwCtMusK.TLVzkQ/Ugsbqp3uXI.CTTz0gBw.24jS" + + assert Password.checkpw("password", hash) + refute Password.checkpw("password1", hash) + end + + test "check argon2 hash" do + hash = + "$argon2id$v=19$m=65536,t=8,p=2$zEMMsTuK5KkL5AFWbX7jyQ$VyaQD7PF6e9btz0oH1YiAkWwIGZ7WNDZP8l+a/O171g" + + assert Password.checkpw("password", hash) + refute Password.checkpw("password1", hash) + end + + test "it returns false when hash invalid" do + hash = + "psBWV8gxkGOZWBz$PmfCycChoxeJ3GgGzwvhlgacb9mUoZ.KUXNCssekER4SJ7bOK53uXrHNb2e4i8yPFgSKyzaW9CcmrDXWIEMtD1" + + assert capture_log(fn -> + refute Password.checkpw("password", hash) + end) =~ "[error] Password hash not recognized" + end + end +end diff --git a/test/pleroma/web/auth/basic_auth_test.exs b/test/pleroma/web/auth/basic_auth_test.exs index 2816aae4c..a357ba4a5 100644 --- a/test/pleroma/web/auth/basic_auth_test.exs +++ b/test/pleroma/web/auth/basic_auth_test.exs @@ -11,7 +11,7 @@ test "with HTTP Basic Auth used, grants access to OAuth scope-restricted endpoin conn: conn } do user = insert(:user) - assert Pleroma.Password.Pbkdf2.verify_pass("test", user.password_hash) + assert Pleroma.Password.checkpw("test", user.password_hash) basic_auth_contents = (URI.encode_www_form(user.nickname) <> ":" <> URI.encode_www_form("test")) diff --git a/test/pleroma/web/auth/pleroma_authenticator_test.exs b/test/pleroma/web/auth/pleroma_authenticator_test.exs index b1397c523..fb3c47417 100644 --- a/test/pleroma/web/auth/pleroma_authenticator_test.exs +++ b/test/pleroma/web/auth/pleroma_authenticator_test.exs @@ -15,7 +15,7 @@ defmodule Pleroma.Web.Auth.PleromaAuthenticatorTest do user = insert(:user, nickname: name, - password_hash: Pleroma.Password.Pbkdf2.hash_pwd_salt(password) + password_hash: Pleroma.Password.hash_pwd_salt(password) ) {:ok, [user: user, name: name, password: password]} @@ -30,7 +30,7 @@ test "get_user/authorization", %{name: name, password: password} do assert {:ok, returned_user} = res assert returned_user.id == user.id - assert "$pbkdf2" <> _ = returned_user.password_hash + assert "$argon2" <> _ = returned_user.password_hash end test "get_user/authorization with invalid password", %{name: name} do diff --git a/test/pleroma/web/auth/totp_authenticator_test.exs b/test/pleroma/web/auth/totp_authenticator_test.exs index ac4209f2d..6d2646b61 100644 --- a/test/pleroma/web/auth/totp_authenticator_test.exs +++ b/test/pleroma/web/auth/totp_authenticator_test.exs @@ -34,7 +34,7 @@ test "checks backup codes" do hashed_codes = backup_codes - |> Enum.map(&Pleroma.Password.Pbkdf2.hash_pwd_salt(&1)) + |> Enum.map(&Pleroma.Password.hash_pwd_salt(&1)) user = insert(:user, diff --git a/test/pleroma/web/mongoose_im_controller_test.exs b/test/pleroma/web/mongoose_im_controller_test.exs index 43c4dfa33..73473ccf5 100644 --- a/test/pleroma/web/mongoose_im_controller_test.exs +++ b/test/pleroma/web/mongoose_im_controller_test.exs @@ -41,13 +41,13 @@ test "/user_exists", %{conn: conn} do end test "/check_password", %{conn: conn} do - user = insert(:user, password_hash: Pleroma.Password.Pbkdf2.hash_pwd_salt("cool")) + user = insert(:user, password_hash: Pleroma.Password.hash_pwd_salt("cool")) _deactivated_user = insert(:user, nickname: "konata", is_active: false, - password_hash: Pleroma.Password.Pbkdf2.hash_pwd_salt("cool") + password_hash: Pleroma.Password.hash_pwd_salt("cool") ) res = diff --git a/test/pleroma/web/o_auth/ldap_authorization_test.exs b/test/pleroma/web/o_auth/ldap_authorization_test.exs index c8a1d65ab..502ee0918 100644 --- a/test/pleroma/web/o_auth/ldap_authorization_test.exs +++ b/test/pleroma/web/o_auth/ldap_authorization_test.exs @@ -18,7 +18,7 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do @tag @skip test "authorizes the existing user using LDAP credentials" do password = "testpassword" - user = insert(:user, password_hash: Pleroma.Password.Pbkdf2.hash_pwd_salt(password)) + user = insert(:user, password_hash: Pleroma.Password.hash_pwd_salt(password)) app = insert(:oauth_app, scopes: ["read", "write"]) host = Pleroma.Config.get([:ldap, :host]) |> to_charlist @@ -101,7 +101,7 @@ test "creates a new user after successful LDAP authorization" do @tag @skip test "disallow authorization for wrong LDAP credentials" do password = "testpassword" - user = insert(:user, password_hash: Pleroma.Password.Pbkdf2.hash_pwd_salt(password)) + user = insert(:user, password_hash: Pleroma.Password.hash_pwd_salt(password)) app = insert(:oauth_app, scopes: ["read", "write"]) host = Pleroma.Config.get([:ldap, :host]) |> to_charlist diff --git a/test/pleroma/web/o_auth/mfa_controller_test.exs b/test/pleroma/web/o_auth/mfa_controller_test.exs index 17bbde85b..dacf03b2b 100644 --- a/test/pleroma/web/o_auth/mfa_controller_test.exs +++ b/test/pleroma/web/o_auth/mfa_controller_test.exs @@ -20,7 +20,7 @@ defmodule Pleroma.Web.OAuth.MFAControllerTest do insert(:user, multi_factor_authentication_settings: %MFA.Settings{ enabled: true, - backup_codes: [Pleroma.Password.Pbkdf2.hash_pwd_salt("test-code")], + backup_codes: [Pleroma.Password.hash_pwd_salt("test-code")], totp: %MFA.Settings.TOTP{secret: otp_secret, confirmed: true} } ) @@ -246,7 +246,7 @@ test "returns access token with valid code", %{conn: conn, app: app} do hashed_codes = backup_codes - |> Enum.map(&Pleroma.Password.Pbkdf2.hash_pwd_salt(&1)) + |> Enum.map(&Pleroma.Password.hash_pwd_salt(&1)) user = insert(:user, diff --git a/test/pleroma/web/o_auth/o_auth_controller_test.exs b/test/pleroma/web/o_auth/o_auth_controller_test.exs index 7240624ef..303bc2cf2 100644 --- a/test/pleroma/web/o_auth/o_auth_controller_test.exs +++ b/test/pleroma/web/o_auth/o_auth_controller_test.exs @@ -316,7 +316,7 @@ test "with valid params, POST /oauth/register?op=connect redirects to `redirect_ app: app, conn: conn } do - user = insert(:user, password_hash: Pleroma.Password.Pbkdf2.hash_pwd_salt("testpassword")) + user = insert(:user, password_hash: Pleroma.Password.hash_pwd_salt("testpassword")) registration = insert(:registration, user: nil) redirect_uri = OAuthController.default_redirect_uri(app) @@ -347,7 +347,7 @@ test "with unlisted `redirect_uri`, POST /oauth/register?op=connect results in H app: app, conn: conn } do - user = insert(:user, password_hash: Pleroma.Password.Pbkdf2.hash_pwd_salt("testpassword")) + user = insert(:user, password_hash: Pleroma.Password.hash_pwd_salt("testpassword")) registration = insert(:registration, user: nil) unlisted_redirect_uri = "http://cross-site-request.com" @@ -917,7 +917,7 @@ test "issues a token for an all-body request" do test "issues a token for `password` grant_type with valid credentials, with full permissions by default" do password = "testpassword" - user = insert(:user, password_hash: Pleroma.Password.Pbkdf2.hash_pwd_salt(password)) + user = insert(:user, password_hash: Pleroma.Password.hash_pwd_salt(password)) app = insert(:oauth_app, scopes: ["read", "write"]) @@ -947,7 +947,7 @@ test "issues a mfa token for `password` grant_type, when MFA enabled" do user = insert(:user, - password_hash: Pleroma.Password.Pbkdf2.hash_pwd_salt(password), + password_hash: Pleroma.Password.hash_pwd_salt(password), multi_factor_authentication_settings: %MFA.Settings{ enabled: true, totp: %MFA.Settings.TOTP{secret: otp_secret, confirmed: true} @@ -1056,7 +1056,7 @@ test "rejects token exchange for valid credentials belonging to unconfirmed user password = "testpassword" {:ok, user} = - insert(:user, password_hash: Pleroma.Password.Pbkdf2.hash_pwd_salt(password)) + insert(:user, password_hash: Pleroma.Password.hash_pwd_salt(password)) |> User.confirmation_changeset(set_confirmation: false) |> User.update_and_set_cache() @@ -1084,7 +1084,7 @@ test "rejects token exchange for valid credentials belonging to deactivated user user = insert(:user, - password_hash: Pleroma.Password.Pbkdf2.hash_pwd_salt(password), + password_hash: Pleroma.Password.hash_pwd_salt(password), is_active: false ) @@ -1112,7 +1112,7 @@ test "rejects token exchange for user with password_reset_pending set to true" d user = insert(:user, - password_hash: Pleroma.Password.Pbkdf2.hash_pwd_salt(password), + password_hash: Pleroma.Password.hash_pwd_salt(password), password_reset_pending: true ) @@ -1141,7 +1141,7 @@ test "rejects token exchange for user with confirmation_pending set to true" do user = insert(:user, - password_hash: Pleroma.Password.Pbkdf2.hash_pwd_salt(password), + password_hash: Pleroma.Password.hash_pwd_salt(password), is_confirmed: false ) @@ -1169,7 +1169,7 @@ test "rejects token exchange for valid credentials belonging to an unapproved us user = insert(:user, - password_hash: Pleroma.Password.Pbkdf2.hash_pwd_salt(password), + password_hash: Pleroma.Password.hash_pwd_salt(password), is_approved: false ) diff --git a/test/pleroma/web/plugs/authentication_plug_test.exs b/test/pleroma/web/plugs/authentication_plug_test.exs index 118ab302a..1fbc17a92 100644 --- a/test/pleroma/web/plugs/authentication_plug_test.exs +++ b/test/pleroma/web/plugs/authentication_plug_test.exs @@ -17,7 +17,7 @@ defmodule Pleroma.Web.Plugs.AuthenticationPlugTest do user = %User{ id: 1, name: "dude", - password_hash: Pleroma.Password.Pbkdf2.hash_pwd_salt("guy") + password_hash: Pleroma.Password.hash_pwd_salt("guy") } conn = @@ -52,7 +52,7 @@ test "with a correct password in the credentials, " <> assert PlugHelper.plug_skipped?(conn, OAuthScopesPlug) end - test "with a bcrypt hash, it updates to a pkbdf2 hash", %{conn: conn} do + test "with a bcrypt hash, it updates to an argon2 hash", %{conn: conn} do user = insert(:user, password_hash: Bcrypt.hash_pwd_salt("123")) assert "$2" <> _ = user.password_hash @@ -67,21 +67,17 @@ test "with a bcrypt hash, it updates to a pkbdf2 hash", %{conn: conn} do assert PlugHelper.plug_skipped?(conn, OAuthScopesPlug) user = User.get_by_id(user.id) - assert "$pbkdf2" <> _ = user.password_hash + assert "$argon2" <> _ = user.password_hash end - @tag :skip_on_mac - test "with a crypt hash, it updates to a pkbdf2 hash", %{conn: conn} do - user = - insert(:user, - password_hash: - "$6$9psBWV8gxkGOZWBz$PmfCycChoxeJ3GgGzwvhlgacb9mUoZ.KUXNCssekER4SJ7bOK53uXrHNb2e4i8yPFgSKyzaW9CcmrDXWIEMtD1" - ) + test "with a pbkdf2 hash, it updates to an argon2 hash", %{conn: conn} do + user = insert(:user, password_hash: Pleroma.Password.Pbkdf2.hash_pwd_salt("123")) + assert "$pbkdf2" <> _ = user.password_hash conn = conn |> assign(:auth_user, user) - |> assign(:auth_credentials, %{password: "password"}) + |> assign(:auth_credentials, %{password: "123"}) |> AuthenticationPlug.call(%{}) assert conn.assigns.user.id == conn.assigns.auth_user.id @@ -89,7 +85,7 @@ test "with a crypt hash, it updates to a pkbdf2 hash", %{conn: conn} do assert PlugHelper.plug_skipped?(conn, OAuthScopesPlug) user = User.get_by_id(user.id) - assert "$pbkdf2" <> _ = user.password_hash + assert "$argon2" <> _ = user.password_hash end describe "checkpw/2" do @@ -101,14 +97,6 @@ test "check pbkdf2 hash" do refute AuthenticationPlug.checkpw("test-password1", hash) end - @tag :skip_on_mac - test "check sha512-crypt hash" do - hash = - "$6$9psBWV8gxkGOZWBz$PmfCycChoxeJ3GgGzwvhlgacb9mUoZ.KUXNCssekER4SJ7bOK53uXrHNb2e4i8yPFgSKyzaW9CcmrDXWIEMtD1" - - assert AuthenticationPlug.checkpw("password", hash) - end - test "check bcrypt hash" do hash = "$2a$10$uyhC/R/zoE1ndwwCtMusK.TLVzkQ/Ugsbqp3uXI.CTTz0gBw.24jS" @@ -116,6 +104,14 @@ test "check bcrypt hash" do refute AuthenticationPlug.checkpw("password1", hash) end + test "check argon2 hash" do + hash = + "$argon2id$v=19$m=65536,t=8,p=2$zEMMsTuK5KkL5AFWbX7jyQ$VyaQD7PF6e9btz0oH1YiAkWwIGZ7WNDZP8l+a/O171g" + + assert AuthenticationPlug.checkpw("password", hash) + refute AuthenticationPlug.checkpw("password1", hash) + end + test "it returns false when hash invalid" do hash = "psBWV8gxkGOZWBz$PmfCycChoxeJ3GgGzwvhlgacb9mUoZ.KUXNCssekER4SJ7bOK53uXrHNb2e4i8yPFgSKyzaW9CcmrDXWIEMtD1" diff --git a/test/pleroma/web/twitter_api/password_controller_test.exs b/test/pleroma/web/twitter_api/password_controller_test.exs index 05c3561bf..4ff792dc8 100644 --- a/test/pleroma/web/twitter_api/password_controller_test.exs +++ b/test/pleroma/web/twitter_api/password_controller_test.exs @@ -96,7 +96,7 @@ test "it returns HTTP 200", %{conn: conn} do assert response =~ "

Password changed!

" user = refresh_record(user) - assert Pleroma.Password.Pbkdf2.verify_pass("test", user.password_hash) + assert Pleroma.Password.checkpw("test", user.password_hash) assert Enum.empty?(Token.get_user_tokens(user)) end diff --git a/test/pleroma/web/twitter_api/util_controller_test.exs b/test/pleroma/web/twitter_api/util_controller_test.exs index 3f839568d..51f216bf1 100644 --- a/test/pleroma/web/twitter_api/util_controller_test.exs +++ b/test/pleroma/web/twitter_api/util_controller_test.exs @@ -553,7 +553,7 @@ test "with proper permissions, valid password and matching new password and conf assert json_response_and_validate_schema(conn, 200) == %{"status" => "success"} fetched_user = User.get_cached_by_id(user.id) - assert Pleroma.Password.Pbkdf2.verify_pass("newpass", fetched_user.password_hash) == true + assert Pleroma.Password.checkpw("newpass", fetched_user.password_hash) == true end end diff --git a/test/support/builders/user_builder.ex b/test/support/builders/user_builder.ex index 6bccbb35a..27470498d 100644 --- a/test/support/builders/user_builder.ex +++ b/test/support/builders/user_builder.ex @@ -7,7 +7,7 @@ def build(data \\ %{}) do email: "test@example.org", name: "Test Name", nickname: "testname", - password_hash: Pleroma.Password.Pbkdf2.hash_pwd_salt("test"), + password_hash: Pleroma.Password.hash_pwd_salt("test"), bio: "A tester.", ap_id: "some id", last_digest_emailed_at: NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second), diff --git a/test/support/factory.ex b/test/support/factory.ex index 84e076137..42c940c52 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -47,12 +47,16 @@ def instance_factory(attrs \\ %{}) do def user_factory(attrs \\ %{}) do pem = Enum.random(@rsa_keys) + # Argon2.hash_pwd_salt("test") + # it really eats CPU time, so we use a precomputed hash + password_hash = + "$argon2id$v=19$m=65536,t=8,p=2$FEAarFuiOsROO24NHIHMYw$oxdaz2fTPpuU+dYCl60FsqE65T1Tjy6lGikKfmql4xo" user = %User{ name: sequence(:name, &"Test テスト User #{&1}"), email: sequence(:email, &"user#{&1}@example.com"), nickname: sequence(:nickname, &"nick#{&1}"), - password_hash: Pleroma.Password.Pbkdf2.hash_pwd_salt("test"), + password_hash: password_hash, bio: sequence(:bio, &"Tester Number #{&1}"), is_discoverable: true, last_digest_emailed_at: NaiveDateTime.utc_now(), From bca1c43dcbbcd1320bc838f1c40b515c5e2c1369 Mon Sep 17 00:00:00 2001 From: timorl Date: Fri, 30 Dec 2022 02:58:06 +0000 Subject: [PATCH 12/22] Add docs about emoji stealing (#364) I managed to steal some emoji, but I had to figure out the specifics the hard way. This should make it easier for future criminals. Feel free to close if this documentation was omitted on purpose, I can imagine some reasons for why it might have. Co-authored-by: timorl Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/364 Co-authored-by: timorl Co-committed-by: timorl --- docs/docs/configuration/custom_emoji.md | 26 +++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/docs/configuration/custom_emoji.md b/docs/docs/configuration/custom_emoji.md index a0a40f294..a883e8bf2 100644 --- a/docs/docs/configuration/custom_emoji.md +++ b/docs/docs/configuration/custom_emoji.md @@ -67,3 +67,29 @@ Priority of tags assigns in emoji.txt and custom.txt: Priority for globs: `special group setting in config.exs > default setting in config.exs` + +## Stealing emoji + +Managing your emoji can be hard work, and you just want to have the cool emoji your friends use? As usual, crime comes to the rescue! + +You can use the `Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy` [Message Rewrite Facility](../configuration/cheatsheet.md#mrf) to automatically add to your instance emoji that messages from specific servers contain. Note that this happens on message processing, so the emoji will be added only after your instance receives some interaction containing emoji _after_ configuring this. + +To activate this you have to [configure](../configuration/cheatsheet.md#mrf_steal_emoji) it in your configuration file. For example if you wanted to steal any emoji that is not related to cinnamon and not larger than about 10K from `coolemoji.space` and `spiceenthusiasts.biz`, you would add the following: +```elixir +config :pleroma, :mrf, + policies: [ + Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy + ] + +config :pleroma, :mrf_steal_emoji, + hosts: [ + "coolemoji.space", + "spiceenthusiasts.biz" + ], + rejected_shortcodes: [ + ".*cinnamon.*" + ], + size_limit: 10000 +``` + +Note that this may not obey emoji licensing restrictions. It's extremely unlikely that anyone will care, but keep this in mind for when Nintendo starts their own instance. From 5d4c291d526e023e0fd769af623fc243fc067626 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 30 Dec 2022 03:43:35 +0000 Subject: [PATCH 13/22] update references to pleroma in docs --- lib/pleroma/password.ex | 3 ++- lib/pleroma/web/api_spec.ex | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/password.ex b/lib/pleroma/password.ex index f8cba61a3..92d78552b 100644 --- a/lib/pleroma/password.ex +++ b/lib/pleroma/password.ex @@ -33,7 +33,8 @@ def checkpw(_password, _password_hash) do false end - @spec maybe_update_password(User.t(), String.t()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} + @spec maybe_update_password(User.t(), String.t()) :: + {:ok, User.t()} | {:error, Ecto.Changeset.t()} def maybe_update_password(%User{password_hash: "$2" <> _} = user, password) do do_update_password(user, password) end diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index 8ac5c8b94..26fed1eef 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -23,19 +23,19 @@ def spec(opts \\ []) do [] end, info: %OpenApiSpex.Info{ - title: "Pleroma API", + title: "Akkoma API", description: """ - This is documentation for client Pleroma API. Most of the endpoints and entities come + This is documentation for the Akkoma API. Most of the endpoints and entities come from Mastodon API and have custom extensions on top. - While this document aims to be a complete guide to the client API Pleroma exposes, - the details are still being worked out. Some endpoints may have incomplete or poorly worded documentation. + While this document aims to be a complete guide to the client API Akkoma exposes, + it may not be complete. Some endpoints may have incomplete or poorly worded documentation. You might want to check the following resources if something is not clear: - [Legacy Pleroma-specific endpoint documentation](https://docs-develop.pleroma.social/backend/development/API/pleroma_api/) - [Mastodon API documentation](https://docs.joinmastodon.org/client/intro/) - - [Differences in Mastodon API responses from vanilla Mastodon](https://docs-develop.pleroma.social/backend/development/API/differences_in_mastoapi_responses/) + - [Differences in Mastodon API responses from vanilla Mastodon](https://docs.akkoma.dev/stable/development/API/differences_in_mastoapi_responses/) - Please report such occurences on our [issue tracker](https://git.pleroma.social/pleroma/pleroma/-/issues). Feel free to submit API questions or proposals there too! + Please report such occurrences on our [issue tracker](https://akkoma.dev/AkkomaGang/akkoma). Feel free to submit API questions or proposals there too! """, # Strip environment from the version version: Application.spec(:pleroma, :vsn) |> to_string() |> String.replace(~r/\+.*$/, ""), From bf7ff6a337a6a329524d3607b5ac2b4caa2df93a Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 30 Dec 2022 20:11:53 +0000 Subject: [PATCH 14/22] Put rich media processing in a Task --- .../web/mastodon_api/views/status_view.ex | 2 +- lib/pleroma/web/rich_media/parser.ex | 19 +++++++++++++++++-- .../controllers/status_controller_test.exs | 2 +- test/pleroma/web/rich_media/helpers_test.exs | 2 +- test/pleroma/web/rich_media/parser_test.exs | 2 +- 5 files changed, 21 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index cc58f803e..8fbf9b6d9 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do # to fetch the preview. However it should be fine considering # pagination is restricted to 40 activities at a time defp fetch_rich_media_for_activities(activities) do - Enum.each(activities, fn activity -> + Enum.map(activities, fn activity -> spawn(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index d6b54943b..1d4cad010 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -15,7 +15,7 @@ def parse(nil), do: {:error, "No URL provided"} if Pleroma.Config.get(:env) == :test do @spec parse(String.t()) :: {:ok, map()} | {:error, any()} - def parse(url), do: parse_url(url) + def parse(url), do: parse_with_timeout(url) else @spec parse(String.t()) :: {:ok, map()} | {:error, any()} def parse(url) do @@ -27,7 +27,7 @@ def parse(url) do defp get_cached_or_parse(url) do case @cachex.fetch(:rich_media_cache, url, fn -> - case parse_url(url) do + case parse_with_timeout(url) do {:ok, _} = res -> {:commit, res} @@ -141,6 +141,21 @@ def parse_url(url) do end end + def parse_with_timeout(url) do + try do + task = + Task.Supervisor.async_nolink(Pleroma.TaskSupervisor, fn -> + parse_url(url) + end) + + Task.await(task, 5000) + catch + :exit, {:timeout, _} -> + Logger.warn("Timeout while fetching rich media for #{url}") + {:error, :timeout} + end + end + defp maybe_parse(html) do Enum.reduce_while(parsers(), %{}, fn parser, acc -> case parser.parse(html, acc) do diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index d76db0c0d..7931d1d69 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -326,7 +326,7 @@ test "posting a fake status", %{conn: conn} do test "fake statuses' preview card is not cached", %{conn: conn} do clear_config([:rich_media, :enabled], true) - Tesla.Mock.mock(fn + Tesla.Mock.mock_global(fn %{ method: :get, url: "https://example.com/twitter-card" diff --git a/test/pleroma/web/rich_media/helpers_test.exs b/test/pleroma/web/rich_media/helpers_test.exs index 689854fb6..c6c3ffd6c 100644 --- a/test/pleroma/web/rich_media/helpers_test.exs +++ b/test/pleroma/web/rich_media/helpers_test.exs @@ -12,7 +12,7 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do import Tesla.Mock setup do - mock(fn env -> apply(HttpRequestMock, :request, [env]) end) + mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) :ok end diff --git a/test/pleroma/web/rich_media/parser_test.exs b/test/pleroma/web/rich_media/parser_test.exs index 2fe7f1b0b..b6444ac82 100644 --- a/test/pleroma/web/rich_media/parser_test.exs +++ b/test/pleroma/web/rich_media/parser_test.exs @@ -8,7 +8,7 @@ defmodule Pleroma.Web.RichMedia.ParserTest do alias Pleroma.Web.RichMedia.Parser setup do - Tesla.Mock.mock(fn + Tesla.Mock.mock_global(fn %{ method: :get, url: "http://example.com/ogp" From c8f2c4b638b763ba4276882eba40f1981aabad9d Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sat, 31 Dec 2022 03:52:52 +0000 Subject: [PATCH 15/22] add changelog entry for timeouts --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3106854ba..6125c0962 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Admin scopes will be dropped on create - Rich media will now backoff for 20 minutes after a failure - Simplified HTTP signature processing +- Rich media will now hard-exit after 5 seconds, to prevent timeline hangs ### Fixed - /api/v1/accounts/lookup will now respect restrict\_unauthenticated From b8f280b4b500a20eeb6046dd29e3070696da8977 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sat, 31 Dec 2022 03:53:52 +0000 Subject: [PATCH 16/22] Rich media doesn't need to be a map --- lib/pleroma/web/mastodon_api/views/status_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 8fbf9b6d9..cc58f803e 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do # to fetch the preview. However it should be fine considering # pagination is restricted to 40 activities at a time defp fetch_rich_media_for_activities(activities) do - Enum.map(activities, fn activity -> + Enum.each(activities, fn activity -> spawn(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) From 745e15468e64991e710467e3809bc62424014a52 Mon Sep 17 00:00:00 2001 From: ilja Date: Sat, 31 Dec 2022 18:09:27 +0000 Subject: [PATCH 17/22] Use same context for quote posts as the post that's being quoted (#379) See https://akkoma.dev/AkkomaGang/akkoma/pulls/350#issuecomment-6109 When making quotes through Mast-API, they will now have the same context as the quoted post. This also results in them being showed when fetching the thread. I checked Misskey to see how it's there, and they show the quotes there as well, see e.g. . An example from Akkoma: Co-authored-by: ilja Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/379 Reviewed-by: floatingghost Co-authored-by: ilja Co-committed-by: ilja --- CHANGELOG.md | 1 + lib/pleroma/web/common_api/activity_draft.ex | 2 +- lib/pleroma/web/common_api/utils.ex | 7 ++++--- .../mastodon_api/controllers/status_controller_test.exs | 1 + 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6125c0962..73563c312 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Non-admin users now cannot register `admin` scope tokens (not security-critical, they didn't work before, but you _could_ create them) - Admin scopes will be dropped on create - Rich media will now backoff for 20 minutes after a failure +- Quote posts are now considered as part of the same thread as the post they are quoting - Simplified HTTP signature processing - Rich media will now hard-exit after 5 seconds, to prevent timeline hangs diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index b3a49de44..8b0eaaadf 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -177,7 +177,7 @@ defp to_and_cc(draft) do end defp context(draft) do - context = Utils.make_context(draft.in_reply_to, draft.in_reply_to_conversation) + context = Utils.make_context(draft) %__MODULE__{draft | context: context} end diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 22594be46..aee19a840 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -230,12 +230,13 @@ def get_content_type(content_type) do end end - def make_context(_, %Participation{} = participation) do + def make_context(%{in_reply_to_conversation: %Participation{} = participation}) do Repo.preload(participation, :conversation).conversation.ap_id end - def make_context(%Activity{data: %{"context" => context}}, _), do: context - def make_context(_, _), do: Utils.generate_context_id() + def make_context(%{in_reply_to: %Activity{data: %{"context" => context}}}), do: context + def make_context(%{quote: %Activity{data: %{"context" => context}}}), do: context + def make_context(_), do: Utils.generate_context_id() def maybe_add_attachments(parsed, _attachments, false = _no_links), do: parsed diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index 7931d1d69..45da22d91 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2023,6 +2023,7 @@ test "posting a quote", %{conn: conn} do assert response["quote_id"] == quoted_status.id assert response["quote"]["id"] == quoted_status.id assert response["quote"]["content"] == quoted_status.object.data["content"] + assert response["pleroma"]["context"] == quoted_status.data["context"] end test "posting a quote, quoting a status that isn't public", %{conn: conn} do From c4b46ca4600ad8a91892dd3fcc033d0e83216ffc Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sat, 31 Dec 2022 18:05:21 +0000 Subject: [PATCH 18/22] Add /api/v1/followed_tags --- CHANGELOG.md | 1 + lib/pleroma/pagination.ex | 6 +- lib/pleroma/user/hashtag_follow.ex | 8 ++- .../web/api_spec/operations/tag_operation.ex | 40 +++++++++++- lib/pleroma/web/api_spec/schemas/tag.ex | 6 ++ .../controllers/tag_controller.ex | 32 +++++++++- .../web/mastodon_api/views/tag_view.ex | 4 ++ lib/pleroma/web/router.ex | 1 + .../controllers/tag_controller_test.exs | 62 +++++++++++++++++++ 9 files changed, 154 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73563c312..ee3c28858 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Argon2 password hashing - Ability to "verify" links in profile fields via rel=me - Mix tasks to dump/load config to/from json for bulk editing +- Followed hashtag list at /api/v1/followed\_tags, API parity with mastodon ### Removed - Non-finch HTTP adapters diff --git a/lib/pleroma/pagination.ex b/lib/pleroma/pagination.ex index 33e45a0eb..28e37933e 100644 --- a/lib/pleroma/pagination.ex +++ b/lib/pleroma/pagination.ex @@ -88,9 +88,9 @@ def paginate(query, options, :offset, table_binding) do defp cast_params(params) do param_types = %{ - min_id: :string, - since_id: :string, - max_id: :string, + min_id: params[:id_type] || :string, + since_id: params[:id_type] || :string, + max_id: params[:id_type] || :string, offset: :integer, limit: :integer, skip_extra_order: :boolean, diff --git a/lib/pleroma/user/hashtag_follow.ex b/lib/pleroma/user/hashtag_follow.ex index 43ed93f4d..dd0254ef4 100644 --- a/lib/pleroma/user/hashtag_follow.ex +++ b/lib/pleroma/user/hashtag_follow.ex @@ -43,7 +43,13 @@ def get(%User{} = user, %Hashtag{} = hashtag) do end def get_by_user(%User{} = user) do - Ecto.assoc(user, :followed_hashtags) + user + |> followed_hashtags_query() |> Repo.all() end + + def followed_hashtags_query(%User{} = user) do + Ecto.assoc(user, :followed_hashtags) + |> Ecto.Query.order_by([h], desc: h.id) + end end diff --git a/lib/pleroma/web/api_spec/operations/tag_operation.ex b/lib/pleroma/web/api_spec/operations/tag_operation.ex index e22457159..ce4f4ad5b 100644 --- a/lib/pleroma/web/api_spec/operations/tag_operation.ex +++ b/lib/pleroma/web/api_spec/operations/tag_operation.ex @@ -44,7 +44,7 @@ def unfollow_operation do tags: ["Tags"], summary: "Unfollow a hashtag", description: "Unfollow a hashtag", - security: [%{"oAuth" => ["write:follow"]}], + security: [%{"oAuth" => ["write:follows"]}], parameters: [id_param()], operationId: "TagController.unfollow", responses: %{ @@ -54,6 +54,26 @@ def unfollow_operation do } end + def show_followed_operation do + %Operation{ + tags: ["Tags"], + summary: "Followed hashtags", + description: "View a list of hashtags the currently authenticated user is following", + parameters: pagination_params(), + security: [%{"oAuth" => ["read:follows"]}], + operationId: "TagController.show_followed", + responses: %{ + 200 => + Operation.response("Hashtags", "application/json", %Schema{ + type: :array, + items: Tag + }), + 403 => Operation.response("Forbidden", "application/json", ApiError), + 404 => Operation.response("Not Found", "application/json", ApiError) + } + } + end + defp id_param do Operation.parameter( :id, @@ -62,4 +82,22 @@ defp id_param do "Name of the hashtag" ) end + + def pagination_params do + [ + Operation.parameter(:max_id, :query, :integer, "Return items older than this ID"), + Operation.parameter( + :min_id, + :query, + :integer, + "Return the oldest items newer than this ID" + ), + Operation.parameter( + :limit, + :query, + %Schema{type: :integer, default: 20}, + "Maximum number of items to return. Will be ignored if it's more than 40" + ) + ] + end end diff --git a/lib/pleroma/web/api_spec/schemas/tag.ex b/lib/pleroma/web/api_spec/schemas/tag.ex index 41b5e5c78..657fc3d2b 100644 --- a/lib/pleroma/web/api_spec/schemas/tag.ex +++ b/lib/pleroma/web/api_spec/schemas/tag.ex @@ -21,6 +21,12 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Tag do following: %Schema{ type: :boolean, description: "Whether the authenticated user is following the hashtag" + }, + history: %Schema{ + type: :array, + items: %Schema{type: :string}, + description: + "A list of historical uses of the hashtag (not implemented, for compatibility only)" } }, example: %{ diff --git a/lib/pleroma/web/mastodon_api/controllers/tag_controller.ex b/lib/pleroma/web/mastodon_api/controllers/tag_controller.ex index b8995eb00..ca5ee48ac 100644 --- a/lib/pleroma/web/mastodon_api/controllers/tag_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/tag_controller.ex @@ -4,9 +4,24 @@ defmodule Pleroma.Web.MastodonAPI.TagController do alias Pleroma.User alias Pleroma.Hashtag + alias Pleroma.Pagination + + import Pleroma.Web.ControllerHelper, + only: [ + add_link_headers: 2 + ] plug(Pleroma.Web.ApiSpec.CastAndValidate) - plug(Pleroma.Web.Plugs.OAuthScopesPlug, %{scopes: ["read"]} when action in [:show]) + + plug( + Pleroma.Web.Plugs.OAuthScopesPlug, + %{scopes: ["read"]} when action in [:show] + ) + + plug( + Pleroma.Web.Plugs.OAuthScopesPlug, + %{scopes: ["read:follows"]} when action in [:show_followed] + ) plug( Pleroma.Web.Plugs.OAuthScopesPlug, @@ -44,4 +59,19 @@ def unfollow(conn, %{id: id}) do _ -> render_error(conn, :not_found, "Hashtag not found") end end + + def show_followed(conn, params) do + with %{assigns: %{user: %User{} = user}} <- conn do + params = Map.put(params, :id_type, :integer) + + hashtags = + user + |> User.HashtagFollow.followed_hashtags_query() + |> Pagination.fetch_paginated(params) + + conn + |> add_link_headers(hashtags) + |> render("index.json", tags: hashtags, for_user: user) + end + end end diff --git a/lib/pleroma/web/mastodon_api/views/tag_view.ex b/lib/pleroma/web/mastodon_api/views/tag_view.ex index 6e491c261..e24d423c2 100644 --- a/lib/pleroma/web/mastodon_api/views/tag_view.ex +++ b/lib/pleroma/web/mastodon_api/views/tag_view.ex @@ -3,6 +3,10 @@ defmodule Pleroma.Web.MastodonAPI.TagView do alias Pleroma.User alias Pleroma.Web.Router.Helpers + def render("index.json", %{tags: tags, for_user: user}) do + safe_render_many(tags, __MODULE__, "show.json", %{for_user: user}) + end + def render("show.json", %{tag: tag, for_user: user}) do following = with %User{} <- user do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index f984ad598..b1433f180 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -606,6 +606,7 @@ defmodule Pleroma.Web.Router do get("/tags/:id", TagController, :show) post("/tags/:id/follow", TagController, :follow) post("/tags/:id/unfollow", TagController, :unfollow) + get("/followed_tags", TagController, :show_followed) end scope "/api/web", Pleroma.Web do diff --git a/test/pleroma/web/mastodon_api/controllers/tag_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/tag_controller_test.exs index a1b73ad78..71c8e7fc0 100644 --- a/test/pleroma/web/mastodon_api/controllers/tag_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/tag_controller_test.exs @@ -94,4 +94,66 @@ test "should 404 if hashtag doesn't exist" do assert response["error"] == "Hashtag not found" end end + + describe "GET /api/v1/followed_tags" do + test "should list followed tags" do + %{user: user, conn: conn} = oauth_access(["read:follows"]) + + response = + conn + |> get("/api/v1/followed_tags") + |> json_response_and_validate_schema(200) + + assert Enum.empty?(response) + + hashtag = insert(:hashtag, name: "jubjub") + {:ok, _user} = User.follow_hashtag(user, hashtag) + + response = + conn + |> get("/api/v1/followed_tags") + |> json_response_and_validate_schema(200) + + assert [%{"name" => "jubjub"}] = response + end + + test "should include a link header to paginate" do + %{user: user, conn: conn} = oauth_access(["read:follows"]) + + for i <- 1..21 do + hashtag = insert(:hashtag, name: "jubjub#{i}}") + {:ok, _user} = User.follow_hashtag(user, hashtag) + end + + response = + conn + |> get("/api/v1/followed_tags") + + json = json_response_and_validate_schema(response, 200) + assert Enum.count(json) == 20 + assert [link_header] = get_resp_header(response, "link") + assert link_header =~ "rel=\"next\"" + next_link = extract_next_link_header(link_header) + + response = + conn + |> get(next_link) + |> json_response_and_validate_schema(200) + + assert Enum.count(response) == 1 + end + + test "should refuse access without read:follows scope" do + %{conn: conn} = oauth_access(["write"]) + + conn + |> get("/api/v1/followed_tags") + |> json_response_and_validate_schema(403) + end + end + + defp extract_next_link_header(header) do + [_, next_link] = Regex.run(~r{<(?.*)>; rel="next"}, header) + next_link + end end From 6e646c4cbc56b3cf8134ca9005c1818c5947dd55 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sun, 1 Jan 2023 18:32:14 +0000 Subject: [PATCH 19/22] Use a genserver to periodically fetch metrics Ref https://github.com/beam-telemetry/telemetry_metrics_prometheus_core/issues/52 --- lib/pleroma/prometheus_exporter.ex | 49 +++++++++++++++++++ .../controllers/metrics_controller.ex | 2 +- lib/pleroma/web/telemetry.ex | 23 +++++++-- .../akkoma_api/metrics_controller_test.exs | 2 + 4 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 lib/pleroma/prometheus_exporter.ex diff --git a/lib/pleroma/prometheus_exporter.ex b/lib/pleroma/prometheus_exporter.ex new file mode 100644 index 000000000..05170c8bb --- /dev/null +++ b/lib/pleroma/prometheus_exporter.ex @@ -0,0 +1,49 @@ +defmodule Pleroma.PrometheusExporter do + @moduledoc """ + Exports metrics in Prometheus format. + Mostly exists because of https://github.com/beam-telemetry/telemetry_metrics_prometheus_core/issues/52 + Basically we need to fetch metrics every so often, or the lib will let them pile up and eventually crash the VM. + It also sorta acts as a cache so there is that too. + """ + + use GenServer + require Logger + + def start_link(_opts) do + GenServer.start_link(__MODULE__, :ok, name: __MODULE__) + end + + def init(_opts) do + schedule_next() + {:ok, ""} + end + + defp schedule_next do + Process.send_after(self(), :gather, 60_000) + end + + # Scheduled function, gather metrics and schedule next run + def handle_info(:gather, _state) do + schedule_next() + state = TelemetryMetricsPrometheus.Core.scrape() + {:noreply, state} + end + + # Trigger the call dynamically, mostly for testing + def handle_call(:gather, _from, _state) do + state = TelemetryMetricsPrometheus.Core.scrape() + {:reply, state, state} + end + + def handle_call(:show, _from, state) do + {:reply, state, state} + end + + def show do + GenServer.call(__MODULE__, :show) + end + + def gather do + GenServer.call(__MODULE__, :gather) + end +end diff --git a/lib/pleroma/web/akkoma_api/controllers/metrics_controller.ex b/lib/pleroma/web/akkoma_api/controllers/metrics_controller.ex index cc7a616ee..ab52cb64d 100644 --- a/lib/pleroma/web/akkoma_api/controllers/metrics_controller.ex +++ b/lib/pleroma/web/akkoma_api/controllers/metrics_controller.ex @@ -15,7 +15,7 @@ defmodule Pleroma.Web.AkkomaAPI.MetricsController do def show(conn, _params) do if Config.get([:instance, :export_prometheus_metrics], true) do conn - |> text(TelemetryMetricsPrometheus.Core.scrape()) + |> text(Pleroma.PrometheusExporter.show()) else conn |> send_resp(404, "Not Found") diff --git a/lib/pleroma/web/telemetry.ex b/lib/pleroma/web/telemetry.ex index acb649421..b03850600 100644 --- a/lib/pleroma/web/telemetry.ex +++ b/lib/pleroma/web/telemetry.ex @@ -2,6 +2,7 @@ defmodule Pleroma.Web.Telemetry do use Supervisor import Telemetry.Metrics alias Pleroma.Stats + alias Pleroma.Config def start_link(arg) do Supervisor.start_link(__MODULE__, arg, name: __MODULE__) @@ -9,14 +10,28 @@ def start_link(arg) do @impl true def init(_arg) do - children = [ - {:telemetry_poller, measurements: periodic_measurements(), period: 10_000}, - {TelemetryMetricsPrometheus.Core, metrics: prometheus_metrics()} - ] + children = + [ + {:telemetry_poller, measurements: periodic_measurements(), period: 10_000} + ] ++ + prometheus_children() Supervisor.init(children, strategy: :one_for_one) end + defp prometheus_children do + config = Config.get([:instance, :export_prometheus_metrics], true) + + if config do + [ + {TelemetryMetricsPrometheus.Core, metrics: prometheus_metrics()}, + Pleroma.PrometheusExporter + ] + else + [] + end + end + # A seperate set of metrics for distributions because phoenix dashboard does NOT handle them well defp distribution_metrics do [ diff --git a/test/pleroma/web/akkoma_api/metrics_controller_test.exs b/test/pleroma/web/akkoma_api/metrics_controller_test.exs index 9482f1312..4b096237c 100644 --- a/test/pleroma/web/akkoma_api/metrics_controller_test.exs +++ b/test/pleroma/web/akkoma_api/metrics_controller_test.exs @@ -5,6 +5,8 @@ defmodule Pleroma.Web.AkkomaAPI.MetricsControllerTest do test "should return metrics when the user has admin:metrics" do %{conn: conn} = oauth_access(["admin:metrics"]) + Pleroma.PrometheusExporter.gather() + resp = conn |> get("/api/v1/akkoma/metrics") From 6a333ade7f0d8e096df017b6fb355fb4d39e393c Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sun, 1 Jan 2023 18:54:08 +0000 Subject: [PATCH 20/22] Fix task name for robotstxt Fixes #408 --- docs/docs/administration/CLI_tasks/robots_txt.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/administration/CLI_tasks/robots_txt.md b/docs/docs/administration/CLI_tasks/robots_txt.md index 6cb9fd673..924f2e319 100644 --- a/docs/docs/administration/CLI_tasks/robots_txt.md +++ b/docs/docs/administration/CLI_tasks/robots_txt.md @@ -11,11 +11,11 @@ If you want to generate a restrictive `robots.txt`, you can run the following mi === "OTP" ```sh - ./bin/pleroma_ctl robots_txt disallow_all + ./bin/pleroma_ctl robotstxt disallow_all ``` === "From Source" ```sh - mix pleroma.robots_txt disallow_all + mix pleroma.robotstxt disallow_all ``` From 57e51fe62cb8bc1899ba2a58720ce3ddabfd083b Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Mon, 2 Jan 2023 03:29:02 +0000 Subject: [PATCH 21/22] Migrate Pleroma.Web to phoenix 1.6 formats --- lib/pleroma/web.ex | 134 ++++++++++-------- .../web/admin_api/views/status_view.ex | 4 +- .../mastodon_api/views/conversation_view.ex | 2 +- .../mastodon_api/views/notification_view.ex | 2 +- .../web/mastodon_api/views/status_view.ex | 2 +- .../web/mastodon_api/views/tag_view.ex | 2 +- lib/pleroma/web/router.ex | 2 +- 7 files changed, 81 insertions(+), 67 deletions(-) diff --git a/lib/pleroma/web.ex b/lib/pleroma/web.ex index 24560d4a3..ecd98b6ca 100644 --- a/lib/pleroma/web.ex +++ b/lib/pleroma/web.ex @@ -132,66 +132,6 @@ defp maybe_halt_on_missing_oauth_scopes_check(conn) do end end - def view do - quote do - use Phoenix.View, - root: "lib/pleroma/web/templates", - namespace: Pleroma.Web - - # Import convenience functions from controllers - import Phoenix.Controller, only: [get_csrf_token: 0, get_flash: 2, view_module: 1] - - import Pleroma.Web.ErrorHelpers - import Pleroma.Web.Gettext - - alias Pleroma.Web.Router.Helpers, as: Routes - - require Logger - - @doc "Same as `render/3` but wrapped in a rescue block" - def safe_render(view, template, assigns \\ %{}) do - Phoenix.View.render(view, template, assigns) - rescue - error -> - Logger.error( - "#{__MODULE__} failed to render #{inspect({view, template})}\n" <> - Exception.format(:error, error, __STACKTRACE__) - ) - - nil - end - - @doc """ - Same as `render_many/4` but wrapped in rescue block. - """ - def safe_render_many(collection, view, template, assigns \\ %{}) do - Enum.map(collection, fn resource -> - as = Map.get(assigns, :as) || view.__resource__ - assigns = Map.put(assigns, as, resource) - safe_render(view, template, assigns) - end) - |> Enum.filter(& &1) - end - end - end - - def router do - quote do - use Phoenix.Router - # credo:disable-for-next-line Credo.Check.Consistency.MultiAliasImportRequireUse - import Plug.Conn - import Phoenix.Controller - end - end - - def channel do - quote do - # credo:disable-for-next-line Credo.Check.Consistency.MultiAliasImportRequireUse - import Phoenix.Channel - import Pleroma.Web.Gettext - end - end - def plug do quote do @behaviour Pleroma.Web.Plug @@ -236,6 +176,80 @@ def call(%Plug.Conn{} = conn, options) do end end + def view do + quote do + use Phoenix.View, + root: "lib/pleroma/web/templates", + namespace: Pleroma.Web + + # Import convenience functions from controllers + import Phoenix.Controller, + only: [get_flash: 1, get_flash: 2, view_module: 1, view_template: 1] + + # Include shared imports and aliases for views + unquote(view_helpers()) + end + end + + def live_view do + quote do + use Phoenix.LiveView, + layout: {Pleroma.Web.LayoutView, "live.html"} + + unquote(view_helpers()) + end + end + + def live_component do + quote do + use Phoenix.LiveComponent + + unquote(view_helpers()) + end + end + + def component do + quote do + use Phoenix.Component + + unquote(view_helpers()) + end + end + + def router do + quote do + use Phoenix.Router + + import Plug.Conn + import Phoenix.Controller + import Phoenix.LiveView.Router + end + end + + def channel do + quote do + use Phoenix.Channel + import Pleroma.Web.Gettext + end + end + + defp view_helpers do + quote do + # Use all HTML functionality (forms, tags, etc) + use Phoenix.HTML + + # Import LiveView and .heex helpers (live_render, live_patch, <.form>, etc) + import Phoenix.LiveView.Helpers + + # Import basic rendering functionality (render, render_layout, etc) + import Phoenix.View + + import Pleroma.Web.ErrorHelpers + import Pleroma.Web.Gettext + alias Pleroma.Web.Router.Helpers, as: Routes + end + end + @doc """ When used, dispatch to the appropriate controller/view/etc. """ diff --git a/lib/pleroma/web/admin_api/views/status_view.ex b/lib/pleroma/web/admin_api/views/status_view.ex index 48d639b41..a252b047c 100644 --- a/lib/pleroma/web/admin_api/views/status_view.ex +++ b/lib/pleroma/web/admin_api/views/status_view.ex @@ -14,11 +14,11 @@ defmodule Pleroma.Web.AdminAPI.StatusView do defdelegate merge_account_views(user), to: AdminAPI.AccountView def render("index.json", %{total: total} = opts) do - %{total: total, activities: safe_render_many(opts.activities, __MODULE__, "show.json", opts)} + %{total: total, activities: render_many(opts.activities, __MODULE__, "show.json", opts)} end def render("index.json", opts) do - safe_render_many(opts.activities, __MODULE__, "show.json", opts) + render_many(opts.activities, __MODULE__, "show.json", opts) end def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} = opts) do diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex index 46b63b54b..9c9b49c59 100644 --- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex +++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex @@ -12,7 +12,7 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do alias Pleroma.Web.MastodonAPI.StatusView def render("participations.json", %{participations: participations, for: user}) do - safe_render_many(participations, __MODULE__, "participation.json", %{ + render_many(participations, __MODULE__, "participation.json", %{ as: :participation, for: user }) diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 463d31d1a..e527ff608 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -66,7 +66,7 @@ def render("index.json", %{notifications: notifications, for: reading_user} = op |> Map.put(:parent_activities, parent_activities) |> Map.put(:relationships, relationships_opt) - safe_render_many(notifications, NotificationView, "show.json", opts) + render_many(notifications, NotificationView, "show.json", opts) end def render( diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index cc58f803e..b9a7e57f5 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -131,7 +131,7 @@ def render("index.json", opts) do |> Map.put(:parent_activities, parent_activities) |> Map.put(:relationships, relationships_opt) - safe_render_many(activities, StatusView, "show.json", opts) + render_many(activities, StatusView, "show.json", opts) end def render( diff --git a/lib/pleroma/web/mastodon_api/views/tag_view.ex b/lib/pleroma/web/mastodon_api/views/tag_view.ex index e24d423c2..02108c736 100644 --- a/lib/pleroma/web/mastodon_api/views/tag_view.ex +++ b/lib/pleroma/web/mastodon_api/views/tag_view.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Web.MastodonAPI.TagView do alias Pleroma.Web.Router.Helpers def render("index.json", %{tags: tags, for_user: user}) do - safe_render_many(tags, __MODULE__, "show.json", %{for_user: user}) + render_many(tags, __MODULE__, "show.json", %{for_user: user}) end def render("show.json", %{tag: tag, for_user: user}) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index b1433f180..faaf3d679 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -921,7 +921,7 @@ defmodule Pleroma.Web.Router do # TODO: Change to Phoenix.Router.routes/1 for Phoenix 1.6.0+ def get_api_routes do - __MODULE__.__routes__() + Phoenix.Router.routes(__MODULE__) |> Enum.reject(fn r -> r.plug == Pleroma.Web.Fallback.RedirectController end) |> Enum.map(fn r -> r.path From 336d06b2a8ca75362578b1d67ea1f32a45c8edd3 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Mon, 2 Jan 2023 15:21:19 +0000 Subject: [PATCH 22/22] Significantly tighten HTTP CSP --- CHANGELOG.md | 1 + lib/pleroma/web/plugs/http_security_plug.ex | 15 +++++---------- .../pleroma/web/plugs/http_security_plug_test.exs | 2 +- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee3c28858..8e638bdd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Quote posts are now considered as part of the same thread as the post they are quoting - Simplified HTTP signature processing - Rich media will now hard-exit after 5 seconds, to prevent timeline hangs +- HTTP Content Security Policy is now far more strict to prevent any potential XSS/CSS leakages ### Fixed - /api/v1/accounts/lookup will now respect restrict\_unauthenticated diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index 5f0b775be..b1f1ada94 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -106,20 +106,15 @@ defp csp_string(conn) do connect_src = if Config.get([:media_proxy, :enabled]) do sources = build_csp_multimedia_source_list() - ["connect-src 'self' blob: ", static_url, ?\s, websocket_url, ?\s, sources] + ["connect-src 'self' ", static_url, ?\s, websocket_url, ?\s, sources] else - ["connect-src 'self' blob: ", static_url, ?\s, websocket_url] + ["connect-src 'self' ", static_url, ?\s, websocket_url] end - style_src = "style-src 'self' 'unsafe-inline'" - font_src = "font-src 'self' data:" + style_src = "style-src 'self' '#{nonce_tag}'" + font_src = "font-src 'self'" - script_src = - if Config.get(:env) == :dev do - "script-src 'self' 'unsafe-eval' '#{nonce_tag}'" - else - "script-src 'self' '#{nonce_tag}'" - end + script_src = "script-src 'self' '#{nonce_tag}'" report = if report_uri, do: ["report-uri ", report_uri, ";report-to csp-endpoint"] insecure = if scheme == "https", do: "upgrade-insecure-requests" diff --git a/test/pleroma/web/plugs/http_security_plug_test.exs b/test/pleroma/web/plugs/http_security_plug_test.exs index d6d841078..d88d4624f 100644 --- a/test/pleroma/web/plugs/http_security_plug_test.exs +++ b/test/pleroma/web/plugs/http_security_plug_test.exs @@ -140,7 +140,7 @@ defp assert_media_img_src(conn, url) do defp assert_connect_src(conn, url) do conn = get(conn, "/api/v1/instance") [csp] = Conn.get_resp_header(conn, "content-security-policy") - assert csp =~ ~r/connect-src 'self' blob: [^;]+ #{url}/ + assert csp =~ ~r/connect-src 'self' [^;]+ #{url}/ end test "it does not send CSP headers when disabled", %{conn: conn} do