From 404a9ccb9a220f3f52ee03bd69bd3746d95794cc Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Wed, 18 Dec 2019 23:11:42 +0300 Subject: [PATCH 1/3] Stats: return status counts by scope --- CHANGELOG.md | 1 + lib/pleroma/stats.ex | 72 +++++++++++++++++-- test/stats_test.exs | 52 ++++++++++++++ .../controllers/instance_controller_test.exs | 10 ++- 4 files changed, 129 insertions(+), 6 deletions(-) create mode 100644 test/stats_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index c133cd9ec..f6cc193a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **Breaking:** Admin API: `PUT /api/pleroma/admin/reports/:id` is now `PATCH /api/pleroma/admin/reports`, see admin_api.md for details - **Breaking:** `/api/pleroma/admin/users/invite_token` now uses `POST`, changed accepted params and returns full invite in json instead of only token string. - **Breaking** replying to reports is now "report notes", enpoint changed from `POST /api/pleroma/admin/reports/:id/respond` to `POST /api/pleroma/admin/reports/:id/notes` +- **Breaking** `/api/v1/stats` now return statuses count by scope (i.e. `all`, `public`, `unlisted`, `direct` and `private`) - Admin API: Return `total` when querying for reports - Mastodon API: Return `pleroma.direct_conversation_id` when creating a direct message (`POST /api/v1/statuses`) - Admin API: Return link alongside with token on password reset diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 8154a09b7..c90e8f409 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -3,11 +3,15 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Stats do + use GenServer + import Ecto.Query + + alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User - use GenServer + require Pleroma.Constants @interval 1000 * 60 * 60 @@ -56,7 +60,7 @@ defp initial_data do %{peers: [], stats: %{}} end - defp get_stat_data do + def get_stat_data do peers = from( u in User, @@ -68,13 +72,71 @@ defp get_stat_data do domain_count = Enum.count(peers) - status_count = Repo.aggregate(User.Query.build(%{local: true}), :sum, :note_count) - user_count = Repo.aggregate(User.Query.build(%{local: true, active: true}), :count, :id) %{ peers: peers, - stats: %{domain_count: domain_count, status_count: status_count, user_count: user_count} + stats: %{domain_count: domain_count, status_count: status_count(), user_count: user_count} } end + + defp status_count do + %{ + all: get_all_statuses_count(), + public: public_statuses_query() |> Repo.aggregate(:count, :id), + unlisted: unlisted_statuses_query() |> Repo.aggregate(:count, :id), + direct: direct_statuses_query() |> Repo.aggregate(:count, :id), + private: private_statuses_query() |> Repo.aggregate(:count, :id) + } + end + + defp get_all_statuses_count do + Repo.aggregate(User.Query.build(%{local: true}), :sum, :note_count) + end + + def public_statuses_query do + from(o in Object, + where: fragment("(?)->'to' \\? ?", o.data, ^Pleroma.Constants.as_public()) + ) + end + + def unlisted_statuses_query do + from(o in Object, + where: not fragment("(?)->'to' \\? ?", o.data, ^Pleroma.Constants.as_public()), + where: fragment("(?)->'cc' \\? ?", o.data, ^Pleroma.Constants.as_public()) + ) + end + + def direct_statuses_query do + private_statuses_ids = from(p in private_statuses_query(), select: p.id) |> Repo.all() + + from(o in Object, + where: + fragment( + "? \\? 'directMessage' AND (?->>'directMessage')::boolean = true", + o.data, + o.data + ) or + (not fragment("(?)->'to' \\? ?", o.data, ^Pleroma.Constants.as_public()) and + not fragment("(?)->'cc' \\? ?", o.data, ^Pleroma.Constants.as_public()) and + o.id not in ^private_statuses_ids) + ) + end + + def private_statuses_query do + from(o in subquery(recipients_query()), + where: ilike(o.recipients, "%/followers%") + ) + end + + defp recipients_query do + from(o in Object, + select: %{ + id: o.id, + recipients: fragment("jsonb_array_elements_text((?)->'to')", o.data) + }, + where: not fragment("(?)->'to' \\? ?", o.data, ^Pleroma.Constants.as_public()), + where: not fragment("(?)->'cc' \\? ?", o.data, ^Pleroma.Constants.as_public()) + ) + end end diff --git a/test/stats_test.exs b/test/stats_test.exs new file mode 100644 index 000000000..31c2f8db3 --- /dev/null +++ b/test/stats_test.exs @@ -0,0 +1,52 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.StatsTest do + use Pleroma.DataCase + + import Pleroma.Factory + + alias Pleroma.Web.CommonAPI + + describe "statuses count" do + setup do + user = insert(:user) + other_user = insert(:user) + + CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"}) + + Enum.each(0..1, fn _ -> + CommonAPI.post(user, %{ + "visibility" => "unlisted", + "status" => "hey" + }) + end) + + Enum.each(0..2, fn _ -> + CommonAPI.post(user, %{ + "visibility" => "direct", + "status" => "hey @#{other_user.nickname}" + }) + end) + + Enum.each(0..3, fn _ -> + CommonAPI.post(user, %{ + "visibility" => "private", + "status" => "hey" + }) + end) + + :ok + end + + test "it returns total number of statuses" do + data = Pleroma.Stats.get_stat_data() + + assert data.stats.status_count.public == 1 + assert data.stats.status_count.unlisted == 2 + assert data.stats.status_count.direct == 3 + assert data.stats.status_count.private == 4 + end + end +end diff --git a/test/web/mastodon_api/controllers/instance_controller_test.exs b/test/web/mastodon_api/controllers/instance_controller_test.exs index e00de6b18..7aa7c8648 100644 --- a/test/web/mastodon_api/controllers/instance_controller_test.exs +++ b/test/web/mastodon_api/controllers/instance_controller_test.exs @@ -58,7 +58,15 @@ test "get instance stats", %{conn: conn} do assert stats assert stats["user_count"] == 1 - assert stats["status_count"] == 1 + + assert stats["status_count"] == %{ + "all" => 1, + "direct" => 0, + "private" => 0, + "public" => 1, + "unlisted" => 0 + } + assert stats["domain_count"] == 2 end From 4079d66f00adc665e4c70abad03eab254695e793 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Thu, 19 Dec 2019 19:47:36 +0300 Subject: [PATCH 2/3] Expose issue via failing test --- test/stats_test.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/stats_test.exs b/test/stats_test.exs index 31c2f8db3..02a92dc64 100644 --- a/test/stats_test.exs +++ b/test/stats_test.exs @@ -43,6 +43,7 @@ defmodule Pleroma.StatsTest do test "it returns total number of statuses" do data = Pleroma.Stats.get_stat_data() + assert data.stats.status_count.all == 10 assert data.stats.status_count.public == 1 assert data.stats.status_count.unlisted == 2 assert data.stats.status_count.direct == 3 From 5fc84552d311efd606f66775c55862b3d11ad258 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Thu, 19 Dec 2019 19:52:55 +0300 Subject: [PATCH 3/3] Fix all count --- lib/pleroma/stats.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index c90e8f409..97e8b1990 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -82,7 +82,7 @@ def get_stat_data do defp status_count do %{ - all: get_all_statuses_count(), + all: all_statuses_query() |> Repo.aggregate(:count, :id), public: public_statuses_query() |> Repo.aggregate(:count, :id), unlisted: unlisted_statuses_query() |> Repo.aggregate(:count, :id), direct: direct_statuses_query() |> Repo.aggregate(:count, :id), @@ -90,8 +90,8 @@ defp status_count do } end - defp get_all_statuses_count do - Repo.aggregate(User.Query.build(%{local: true}), :sum, :note_count) + defp all_statuses_query do + from(o in Object, where: fragment("(?)->>'type' = 'Note'", o.data)) end def public_statuses_query do