Merge branch 'hashtag-benchmarks' into 'develop'

Hashtag benchmarks

See merge request pleroma/pleroma!2116

benchmarks/load_testing/generator.ex

@@ -9,7 +9,7 @@ def generate_like_activities(user, posts) do
     {time, _} =
       :timer.tc(fn ->
         Task.async_stream(
-           Enum.take_random(posts, count_likes),
+          Enum.take_random(posts, count_likes),
           fn post -> {:ok, _, _} = CommonAPI.favorite(post.id, user) end,
           max_concurrency: 10,
           timeout: 30_000
@@ -142,6 +142,48 @@ defp do_generate_activity(users) do
     CommonAPI.post(Enum.random(users), post)
   end
 
+  def generate_power_intervals(opts \\ []) do
+    count = Keyword.get(opts, :count, 20)
+    power = Keyword.get(opts, :power, 2)
+    IO.puts("Generating #{count} intervals for a power #{power} series...")
+    counts = Enum.map(1..count, fn n -> :math.pow(n, power) end)
+    sum = Enum.sum(counts)
+
+    densities =
+      Enum.map(counts, fn c ->
+        c / sum
+      end)
+
+    densities
+    |> Enum.reduce(0, fn density, acc ->
+      if acc == 0 do
+        [{0, density}]
+      else
+        [{_, lower} | _] = acc
+        [{lower, lower + density} | acc]
+      end
+    end)
+    |> Enum.reverse()
+  end
+
+  def generate_tagged_activities(opts \\ []) do
+    tag_count = Keyword.get(opts, :tag_count, 20)
+    users = Keyword.get(opts, :users, Repo.all(User))
+    activity_count = Keyword.get(opts, :count, 200_000)
+
+    intervals = generate_power_intervals(count: tag_count)
+
+    IO.puts(
+      "Generating #{activity_count} activities using #{tag_count} different tags of format `tag_n`, starting at tag_0"
+    )
+
+    Enum.each(1..activity_count, fn _ ->
+      random = :rand.uniform()
+      i = Enum.find_index(intervals, fn {lower, upper} -> lower <= random && upper > random end)
+      CommonAPI.post(Enum.random(users), %{"status" => "a post with the tag #tag_#{i}"})
+    end)
+  end
+
   defp do_generate_activity_with_mention(user, users) do
     mentions_cnt = Enum.random([2, 3, 4, 5])
     with_user = Enum.random([true, false])

benchmarks/mix/tasks/pleroma/benchmarks/tags.ex

@@ -0,0 +1,87 @@
+defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
+  use Mix.Task
+  alias Pleroma.Repo
+  alias Pleroma.LoadTesting.Generator
+  import Ecto.Query
+
+  def run(_args) do
+    Mix.Pleroma.start_pleroma()
+    activities_count = Repo.aggregate(from(a in Pleroma.Activity), :count, :id)
+
+    if activities_count == 0 do
+      IO.puts("Did not find any activities, cleaning and generating")
+      clean_tables()
+      Generator.generate_users(users_max: 10)
+      Generator.generate_tagged_activities()
+    else
+      IO.puts("Found #{activities_count} activities, won't generate new ones")
+    end
+
+    tags = Enum.map(0..20, fn i -> {"For #tag_#{i}", "tag_#{i}"} end)
+
+    Enum.each(tags, fn {_, tag} ->
+      query =
+        from(o in Pleroma.Object,
+          where: fragment("(?)->'tag' \\? (?)", o.data, ^tag)
+        )
+
+      count = Repo.aggregate(query, :count, :id)
+      IO.puts("Database contains #{count} posts tagged with #{tag}")
+    end)
+
+    user = Repo.all(Pleroma.User) |> List.first()
+
+    Benchee.run(
+      %{
+        "Hashtag fetching, any" => fn tags ->
+          Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
+            %{
+              "any" => tags
+            },
+            user,
+            false
+          )
+        end,
+        # Will always return zero results because no overlapping hashtags are generated.
+        "Hashtag fetching, all" => fn tags ->
+          Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
+            %{
+              "all" => tags
+            },
+            user,
+            false
+          )
+        end
+      },
+      inputs:
+        tags
+        |> Enum.map(fn {_, v} -> v end)
+        |> Enum.chunk_every(2)
+        |> Enum.map(fn tags -> {"For #{inspect(tags)}", tags} end),
+      time: 5
+    )
+
+    Benchee.run(
+      %{
+        "Hashtag fetching" => fn tag ->
+          Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
+            %{
+              "tag" => tag
+            },
+            user,
+            false
+          )
+        end
+      },
+      inputs: tags,
+      time: 5
+    )
+  end
+
+  defp clean_tables do
+    IO.puts("Deleting old data...\n")
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
+  end
+end

lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex

@@ -77,10 +77,7 @@ def public(%{assigns: %{user: user}} = conn, params) do
     |> render("index.json", activities: activities, for: user, as: :activity)
   end
 
-  # GET /api/v1/timelines/tag/:tag
-  def hashtag(%{assigns: %{user: user}} = conn, params) do
-    local_only = truthy_param?(params["local"])
-
+  def hashtag_fetching(params, user, local_only) do
     tags =
       [params["tag"], params["any"]]
       |> List.flatten()
@@ -98,7 +95,7 @@ def hashtag(%{assigns: %{user: user}} = conn, params) do
       |> Map.get("none", [])
       |> Enum.map(&String.downcase(&1))
 
-    activities =
+    _activities =
       params
       |> Map.put("type", "Create")
       |> Map.put("local_only", local_only)
@@ -109,6 +106,13 @@ def hashtag(%{assigns: %{user: user}} = conn, params) do
       |> Map.put("tag_all", tag_all)
       |> Map.put("tag_reject", tag_reject)
       |> ActivityPub.fetch_public_activities()
+  end
+
+  # GET /api/v1/timelines/tag/:tag
+  def hashtag(%{assigns: %{user: user}} = conn, params) do
+    local_only = truthy_param?(params["local"])
+
+    activities = hashtag_fetching(params, user, local_only)
 
     conn
     |> add_link_headers(activities, %{"local" => local_only})