From 3478492945d7069545593cc0e6e6b9e536ddb489 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sat, 11 Dec 2021 18:48:46 +0000 Subject: [PATCH] integrate search endpoint with ES --- config/config.exs | 3 ++ .../elasticsearch/document_mappings/note.ex | 1 + lib/pleroma/elasticsearch/store.ex | 11 +++++- lib/pleroma/web/common_api.ex | 19 +++++++++- .../controllers/search_controller.ex | 37 +++++++++++++++++++ 5 files changed, 69 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 681b49827..dea05d276 100644 --- a/config/config.exs +++ b/config/config.exs @@ -851,6 +851,9 @@ {Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]} ] +config :pleroma, :search, + provider: :builtin + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/lib/pleroma/elasticsearch/document_mappings/note.ex b/lib/pleroma/elasticsearch/document_mappings/note.ex index f4e3307fe..60efde599 100644 --- a/lib/pleroma/elasticsearch/document_mappings/note.ex +++ b/lib/pleroma/elasticsearch/document_mappings/note.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Elasticsearch.DocumentMappings.Activity do def id(obj), do: obj.id def encode(%{object: %{data: %{ "type" => "Note" }}} = activity) do %{ + _timestamp: activity.inserted_at, user: activity.user_actor.nickname, content: activity.object.data["content"], instance: URI.parse(activity.user_actor.ap_id).host, diff --git a/lib/pleroma/elasticsearch/store.ex b/lib/pleroma/elasticsearch/store.ex index 2ff4bf889..d9e9ed1a7 100644 --- a/lib/pleroma/elasticsearch/store.ex +++ b/lib/pleroma/elasticsearch/store.ex @@ -26,11 +26,20 @@ def bulk_post(data, :activities) do end) |> List.flatten() - IO.inspect Elastix.Bulk.post( + Elastix.Bulk.post( url(), d, index: "activities", type: "activity" ) end + + def search(query) do + Elastix.Search.search( + url(), + "activities", + ["activity"], + %{query: %{term: %{content: query}}} + ) + end end diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 6f685cb7b..95ac7b71a 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -16,6 +16,8 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.CommonAPI.ActivityDraft + alias Pleroma.Elasticsearch + alias Pleroma.Config import Pleroma.Web.Gettext import Pleroma.Web.CommonAPI.Utils @@ -395,9 +397,24 @@ def listen(user, data) do end end + def maybe_put_into_elasticsearch({:ok, activity}) do + if Config.get([:search, :provider]) == :elasticsearch do + actor = Pleroma.Activity.user_actor(activity) + activity + |> Map.put(:user_actor, actor) + |> Elasticsearch.put() + end + end + + def maybe_put_into_elasticsearch(_) do + {:ok, :skipped} + end + def post(user, %{status: _} = data) do with {:ok, draft} <- ActivityDraft.create(user, data) do - ActivityPub.create(draft.changes, draft.preview?) + activity = ActivityPub.create(draft.changes, draft.preview?) + maybe_put_into_elasticsearch(activity) + activity end end diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 64b177eb3..484a959af 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -45,6 +45,43 @@ def search(conn, params), do: do_search(:v1, conn, params) defp do_search(version, %{assigns: %{user: user}} = conn, %{q: query} = params) do query = String.trim(query) + options = search_options(params, user) + if Pleroma.Config.get([:search, :provider]) == :elasticsearch do + elasticsearch_search(conn, query, options) + else + builtin_search(version, conn, params) + end + end + + defp elasticsearch_search(%{assigns: %{user: user}} = conn, query, options) do + with {:ok, raw_results} <- Pleroma.Elasticsearch.search(query) do + results = raw_results + |> Map.get(:body) + |> Map.get("hits") + |> Map.get("hits") + |> Enum.map(fn result -> result["_id"] end) + |> Pleroma.Activity.all_by_ids_with_object() + + json( + conn, + %{ + accounts: [], + hashtags: [], + statuses: StatusView.render("index.json", + activities: results, + for: user, + as: :activity + )} + ) + else + {:error, _} -> + conn + |> put_status(:internal_server_error) + |> json(%{error: "Search failed"}) + end + end + + defp builtin_search(version, %{assigns: %{user: user}} = conn, %{q: query} = params) do options = search_options(params, user) timeout = Keyword.get(Repo.config(), :timeout, 15_000) default_values = %{"statuses" => [], "accounts" => [], "hashtags" => []}