From 6ca70833f5a9c40d440faa986c010fa8fce3ed69 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Mon, 13 Dec 2021 20:27:35 +0000 Subject: [PATCH] add user import --- lib/mix/tasks/pleroma/search.ex | 25 ++++++++++--------- .../{note.ex => activity.ex} | 0 .../elasticsearch/document_mappings/user.ex | 12 +++++++++ lib/pleroma/elasticsearch/store.ex | 25 +++++++++++++++++++ priv/es-mappings/user.json | 4 +++ 5 files changed, 54 insertions(+), 12 deletions(-) rename lib/pleroma/elasticsearch/document_mappings/{note.ex => activity.ex} (100%) create mode 100644 lib/pleroma/elasticsearch/document_mappings/user.ex diff --git a/lib/mix/tasks/pleroma/search.ex b/lib/mix/tasks/pleroma/search.ex index 4e8e480d4..64396b92f 100644 --- a/lib/mix/tasks/pleroma/search.ex +++ b/lib/mix/tasks/pleroma/search.ex @@ -8,21 +8,28 @@ defmodule Mix.Tasks.Pleroma.Search do import Ecto.Query alias Pleroma.Activity alias Pleroma.Pagination + alias Pleroma.User @shortdoc "Manages elasticsearch" - def run(["import" | _rest]) do + def run(["import", "activities" | _rest]) do start_pleroma() from(a in Activity, where: not ilike(a.actor, "%/relay")) |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data)) |> Activity.with_preloaded_object() |> Activity.with_preloaded_user_actor() - |> get_all + |> get_all(:activities) end - defp get_all(query, max_id \\ nil) do - IO.puts(max_id) + def run(["import", "users" | _rest]) do + start_pleroma() + + from(u in User, where: not ilike(u.ap_id, "%/relay")) + |> get_all(:users) + end + + defp get_all(query, index, max_id \\ nil) do params = %{limit: 2000} params = @@ -40,15 +47,9 @@ defp get_all(query, max_id \\ nil) do :ok else res - |> Enum.filter(fn x -> - t = x.object - |> Map.get(:data, %{}) - |> Map.get("type", "") - t == "Note" - end) - |> Pleroma.Elasticsearch.bulk_post(:activities) + |> Pleroma.Elasticsearch.bulk_post(index) - get_all(query, List.last(res).id) + get_all(query, index, List.last(res).id) end end end diff --git a/lib/pleroma/elasticsearch/document_mappings/note.ex b/lib/pleroma/elasticsearch/document_mappings/activity.ex similarity index 100% rename from lib/pleroma/elasticsearch/document_mappings/note.ex rename to lib/pleroma/elasticsearch/document_mappings/activity.ex diff --git a/lib/pleroma/elasticsearch/document_mappings/user.ex b/lib/pleroma/elasticsearch/document_mappings/user.ex new file mode 100644 index 000000000..5d9945c23 --- /dev/null +++ b/lib/pleroma/elasticsearch/document_mappings/user.ex @@ -0,0 +1,12 @@ +defmodule Pleroma.Elasticsearch.DocumentMappings.User do + def id(obj), do: obj.id + + def encode(%{actor_type: "Person"} = user) do + %{ + timestamp: user.inserted_at, + instance: URI.parse(user.ap_id).host, + nickname: user.nickname, + bio: user.bio + } + end +end diff --git a/lib/pleroma/elasticsearch/store.ex b/lib/pleroma/elasticsearch/store.ex index f309bf7a5..776bad921 100644 --- a/lib/pleroma/elasticsearch/store.ex +++ b/lib/pleroma/elasticsearch/store.ex @@ -44,6 +44,12 @@ def put(%Activity{} = activity) do def bulk_post(data, :activities) do d = data + |> Enum.filter(fn x -> + t = x.object + |> Map.get(:data, %{}) + |> Map.get("type", "") + t == "Note" + end) |> Enum.map(fn d -> [ %{index: %{_id: DocumentMappings.Activity.id(d)}}, @@ -60,6 +66,25 @@ def bulk_post(data, :activities) do ) end + def bulk_post(data, :users) do + d = + data + |> Enum.map(fn d -> + [ + %{index: %{_id: DocumentMappings.User.id(d)}}, + DocumentMappings.User.encode(d) + ] + end) + |> List.flatten() + + Elastix.Bulk.post( + url(), + d, + index: "users", + type: "user" + ) + end + def search_activities(q) do Elastix.Search.search( url(), diff --git a/priv/es-mappings/user.json b/priv/es-mappings/user.json index 20beb75c2..77cc66a4b 100644 --- a/priv/es-mappings/user.json +++ b/priv/es-mappings/user.json @@ -1,5 +1,9 @@ { "properties": { + "timestamp": { + "type": "date", + "index": true + }, "instance": { "type": "keyword" },