From 2937495712d760ca8a9388e69cc2336636830743 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Thu, 30 Jun 2022 18:44:31 +0000 Subject: [PATCH] fix ES import from live (#29) Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/29 --- config/config.exs | 2 +- docs/configuration/search.md | 6 +++--- lib/mix/pleroma.ex | 13 +++++++++++- lib/mix/tasks/pleroma/search.ex | 20 +++++++++++++++++++ lib/mix/tasks/pleroma/search/elasticsearch.ex | 9 --------- 5 files changed, 36 insertions(+), 14 deletions(-) create mode 100644 lib/mix/tasks/pleroma/search.ex delete mode 100644 lib/mix/tasks/pleroma/search/elasticsearch.ex diff --git a/config/config.exs b/config/config.exs index 727a2b0cb..b00fd18e3 100644 --- a/config/config.exs +++ b/config/config.exs @@ -874,7 +874,7 @@ settings: "priv/es-mappings/activity.json", store: Pleroma.Search.Elasticsearch.Store, sources: [Pleroma.Activity], - bulk_page_size: 5000, + bulk_page_size: 1000, bulk_wait_interval: 15_000 } } diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 7c1093ab9..e1f23b505 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -154,10 +154,10 @@ To start the initial indexing, run the `build` command: === "OTP" ```sh -./bin/pleroma_ctl search.elasticsearch index activities --cluster Pleroma.Search.Elasticsearch.Cluster +./bin/pleroma_ctl search import activities ``` === "From Source" ```sh -mix elasticsearch.build activities --cluster Pleroma.Search.Elasticsearch.Cluster -``` \ No newline at end of file +mix pleroma.search import activities +``` diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index 2b6c7d6bb..02c40850a 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -57,7 +57,8 @@ def start_pleroma do {Majic.Pool, [name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]} ] ++ - http_children(adapter) + http_children(adapter) ++ + elasticsearch_children() cachex_children = Enum.map(@cachex_children, &Pleroma.Application.build_cachex(&1, [])) @@ -136,4 +137,14 @@ defp http_children(Tesla.Adapter.Gun) do end defp http_children(_), do: [] + + def elasticsearch_children do + config = Pleroma.Config.get([Pleroma.Search, :module]) + + if config == Pleroma.Search.Elasticsearch do + [Pleroma.Search.Elasticsearch.Cluster] + else + [] + end + end end diff --git a/lib/mix/tasks/pleroma/search.ex b/lib/mix/tasks/pleroma/search.ex new file mode 100644 index 000000000..102bc5b63 --- /dev/null +++ b/lib/mix/tasks/pleroma/search.ex @@ -0,0 +1,20 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Search do + use Mix.Task + import Mix.Pleroma + + @shortdoc "Manages elasticsearch" + + def run(["import", "activities" | _rest]) do + start_pleroma() + + Elasticsearch.Index.Bulk.upload( + Pleroma.Search.Elasticsearch.Cluster, + "activities", + Pleroma.Config.get([Pleroma.Search.Elasticsearch.Cluster, :indexes, :activities]) + ) + end +end diff --git a/lib/mix/tasks/pleroma/search/elasticsearch.ex b/lib/mix/tasks/pleroma/search/elasticsearch.ex deleted file mode 100644 index 1d7d7a29a..000000000 --- a/lib/mix/tasks/pleroma/search/elasticsearch.ex +++ /dev/null @@ -1,9 +0,0 @@ -defmodule Mix.Tasks.Pleroma.Search.Elasticsearch do - alias Mix.Tasks.Elasticsearch.Build - import Mix.Pleroma - - def run(["index" | args]) do - start_pleroma() - Build.run(args) - end -end