From 9002e5155bfe0b7bedfac321f1fb07ddd77acee5 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sat, 11 Dec 2021 17:36:49 +0000 Subject: [PATCH] Add import functionality --- lib/mix/tasks/pleroma/search.ex | 46 +++++++++++++++++++ .../elasticsearch/document_mappings/note.ex | 13 ++++++ lib/pleroma/elasticsearch/store.ex | 36 +++++++++++++++ mix.exs | 3 +- mix.lock | 4 ++ 5 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 lib/mix/tasks/pleroma/search.ex create mode 100644 lib/pleroma/elasticsearch/document_mappings/note.ex create mode 100644 lib/pleroma/elasticsearch/store.ex diff --git a/lib/mix/tasks/pleroma/search.ex b/lib/mix/tasks/pleroma/search.ex new file mode 100644 index 000000000..4acd0e34a --- /dev/null +++ b/lib/mix/tasks/pleroma/search.ex @@ -0,0 +1,46 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Search do + use Mix.Task + import Mix.Pleroma + import Ecto.Query + alias Pleroma.Elasticsearch + alias Pleroma.Activity + alias Pleroma.Repo + alias Pleroma.Pagination + + @shortdoc "Manages elasticsearch" + + def run(["import" | rest]) do + start_pleroma() + + query = from(a in Activity, where: not ilike(a.actor, "%/relay")) + |> Activity.with_preloaded_object + |> Activity.with_preloaded_user_actor + |> get_all + end + + defp get_all(query, max_id \\ nil) do + params = %{limit: 20} + params = if max_id == nil do + params + else + Map.put(params, :max_id, max_id) + end + + res = query + |> Pagination.fetch_paginated(params) + + if res == [] do + :ok + else + res + |> Pleroma.Elasticsearch.bulk_post(:activities) + + get_all(query, List.last(res).id) + end + end + +end diff --git a/lib/pleroma/elasticsearch/document_mappings/note.ex b/lib/pleroma/elasticsearch/document_mappings/note.ex new file mode 100644 index 000000000..f4e3307fe --- /dev/null +++ b/lib/pleroma/elasticsearch/document_mappings/note.ex @@ -0,0 +1,13 @@ +defmodule Pleroma.Elasticsearch.DocumentMappings.Activity do + alias Pleroma.Object + + def id(obj), do: obj.id + def encode(%{object: %{data: %{ "type" => "Note" }}} = activity) do + %{ + user: activity.user_actor.nickname, + content: activity.object.data["content"], + instance: URI.parse(activity.user_actor.ap_id).host, + hashtags: Object.hashtags(activity.object) + } + end +end diff --git a/lib/pleroma/elasticsearch/store.ex b/lib/pleroma/elasticsearch/store.ex new file mode 100644 index 000000000..2ff4bf889 --- /dev/null +++ b/lib/pleroma/elasticsearch/store.ex @@ -0,0 +1,36 @@ +defmodule Pleroma.Elasticsearch do + alias Pleroma.Activity + alias Pleroma.Elasticsearch.DocumentMappings + + defp url do + Pleroma.Config.get([:elasticsearch, :url]) + end + + def put(%Activity{} = activity) do + Elastix.Document.index( + url(), + "activities", + "activity", + DocumentMappings.Activity.id(activity), + DocumentMappings.Activity.encode(activity) + ) + end + + def bulk_post(data, :activities) do + d = data + |> Enum.map(fn d -> + [ + %{index: %{_id: DocumentMappings.Activity.id(d)}}, + DocumentMappings.Activity.encode(d) + ] + end) + |> List.flatten() + + IO.inspect Elastix.Bulk.post( + url(), + d, + index: "activities", + type: "activity" + ) + end +end diff --git a/mix.exs b/mix.exs index e69c737dd..f49353f7f 100644 --- a/mix.exs +++ b/mix.exs @@ -91,7 +91,7 @@ defp elixirc_paths(:test), do: ["lib", "test/support"] defp elixirc_paths(_), do: ["lib"] defp warnings_as_errors(:prod), do: false - defp warnings_as_errors(_), do: true + defp warnings_as_errors(_), do: false # Specifies OAuth dependencies. defp oauth_deps do @@ -197,6 +197,7 @@ defp deps do ref: "289cda1b6d0d70ccb2ba508a2b0bd24638db2880"}, {:eblurhash, "~> 1.1.0"}, {:open_api_spex, "~> 3.10"}, + {:elastix, ">= 0.0.0"}, # indirect dependency version override {:plug, "~> 1.10.4", override: true}, diff --git a/mix.lock b/mix.lock index 18d5e3bea..bec9d025e 100644 --- a/mix.lock +++ b/mix.lock @@ -34,6 +34,8 @@ "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"}, "ecto_sql": {:hex, :ecto_sql, "3.6.2", "9526b5f691701a5181427634c30655ac33d11e17e4069eff3ae1176c764e0ba3", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.6.2", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.4.0 or ~> 0.5.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5ec9d7e6f742ea39b63aceaea9ac1d1773d574ea40df5a53ef8afbd9242fdb6b"}, "eimp": {:hex, :eimp, "1.0.14", "fc297f0c7e2700457a95a60c7010a5f1dcb768a083b6d53f49cd94ab95a28f22", [:rebar3], [{:p1_utils, "1.0.18", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm", "501133f3112079b92d9e22da8b88bf4f0e13d4d67ae9c15c42c30bd25ceb83b6"}, + "elasticsearch": {:hex, :elasticsearch, "1.0.1", "8339538d90af6b280f10ecd02b1eae372f09373e629b336a13461babf7366495", [:mix], [{:httpoison, ">= 0.0.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sigaws, "~> 0.7", [hex: :sigaws, repo: "hexpm", optional: true]}, {:vex, "~> 0.6", [hex: :vex, repo: "hexpm", optional: false]}], "hexpm", "83e7d8b8bee3e7e19a06ab4d357d24845ac1da894e79678227fd52c0b7f71867"}, + "elastix": {:hex, :elastix, "0.10.0", "7567da885677ba9deffc20063db5f3ca8cd10f23cff1ab3ed9c52b7063b7e340", [:mix], [{:httpoison, "~> 1.4", [hex: :httpoison, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0", [hex: :poison, repo: "hexpm", optional: true]}, {:retry, "~> 0.8", [hex: :retry, repo: "hexpm", optional: false]}], "hexpm", "5fb342ce068b20f7845f5dd198c2dc80d967deafaa940a6e51b846db82696d1d"}, "elixir_make": {:hex, :elixir_make, "0.6.2", "7dffacd77dec4c37b39af867cedaabb0b59f6a871f89722c25b28fcd4bd70530", [:mix], [], "hexpm", "03e49eadda22526a7e5279d53321d1cced6552f344ba4e03e619063de75348d9"}, "esshd": {:hex, :esshd, "0.1.1", "d4dd4c46698093a40a56afecce8a46e246eb35463c457c246dacba2e056f31b5", [:mix], [], "hexpm", "d73e341e3009d390aa36387dc8862860bf9f874c94d9fd92ade2926376f49981"}, "eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"}, @@ -112,6 +114,7 @@ "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, "recon": {:hex, :recon, "2.5.1", "430ffa60685ac1efdfb1fe4c97b8767c92d0d92e6e7c3e8621559ba77598678a", [:mix, :rebar3], [], "hexpm", "5721c6b6d50122d8f68cccac712caa1231f97894bab779eff5ff0f886cb44648"}, "remote_ip": {:git, "https://git.pleroma.social/pleroma/remote_ip.git", "b647d0deecaa3acb140854fe4bda5b7e1dc6d1c8", [ref: "b647d0deecaa3acb140854fe4bda5b7e1dc6d1c8"]}, + "retry": {:hex, :retry, "0.15.0", "ba6aaeba92905a396c18c299a07e638947b2ba781e914f803202bc1b9ae867c3", [:mix], [], "hexpm", "93d3310bce78c0a30cc94610684340a14adfc9136856a3f662e4d9ce6013c784"}, "sleeplocks": {:hex, :sleeplocks, "1.1.1", "3d462a0639a6ef36cc75d6038b7393ae537ab394641beb59830a1b8271faeed3", [:rebar3], [], "hexpm", "84ee37aeff4d0d92b290fff986d6a95ac5eedf9b383fadfd1d88e9b84a1c02e1"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm", "2e1ec458f892ffa81f9f8386e3f35a1af6db7a7a37748a64478f13163a1f3573"}, @@ -125,6 +128,7 @@ "ueberauth": {:hex, :ueberauth, "0.6.3", "d42ace28b870e8072cf30e32e385579c57b9cc96ec74fa1f30f30da9c14f3cc0", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "afc293d8a1140d6591b53e3eaf415ca92842cb1d32fad3c450c6f045f7f91b60"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, "unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm", "6c7729a2d214806450d29766abc2afaa7a2cbecf415be64f36a6691afebb50e5"}, + "vex": {:hex, :vex, "0.9.0", "613ea5eb3055662e7178b83e25b2df0975f68c3d8bb67c1645f0573e1a78d606", [:mix], [], "hexpm", "c69fff44d5c8aa3f1faee71bba1dcab05dd36364c5a629df8bb11751240c857f"}, "web_push_encryption": {:git, "https://github.com/lanodan/elixir-web-push-encryption.git", "026a043037a89db4da8f07560bc8f9c68bcf0cc0", [branch: "bugfix/otp-24"]}, "websocket_client": {:git, "https://github.com/jeremyong/websocket_client.git", "9a6f65d05ebf2725d62fb19262b21f1805a59fbf", []}, }