From ee620ecbf11398277551ef603355a56a53690461 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 12 Aug 2019 13:13:01 +0300 Subject: [PATCH] Add caching for emoji pack sharing --- config/config.exs | 3 +- docs/config.md | 2 + lib/pleroma/application.ex | 6 ++- .../web/emoji_api/emoji_api_controller.ex | 42 ++++++++++++++++++- 4 files changed, 50 insertions(+), 3 deletions(-) diff --git a/config/config.exs b/config/config.exs index c7e0cf09f..4c758d4a0 100644 --- a/config/config.exs +++ b/config/config.exs @@ -122,7 +122,8 @@ # Put groups that have higher priority than defaults here. Example in `docs/config/custom_emoji.md` Custom: ["/emoji/*.png", "/emoji/**/*.png"] ], - default_manifest: "https://git.pleroma.social/pleroma/emoji-index/raw/master/index.json" + default_manifest: "https://git.pleroma.social/pleroma/emoji-index/raw/master/index.json", + shared_pack_cache_seconds_per_file: 60 config :pleroma, :uri_schemes, valid_schemes: [ diff --git a/docs/config.md b/docs/config.md index 3f37fa561..1179def56 100644 --- a/docs/config.md +++ b/docs/config.md @@ -707,6 +707,8 @@ Configure OAuth 2 provider capabilities: * `pack_extensions`: A list of file extensions for emojis, when no emoji.txt for a pack is present. Example `[".png", ".gif"]` * `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]` * `default_manifest`: Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download. Currently only one manifest can be added (no arrays). +* `shared_pack_cache_seconds_per_file`: When an emoji pack is shared, the archive is created and cached in + memory for this amount of seconds multiplied by the number of files. ## Database options diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index dabce771d..a339e2c48 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -102,10 +102,14 @@ defp cachex_children do build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000), build_cachex("scrubber", limit: 2500), build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500), - build_cachex("web_resp", limit: 2500) + build_cachex("web_resp", limit: 2500), + build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10) ] end + defp emoji_packs_expiration, + do: expiration(default: :timer.seconds(5 * 60), interval: :timer.seconds(60)) + defp idempotency_expiration, do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60)) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 915059783..8219eaaa1 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -1,6 +1,8 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do use Pleroma.Web, :controller + require Logger + def reload(conn, _params) do Pleroma.Emoji.reload() @@ -12,6 +14,8 @@ def reload(conn, _params) do "emoji" ) + @cache_seconds_per_file Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file]) + def list_packs(conn, _params) do pack_infos = case File.ls(@emoji_dir_path) do @@ -66,13 +70,49 @@ defp can_download?(pack, pack_path) do end) end - defp make_archive(name, pack, pack_dir) do + defp create_archive_and_cache(name, pack, pack_dir, md5) do files = ['pack.yml'] ++ (pack["files"] |> Enum.map(fn {_, path} -> to_charlist(path) end)) {:ok, {_, zip_result}} = :zip.zip('#{name}.zip', files, [:memory, cwd: to_charlist(pack_dir)]) + cache_ms = :timer.seconds(@cache_seconds_per_file * Enum.count(files)) + + Cachex.put!( + :emoji_packs_cache, + name, + # if pack.yml MD5 changes, the cache is not valid anymore + %{pack_yml_md5: md5, pack_data: zip_result}, + # Add a minute to cache time for every file in the pack + ttl: cache_ms + ) + + Logger.debug("Create an archive for the '#{name}' shared emoji pack, \ +keeping it in cache for #{div(cache_ms, 1000)}s") + + zip_result + end + + defp make_archive(name, pack, pack_dir) do + # Having a different pack.yml md5 invalidates cache + pack_yml_md5 = :crypto.hash(:md5, File.read!(Path.join(pack_dir, "pack.yml"))) + + maybe_cached_pack = Cachex.get!(:emoji_packs_cache, name) + + zip_result = + if is_nil(maybe_cached_pack) do + create_archive_and_cache(name, pack, pack_dir, pack_yml_md5) + else + if maybe_cached_pack[:pack_yml_md5] == pack_yml_md5 do + Logger.debug("Using cache for the '#{name}' shared emoji pack") + + maybe_cached_pack[:pack_data] + else + create_archive_and_cache(name, pack, pack_dir, pack_yml_md5) + end + end + zip_result end