sam/pleroma
sam/pleroma
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'feature/1854-emoji-pagination' into 'develop'

Browse source 
Emoji and packs pagination

Closes #1854

See merge request pleroma/pleroma!2658
This commit is contained in:
lain 2020-06-22 08:41:09 +00:00
commit 93ae1c8bd3
10 changed files with 213 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
CHANGELOG.md
View file

@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- **Breaking:** removed `with_move` parameter from notifications timeline.
### Added
- Chats: Added support for federated chats. For details, see the docs.
- ActivityPub: Added support for existing AP ids for instances migrated from Mastodon.
- Instance: Add `background_image` to configuration and `/api/v1/instance`
@ -34,6 +35,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Notifications: Added `follow_request` notification type.
- Added `:reject_deletes` group to SimplePolicy
- MRF (`EmojiStealPolicy`): New MRF Policy which allows to automatically download emojis from remote instances
- Support pagination in emoji packs API (for packs and for files in pack)
<details>
<summary>API Changes</summary>
- Mastodon API: Extended `/api/v1/instance`.

34
docs/API/pleroma_api.md
View file

@ -450,18 +450,44 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
* Response: JSON, list with updated files for updated pack (hashmap -> shortcode => filename) with status 200, either error status with error message.
## `GET /api/pleroma/emoji/packs`
### Lists local custom emoji packs
* Method `GET`
* Authentication: not required
* Params: None
* Response: JSON, "ok" and 200 status and the JSON hashmap of pack name to pack contents
* Params:
* `page`: page number for packs (default 1)
* `page_size`: page size for packs (default 50)
* Response: `packs` key with JSON hashmap of pack name to pack contents and `count` key for count of packs.
```json
{
"packs": {
"pack_name": {...}, // pack contents
...
},
"count": 0 // packs count
}
```
## `GET /api/pleroma/emoji/packs/:name`
### Get pack.json for the pack
* Method `GET`
* Authentication: not required
* Params: None
* Response: JSON, pack json with `files` and `pack` keys with 200 status or 404 if the pack does not exist
* Params:
* `page`: page number for files (default 1)
* `page_size`: page size for files (default 30)
* Response: JSON, pack json with `files`, `files_count` and `pack` keys with 200 status or 404 if the pack does not exist.
```json
{
"files": {...},
"files_count": 0, // emoji count in pack
"pack": {...}
}
```
## `GET /api/pleroma/emoji/packs/:name/archive`
### Requests a local pack archive from the instance

54
lib/pleroma/emoji/pack.ex
View file

@ -1,6 +1,7 @@
defmodule Pleroma.Emoji.Pack do
@derive {Jason.Encoder, only: [:files, :pack]}
@derive {Jason.Encoder, only: [:files, :pack, :files_count]}
defstruct files: %{},
files_count: 0,
pack_file: nil,
path: nil,
pack: %{},
@ -8,6 +9,7 @@ defmodule Pleroma.Emoji.Pack do
@type t() :: %__MODULE__{
files: %{String.t() => Path.t()},
files_count: non_neg_integer(),
pack_file: Path.t(),
path: Path.t(),
pack: map(),
@ -16,7 +18,7 @@ defmodule Pleroma.Emoji.Pack do
alias Pleroma.Emoji
@spec create(String.t()) :: :ok | {:error, File.posix()} | {:error, :empty_values}
@spec create(String.t()) :: {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
def create(name) do
with :ok <- validate_not_empty([name]),
dir <- Path.join(emoji_path(), name),
@ -26,10 +28,27 @@ def create(name) do
end
end
@spec show(String.t()) :: {:ok, t()} | {:error, atom()}
def show(name) do
defp paginate(entities, 1, page_size), do: Enum.take(entities, page_size)
defp paginate(entities, page, page_size) do
entities
|> Enum.chunk_every(page_size)
|> Enum.at(page - 1)
end
@spec show(keyword()) :: {:ok, t()} | {:error, atom()}
def show(opts) do
name = opts[:name]
with :ok <- validate_not_empty([name]),
{:ok, pack} <- load_pack(name) do
shortcodes =
pack.files
|> Map.keys()
|> paginate(opts[:page], opts[:page_size])
pack = Map.put(pack, :files, Map.take(pack.files, shortcodes))
{:ok, validate_pack(pack)}
end
end
@ -120,10 +139,10 @@ def list_remote(url) do
end
end
@spec list_local() :: {:ok, map()}
def list_local do
@spec list_local(keyword()) :: {:ok, map(), non_neg_integer()}
def list_local(opts) do
with {:ok, results} <- list_packs_dir() do
packs =
all_packs =
results
|> Enum.map(fn name ->
case load_pack(name) do
@ -132,9 +151,13 @@ def list_local do
end
end)
|> Enum.reject(&is_nil/1)
packs =
all_packs
|> paginate(opts[:page], opts[:page_size])
|> Map.new(fn pack -> {pack.name, validate_pack(pack)} end)
{:ok, packs}
{:ok, packs, length(all_packs)}
end
end
@ -146,7 +169,7 @@ def get_archive(name) do
end
end
@spec download(String.t(), String.t(), String.t()) :: :ok | {:error, atom()}
@spec download(String.t(), String.t(), String.t()) :: {:ok, t()} | {:error, atom()}
def download(name, url, as) do
uri = url |> String.trim() |> URI.parse()
@ -197,7 +220,12 @@ def load_pack(name) do
|> Map.put(:path, Path.dirname(pack_file))
|> Map.put(:name, name)
{:ok, pack}
files_count =
pack.files
|> Map.keys()
|> length()
{:ok, Map.put(pack, :files_count, files_count)}
else
{:error, :not_found}
end
@ -296,7 +324,9 @@ defp downloadable?(pack) do
# Otherwise, they'd have to download it from external-src
pack.pack["share-files"] &&
Enum.all?(pack.files, fn {_, file} ->
File.exists?(Path.join(pack.path, file))
pack.path
|> Path.join(file)
|> File.exists?()
end)
end
@ -440,7 +470,7 @@ defp list_packs_dir do
# with the API so it should be sufficient
with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(emoji_path)},
{:ls, {:ok, results}} <- {:ls, File.ls(emoji_path)} do
{:ok, results}
{:ok, Enum.sort(results)}
else
{:create_dir, {:error, e}} -> {:error, :create_dir, e}
{:ls, {:error, e}} -> {:error, :ls, e}

30
lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex
View file

@ -33,6 +33,20 @@ def index_operation do
tags: ["Emoji Packs"],
summary: "Lists local custom emoji packs",
operationId: "PleromaAPI.EmojiPackController.index",
parameters: [
Operation.parameter(
:page,
:query,
%Schema{type: :integer, default: 1},
"Page"
),
Operation.parameter(
:page_size,
:query,
%Schema{type: :integer, default: 50},
"Number of emoji packs to return"
)
],
responses: %{
200 => emoji_packs_response()
}
@ -44,7 +58,21 @@ def show_operation do
tags: ["Emoji Packs"],
summary: "Show emoji pack",
operationId: "PleromaAPI.EmojiPackController.show",
parameters: [name_param()],
parameters: [
name_param(),
Operation.parameter(
:page,
:query,
%Schema{type: :integer, default: 1},
"Page"
),
Operation.parameter(
:page_size,
:query,
%Schema{type: :integer, default: 30},
"Number of emoji to return"
)
],
responses: %{
200 => Operation.response("Emoji Pack", "application/json", emoji_pack()),
400 => Operation.response("Bad Request", "application/json", ApiError),

10
lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex
View file

@ -37,14 +37,14 @@ def remote(conn, %{url: url}) do
end
end
def index(conn, _params) do
def index(conn, params) do
emoji_path =
[:instance, :static_dir]
|> Pleroma.Config.get!()
|> Path.join("emoji")
with {:ok, packs} <- Pack.list_local() do
json(conn, packs)
with {:ok, packs, count} <- Pack.list_local(page: params.page, page_size: params.page_size) do
json(conn, %{packs: packs, count: count})
else
{:error, :create_dir, e} ->
conn
@ -60,10 +60,10 @@ def index(conn, _params) do
end
end
def show(conn, %{name: name}) do
def show(conn, %{name: name, page: page, page_size: page_size}) do
name = String.trim(name)
with {:ok, pack} <- Pack.show(name) do
with {:ok, pack} <- Pack.show(name: name, page: page, page_size: page_size) do
json(conn, pack)
else
{:error, :not_found} ->

BIN
test/instance_static/emoji/test_pack/blank2.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 95 B

3
test/instance_static/emoji/test_pack/pack.json
View file

@ -1,6 +1,7 @@
{
"files": {
"blank": "blank.png"
"blank": "blank.png",
"blank2": "blank2.png"
},
"pack": {
"description": "Test description",

BIN
test/instance_static/emoji/test_pack_nonshared/nonshared.zip
View file

Binary file not shown.

2
test/instance_static/emoji/test_pack_nonshared/pack.json
View file

@ -4,7 +4,7 @@
"homepage": "https://pleroma.social",
"description": "Test description",
"fallback-src": "https://nonshared-pack",
"fallback-src-sha256": "74409E2674DAA06C072729C6C8426C4CB3B7E0B85ED77792DB7A436E11D76DAF",
"fallback-src-sha256": "1967BB4E42BCC34BCC12D57BE7811D3B7BE52F965BCE45C87BD377B9499CE11D",
"share-files": false
},
"files": {

128
test/web/pleroma_api/controllers/emoji_pack_controller_test.exs
View file

@ -30,15 +30,55 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackControllerTest do
test "GET /api/pleroma/emoji/packs", %{conn: conn} do
resp = conn |> get("/api/pleroma/emoji/packs") |> json_response_and_validate_schema(200)
shared = resp["test_pack"]
assert shared["files"] == %{"blank" => "blank.png"}
assert resp["count"] == 3
assert resp["packs"]
|> Map.keys()
|> length() == 3
shared = resp["packs"]["test_pack"]
assert shared["files"] == %{"blank" => "blank.png", "blank2" => "blank2.png"}
assert Map.has_key?(shared["pack"], "download-sha256")
assert shared["pack"]["can-download"]
assert shared["pack"]["share-files"]
non_shared = resp["test_pack_nonshared"]
non_shared = resp["packs"]["test_pack_nonshared"]
assert non_shared["pack"]["share-files"] == false
assert non_shared["pack"]["can-download"] == false
resp =
conn
|> get("/api/pleroma/emoji/packs?page_size=1")
|> json_response_and_validate_schema(200)
assert resp["count"] == 3
packs = Map.keys(resp["packs"])
assert length(packs) == 1
[pack1] = packs
resp =
conn
|> get("/api/pleroma/emoji/packs?page_size=1&page=2")
|> json_response_and_validate_schema(200)
assert resp["count"] == 3
packs = Map.keys(resp["packs"])
assert length(packs) == 1
[pack2] = packs
resp =
conn
|> get("/api/pleroma/emoji/packs?page_size=1&page=3")
|> json_response_and_validate_schema(200)
assert resp["count"] == 3
packs = Map.keys(resp["packs"])
assert length(packs) == 1
[pack3] = packs
assert [pack1, pack2, pack3] |> Enum.uniq() |> length() == 3
end
describe "GET /api/pleroma/emoji/packs/remote" do
@ -332,7 +372,7 @@ test "for a pack with a fallback source", ctx do
Map.put(
new_data,
"fallback-src-sha256",
"74409E2674DAA06C072729C6C8426C4CB3B7E0B85ED77792DB7A436E11D76DAF"
"1967BB4E42BCC34BCC12D57BE7811D3B7BE52F965BCE45C87BD377B9499CE11D"
)
assert ctx[:admin_conn]
@ -398,7 +438,7 @@ test "don't rewrite old emoji", %{admin_conn: admin_conn} do
assert admin_conn
|> put_req_header("content-type", "multipart/form-data")
|> post("/api/pleroma/emoji/packs/test_pack/files", %{
shortcode: "blank2",
shortcode: "blank3",
filename: "dir/blank.png",
file: %Plug.Upload{
filename: "blank.png",
@ -407,7 +447,8 @@ test "don't rewrite old emoji", %{admin_conn: admin_conn} do
})
|> json_response_and_validate_schema(200) == %{
"blank" => "blank.png",
"blank2" => "dir/blank.png"
"blank2" => "blank2.png",
"blank3" => "dir/blank.png"
}
assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png")
@ -431,7 +472,7 @@ test "rewrite old emoji with force option", %{admin_conn: admin_conn} do
assert admin_conn
|> put_req_header("content-type", "multipart/form-data")
|> post("/api/pleroma/emoji/packs/test_pack/files", %{
shortcode: "blank2",
shortcode: "blank3",
filename: "dir/blank.png",
file: %Plug.Upload{
filename: "blank.png",
@ -440,7 +481,8 @@ test "rewrite old emoji with force option", %{admin_conn: admin_conn} do
})
|> json_response_and_validate_schema(200) == %{
"blank" => "blank.png",
"blank2" => "dir/blank.png"
"blank2" => "blank2.png",
"blank3" => "dir/blank.png"
}
assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png")
@ -448,14 +490,15 @@ test "rewrite old emoji with force option", %{admin_conn: admin_conn} do
assert admin_conn
|> put_req_header("content-type", "multipart/form-data")
|> patch("/api/pleroma/emoji/packs/test_pack/files", %{
shortcode: "blank2",
new_shortcode: "blank3",
shortcode: "blank3",
new_shortcode: "blank4",
new_filename: "dir_2/blank_3.png",
force: true
})
|> json_response_and_validate_schema(200) == %{
"blank" => "blank.png",
"blank3" => "dir_2/blank_3.png"
"blank2" => "blank2.png",
"blank4" => "dir_2/blank_3.png"
}
assert File.exists?("#{@emoji_path}/test_pack/dir_2/blank_3.png")
@ -481,7 +524,7 @@ test "add file with not loaded pack", %{admin_conn: admin_conn} do
assert admin_conn
|> put_req_header("content-type", "multipart/form-data")
|> post("/api/pleroma/emoji/packs/not_loaded/files", %{
shortcode: "blank2",
shortcode: "blank3",
filename: "dir/blank.png",
file: %Plug.Upload{
filename: "blank.png",
@ -535,7 +578,8 @@ test "new with shortcode as file with update", %{admin_conn: admin_conn} do
})
|> json_response_and_validate_schema(200) == %{
"blank" => "blank.png",
"blank4" => "dir/blank.png"
"blank4" => "dir/blank.png",
"blank2" => "blank2.png"
}
assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png")
@ -549,7 +593,8 @@ test "new with shortcode as file with update", %{admin_conn: admin_conn} do
})
|> json_response_and_validate_schema(200) == %{
"blank3" => "dir_2/blank_3.png",
"blank" => "blank.png"
"blank" => "blank.png",
"blank2" => "blank2.png"
}
refute File.exists?("#{@emoji_path}/test_pack/dir/")
@ -557,7 +602,10 @@ test "new with shortcode as file with update", %{admin_conn: admin_conn} do
assert admin_conn
|> delete("/api/pleroma/emoji/packs/test_pack/files?shortcode=blank3")
|> json_response_and_validate_schema(200) == %{"blank" => "blank.png"}
|> json_response_and_validate_schema(200) == %{
"blank" => "blank.png",
"blank2" => "blank2.png"
}
refute File.exists?("#{@emoji_path}/test_pack/dir_2/")
@ -581,7 +629,8 @@ test "new with shortcode from url", %{admin_conn: admin_conn} do
})
|> json_response_and_validate_schema(200) == %{
"blank_url" => "blank_url.png",
"blank" => "blank.png"
"blank" => "blank.png",
"blank2" => "blank2.png"
}
assert File.exists?("#{@emoji_path}/test_pack/blank_url.png")
@ -602,15 +651,16 @@ test "new without shortcode", %{admin_conn: admin_conn} do
})
|> json_response_and_validate_schema(200) == %{
"shortcode" => "shortcode.png",
"blank" => "blank.png"
"blank" => "blank.png",
"blank2" => "blank2.png"
}
end
test "remove non existing shortcode in pack.json", %{admin_conn: admin_conn} do
assert admin_conn
|> delete("/api/pleroma/emoji/packs/test_pack/files?shortcode=blank2")
|> delete("/api/pleroma/emoji/packs/test_pack/files?shortcode=blank3")
|> json_response_and_validate_schema(:bad_request) == %{
"error" => "Emoji \"blank2\" does not exist"
"error" => "Emoji \"blank3\" does not exist"
}
end
@ -618,12 +668,12 @@ test "update non existing emoji", %{admin_conn: admin_conn} do
assert admin_conn
|> put_req_header("content-type", "multipart/form-data")
|> patch("/api/pleroma/emoji/packs/test_pack/files", %{
shortcode: "blank2",
new_shortcode: "blank3",
shortcode: "blank3",
new_shortcode: "blank4",
new_filename: "dir_2/blank_3.png"
})
|> json_response_and_validate_schema(:bad_request) == %{
"error" => "Emoji \"blank2\" does not exist"
"error" => "Emoji \"blank3\" does not exist"
}
end
@ -651,7 +701,8 @@ test "creating and deleting a pack", %{admin_conn: admin_conn} do
assert Jason.decode!(File.read!("#{@emoji_path}/test_created/pack.json")) == %{
"pack" => %{},
"files" => %{}
"files" => %{},
"files_count" => 0
}
assert admin_conn
@ -709,14 +760,14 @@ test "filesystem import", %{admin_conn: admin_conn, conn: conn} do
resp = conn |> get("/api/pleroma/emoji/packs") |> json_response_and_validate_schema(200)
refute Map.has_key?(resp, "test_pack_for_import")
refute Map.has_key?(resp["packs"], "test_pack_for_import")
assert admin_conn
|> get("/api/pleroma/emoji/packs/import")
|> json_response_and_validate_schema(200) == ["test_pack_for_import"]
resp = conn |> get("/api/pleroma/emoji/packs") |> json_response_and_validate_schema(200)
assert resp["test_pack_for_import"]["files"] == %{"blank" => "blank.png"}
assert resp["packs"]["test_pack_for_import"]["files"] == %{"blank" => "blank.png"}
File.rm!("#{@emoji_path}/test_pack_for_import/pack.json")
refute File.exists?("#{@emoji_path}/test_pack_for_import/pack.json")
@ -736,7 +787,7 @@ test "filesystem import", %{admin_conn: admin_conn, conn: conn} do
resp = conn |> get("/api/pleroma/emoji/packs") |> json_response_and_validate_schema(200)
assert resp["test_pack_for_import"]["files"] == %{
assert resp["packs"]["test_pack_for_import"]["files"] == %{
"blank" => "blank.png",
"blank2" => "blank.png",
"foo" => "blank.png"
@ -746,7 +797,8 @@ test "filesystem import", %{admin_conn: admin_conn, conn: conn} do
describe "GET /api/pleroma/emoji/packs/:name" do
test "shows pack.json", %{conn: conn} do
assert %{
"files" => %{"blank" => "blank.png"},
"files" => files,
"files_count" => 2,
"pack" => %{
"can-download" => true,
"description" => "Test description",
@ -759,6 +811,28 @@ test "shows pack.json", %{conn: conn} do
conn
|> get("/api/pleroma/emoji/packs/test_pack")
|> json_response_and_validate_schema(200)
assert files == %{"blank" => "blank.png", "blank2" => "blank2.png"}
assert %{
"files" => files,
"files_count" => 2
} =
conn
|> get("/api/pleroma/emoji/packs/test_pack?page_size=1")
|> json_response_and_validate_schema(200)
assert files |> Map.keys() |> length() == 1
assert %{
"files" => files,
"files_count" => 2
} =
conn
|> get("/api/pleroma/emoji/packs/test_pack?page_size=1&page=2")
|> json_response_and_validate_schema(200)
assert files |> Map.keys() |> length() == 1
end
test "non existing pack", %{conn: conn} do