sam/pleroma
sam/pleroma
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'issue/1936' into 'develop'

Browse source 
[#1936] Ability to search for banned MediaProxy URLs 

See merge request pleroma/pleroma!2873
This commit is contained in:
lain 2020-08-17 13:05:20 +00:00
commit 34b099fffa
6 changed files with 121 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
docs/API/admin_api.md
View file

@ -1266,11 +1266,14 @@ Loads json generated from `config/descriptions.exs`.
- Params:
- *optional* `page`: **integer** page number
- *optional* `page_size`: **integer** number of log entries per page (default is `50`)
- *optional* `query`: **string** search term
- Response:
``` json
{
"page_size": integer,
"count": integer,
"urls": [
"http://example.com/media/a688346.jpg",
"http://example.com/media/fb1f4d.jpg"
@ -1290,12 +1293,7 @@ Loads json generated from `config/descriptions.exs`.
- Response:
``` json
{
"urls": [
"http://example.com/media/a688346.jpg",
"http://example.com/media/fb1f4d.jpg"
]
}
{ }
```
@ -1311,11 +1309,6 @@ Loads json generated from `config/descriptions.exs`.
- Response:
``` json
{
"urls": [
"http://example.com/media/a688346.jpg",
"http://example.com/media/fb1f4d.jpg"
]
}
{ }
```

43
lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex
View file

@ -26,29 +26,40 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do
defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation
def index(%{assigns: %{user: _}} = conn, params) do
cursor =
:banned_urls_cache
|> :ets.table([{:traverse, {:select, Cachex.Query.create(true, :key)}}])
|> :qlc.cursor()
entries = fetch_entries(params)
urls = paginate_entries(entries, params.page, params.page_size)
urls =
case params.page do
1 ->
:qlc.next_answers(cursor, params.page_size)
render(conn, "index.json",
urls: urls,
page_size: params.page_size,
count: length(entries)
)
end
_ ->
:qlc.next_answers(cursor, (params.page - 1) * params.page_size)
:qlc.next_answers(cursor, params.page_size)
end
defp fetch_entries(params) do
MediaProxy.cache_table()
|> Cachex.stream!(Cachex.Query.create(true, :key))
|> filter_entries(params[:query])
end
:qlc.delete_cursor(cursor)
defp filter_entries(stream, query) when is_binary(query) do
regex = ~r/#{query}/i
render(conn, "index.json", urls: urls)
stream
|> Enum.filter(fn url -> String.match?(url, regex) end)
|> Enum.to_list()
end
defp filter_entries(stream, _), do: Enum.to_list(stream)
defp paginate_entries(entries, page, page_size) do
offset = page_size * (page - 1)
Enum.slice(entries, offset, page_size)
end
def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do
MediaProxy.remove_from_banned_urls(urls)
render(conn, "index.json", urls: urls)
json(conn, %{})
end
def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _) do
@ -58,6 +69,6 @@ def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _
MediaProxy.put_in_banned_urls(urls)
end
render(conn, "index.json", urls: urls)
json(conn, %{})
end
end

8
lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex
View file

@ -5,7 +5,11 @@
defmodule Pleroma.Web.AdminAPI.MediaProxyCacheView do
use Pleroma.Web, :view
def render("index.json", %{urls: urls}) do
%{urls: urls}
def render("index.json", %{urls: urls, page_size: page_size, count: count}) do
%{
urls: urls,
count: count,
page_size: page_size
}
end
end

47
lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex
View file

@ -21,6 +21,12 @@ def index_operation do
operationId: "AdminAPI.MediaProxyCacheController.index",
security: [%{"oAuth" => ["read:media_proxy_caches"]}],
parameters: [
Operation.parameter(
:query,
:query,
%Schema{type: :string, default: nil},
"Page"
),
Operation.parameter(
:page,
:query,
@ -36,7 +42,26 @@ def index_operation do
| admin_api_params()
],
responses: %{
200 => success_response()
200 =>
Operation.response(
"Array of banned MediaProxy URLs in Cachex",
"application/json",
%Schema{
type: :object,
properties: %{
count: %Schema{type: :integer},
page_size: %Schema{type: :integer},
urls: %Schema{
type: :array,
items: %Schema{
type: :string,
format: :uri,
description: "MediaProxy URLs"
}
}
}
}
)
}
}
end
@ -61,7 +86,7 @@ def delete_operation do
required: true
),
responses: %{
200 => success_response(),
200 => empty_object_response(),
400 => Operation.response("Error", "application/json", ApiError)
}
}
@ -88,25 +113,9 @@ def purge_operation do
required: true
),
responses: %{
200 => success_response(),
200 => empty_object_response(),
400 => Operation.response("Error", "application/json", ApiError)
}
}
end
defp success_response do
Operation.response("Array of banned MediaProxy URLs in Cachex", "application/json", %Schema{
type: :object,
properties: %{
urls: %Schema{
type: :array,
items: %Schema{
type: :string,
format: :uri,
description: "MediaProxy URLs"
}
}
}
})
end
end

13
lib/pleroma/web/media_proxy/media_proxy.ex
View file

@ -9,28 +9,31 @@ defmodule Pleroma.Web.MediaProxy do
alias Pleroma.Web.MediaProxy.Invalidation
@base64_opts [padding: false]
@cache_table :banned_urls_cache
def cache_table, do: @cache_table
@spec in_banned_urls(String.t()) :: boolean()
def in_banned_urls(url), do: elem(Cachex.exists?(:banned_urls_cache, url(url)), 1)
def in_banned_urls(url), do: elem(Cachex.exists?(@cache_table, url(url)), 1)
def remove_from_banned_urls(urls) when is_list(urls) do
Cachex.execute!(:banned_urls_cache, fn cache ->
Cachex.execute!(@cache_table, fn cache ->
Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1))
end)
end
def remove_from_banned_urls(url) when is_binary(url) do
Cachex.del(:banned_urls_cache, url(url))
Cachex.del(@cache_table, url(url))
end
def put_in_banned_urls(urls) when is_list(urls) do
Cachex.execute!(:banned_urls_cache, fn cache ->
Cachex.execute!(@cache_table, fn cache ->
Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true))
end)
end
def put_in_banned_urls(url) when is_binary(url) do
Cachex.put(:banned_urls_cache, url(url), true)
Cachex.put(@cache_table, url(url), true)
end
def url(url) when is_nil(url) or url == "", do: nil

72
test/web/admin_api/controllers/media_proxy_cache_controller_test.exs
View file

@ -48,6 +48,9 @@ test "shows banned MediaProxy URLs", %{conn: conn} do
|> get("/api/pleroma/admin/media_proxy_caches?page_size=2")
|> json_response_and_validate_schema(200)
assert response["page_size"] == 2
assert response["count"] == 5
assert response["urls"] == [
"http://localhost:4001/media/fb1f4d.jpg",
"http://localhost:4001/media/a688346.jpg"
@ -63,6 +66,9 @@ test "shows banned MediaProxy URLs", %{conn: conn} do
"http://localhost:4001/media/tb13f47.jpg"
]
assert response["page_size"] == 2
assert response["count"] == 5
response =
conn
|> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=3")
@ -70,6 +76,30 @@ test "shows banned MediaProxy URLs", %{conn: conn} do
assert response["urls"] == ["http://localhost:4001/media/wb1f46.jpg"]
end
test "search banned MediaProxy URLs", %{conn: conn} do
MediaProxy.put_in_banned_urls([
"http://localhost:4001/media/a688346.jpg",
"http://localhost:4001/media/ff44b1f4d.jpg"
])
MediaProxy.put_in_banned_urls("http://localhost:4001/media/gb1f44.jpg")
MediaProxy.put_in_banned_urls("http://localhost:4001/media/tb13f47.jpg")
MediaProxy.put_in_banned_urls("http://localhost:4001/media/wb1f46.jpg")
response =
conn
|> get("/api/pleroma/admin/media_proxy_caches?page_size=2&query=F44")
|> json_response_and_validate_schema(200)
assert response["urls"] == [
"http://localhost:4001/media/gb1f44.jpg",
"http://localhost:4001/media/ff44b1f4d.jpg"
]
assert response["page_size"] == 2
assert response["count"] == 2
end
end
describe "POST /api/pleroma/admin/media_proxy_caches/delete" do
@ -79,15 +109,13 @@ test "deleted MediaProxy URLs from banned", %{conn: conn} do
"http://localhost:4001/media/fb1f4d.jpg"
])
response =
conn
|> put_req_header("content-type", "application/json")
|> post("/api/pleroma/admin/media_proxy_caches/delete", %{
urls: ["http://localhost:4001/media/a688346.jpg"]
})
|> json_response_and_validate_schema(200)
conn
|> put_req_header("content-type", "application/json")
|> post("/api/pleroma/admin/media_proxy_caches/delete", %{
urls: ["http://localhost:4001/media/a688346.jpg"]
})
|> json_response_and_validate_schema(200)
assert response["urls"] == ["http://localhost:4001/media/a688346.jpg"]
refute MediaProxy.in_banned_urls("http://localhost:4001/media/a688346.jpg")
assert MediaProxy.in_banned_urls("http://localhost:4001/media/fb1f4d.jpg")
end
@ -106,13 +134,10 @@ test "perform invalidates cache of MediaProxy", %{conn: conn} do
purge: fn _, _ -> {"ok", 0} end
]}
] do
response =
conn
|> put_req_header("content-type", "application/json")
|> post("/api/pleroma/admin/media_proxy_caches/purge", %{urls: urls, ban: false})
|> json_response_and_validate_schema(200)
assert response["urls"] == urls
conn
|> put_req_header("content-type", "application/json")
|> post("/api/pleroma/admin/media_proxy_caches/purge", %{urls: urls, ban: false})
|> json_response_and_validate_schema(200)
refute MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg")
refute MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")
@ -126,16 +151,13 @@ test "perform invalidates cache of MediaProxy and adds url to banned", %{conn: c
]
with_mocks [{MediaProxy.Invalidation.Script, [], [purge: fn _, _ -> {"ok", 0} end]}] do
response =
conn
|> put_req_header("content-type", "application/json")
|> post("/api/pleroma/admin/media_proxy_caches/purge", %{
urls: urls,
ban: true
})
|> json_response_and_validate_schema(200)
assert response["urls"] == urls
conn
|> put_req_header("content-type", "application/json")
|> post(
"/api/pleroma/admin/media_proxy_caches/purge",
%{urls: urls, ban: true}
)
|> json_response_and_validate_schema(200)
assert MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg")
assert MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")