Merge remote-tracking branch 'upstream/develop' into media-proxy

This commit is contained in:
href 2017-12-11 02:45:28 +01:00
commit 9093b2cf49
No known key found for this signature in database
GPG key ID: EE8296C1A152C325
19 changed files with 226 additions and 39 deletions

View file

@ -22,16 +22,14 @@ No release has been made yet, but several servers have been online for months al
### Dependencies
* Postgresql version 9.6 or newer
* Elixir version 1.4 or newer
* Elixir version 1.4 or newer (you will also need erlang-dev, erlang-parsetools, erlang-xmerl packages)
* Build-essential tools
### Configuration
* Run `mix deps.get` to install elixir dependencies.
* Run `mix generate_config`. This will ask you a few questions about your instance and generate a configuration file in `config/generated_config.exs`. Check that and copy it to either `config/dev.secret.exs` or `config/prod.secret.exs`.
* Configure your database settings in `{dev,prod}.secret.exs` and either create the database with psql or run `mix ecto.create`.
* Run `mix generate_config`. This will ask you a few questions about your instance and generate a configuration file in `config/generated_config.exs`. Check that and copy it to either `config/dev.secret.exs` or `config/prod.secret.exs`. It will also create a `config/setup_db.psql`, which you need to run as PostgreSQL superuser (i.e. `sudo su postgres -c "psql -f config/setup_db.psql"`). It will setup a pleroma db user, database and will setup needed extensions that need to be set up once as superuser.
* Run `mix ecto.migrate` to run the database migrations. You will have to do this again after certain updates.

View file

@ -22,6 +22,9 @@ server {
server_name example.tld;
location / {
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_pass http://localhost:4000;
}

View file

@ -9,9 +9,14 @@ def run(_) do
name = IO.gets("What is the name of your instance? (e.g. Pleroma/Soykaf): ") |> String.trim
email = IO.gets("What's your admin email address: ") |> String.trim
secret = :crypto.strong_rand_bytes(64) |> Base.encode64 |> binary_part(0, 64)
dbpass = :crypto.strong_rand_bytes(64) |> Base.encode64 |> binary_part(0, 64)
resultSql = EEx.eval_file("lib/mix/tasks/sample_psql.eex", [dbpass: dbpass])
result = EEx.eval_file("lib/mix/tasks/sample_config.eex", [domain: domain, email: email, name: name, secret: secret, dbpass: dbpass])
result = EEx.eval_file("lib/mix/tasks/sample_config.eex", [domain: domain, email: email, name: name, secret: secret])
IO.puts("\nWriting config to config/generated_config.exs.\n\nCheck it and configure your database, then copy it to either config/dev.secret.exs or config/prod.secret.exs")
File.write("config/generated_config.exs", result)
IO.puts("\nWriting setup_db.psql, please run it as postgre superuser, i.e.: sudo su postgres -c 'psql -f config/setup_db.psql'")
File.write("config/setup_db.psql", resultSql)
end
end

View file

@ -13,8 +13,8 @@ config :pleroma, :instance,
# Configure your database
config :pleroma, Pleroma.Repo,
adapter: Ecto.Adapters.Postgres,
username: "postgres",
password: "postgres",
username: "pleroma",
password: "<%= dbpass %>",
database: "pleroma_dev",
hostname: "localhost",
pool_size: 10

View file

@ -0,0 +1,8 @@
CREATE USER pleroma WITH ENCRYPTED PASSWORD '<%= dbpass %>' CREATEDB;
-- in case someone runs this second time accidentally
ALTER USER pleroma WITH ENCRYPTED PASSWORD '<%= dbpass %>' CREATEDB;
CREATE DATABASE pleroma_dev;
ALTER DATABASE pleroma_dev OWNER TO pleroma;
\c pleroma_dev;
--Extensions made by ecto.migrate that need superuser access
CREATE EXTENSION IF NOT EXISTS citext;

View file

@ -7,7 +7,7 @@ defmodule Pleroma.Activity do
field :data, :map
field :local, :boolean, default: true
field :actor, :string
has_many :notifications, Notification
has_many :notifications, Notification, on_delete: :delete_all
timestamps()
end

View file

@ -1,7 +1,7 @@
defmodule Pleroma.Formatter do
alias Pleroma.User
@link_regex ~r/https?:\/\/[\w\.\/?=\-#%&@~]+[\w\/]/u
@link_regex ~r/https?:\/\/[\w\.\/?=\-#%&@~\(\)]+[\w\/]/u
def linkify(text) do
Regex.replace(@link_regex, text, "<a href='\\0'>\\0</a>")
end
@ -24,6 +24,15 @@ def parse_mentions(text) do
|> Enum.filter(fn ({_match, user}) -> user end)
end
def html_escape(text) do
Regex.split(@link_regex, text, include_captures: true)
|> Enum.map_every(2, fn chunk ->
{:safe, part} = Phoenix.HTML.html_escape(chunk)
part
end)
|> Enum.join("")
end
@finmoji [
"a_trusted_friend",
"alandislands",

View file

@ -12,6 +12,7 @@ def call(%{assigns: %{user: %User{}}} = conn, _), do: conn
def call(conn, opts) do
with {:ok, username, password} <- decode_header(conn),
{:ok, user} <- opts[:fetcher].(username),
false <- !!user.info["deactivated"],
saved_user_id <- get_session(conn, :user_id),
{:ok, verified_user} <- verify(user, password, saved_user_id)
do

View file

@ -16,7 +16,8 @@ def call(conn, _) do
end
with token when not is_nil(token) <- token,
%Token{user_id: user_id} <- Repo.get_by(Token, token: token),
%User{} = user <- Repo.get(User, user_id) do
%User{} = user <- Repo.get(User, user_id),
false <- !!user.info["deactivated"] do
conn
|> assign(:user, user)
else

View file

@ -5,7 +5,7 @@ defmodule Pleroma.User do
alias Pleroma.{Repo, User, Object, Web, Activity, Notification}
alias Comeonin.Pbkdf2
alias Pleroma.Web.{OStatus, Websub}
alias Pleroma.Web.ActivityPub.Utils
alias Pleroma.Web.ActivityPub.{Utils, ActivityPub}
schema "users" do
field :bio, :string
@ -113,7 +113,7 @@ def password_update_changeset(struct, params) do
end
def reset_password(user, data) do
Repo.update(password_update_changeset(user, data))
update_and_set_cache(password_update_changeset(user, data))
end
def register_changeset(struct, params \\ %{}) do
@ -142,9 +142,9 @@ def register_changeset(struct, params \\ %{}) do
end
end
def follow(%User{} = follower, %User{} = followed) do
def follow(%User{} = follower, %User{info: info} = followed) do
ap_followers = followed.follower_address
if following?(follower, followed) do
if following?(follower, followed) or info["deactivated"] do
{:error,
"Could not follow user: #{followed.nickname} is already on your list."}
else
@ -157,7 +157,7 @@ def follow(%User{} = follower, %User{} = followed) do
follower = follower
|> follow_changeset(%{following: following})
|> Repo.update
|> update_and_set_cache
{:ok, _} = update_follower_count(followed)
@ -173,7 +173,7 @@ def unfollow(%User{} = follower, %User{} = followed) do
{ :ok, follower } = follower
|> follow_changeset(%{following: following})
|> Repo.update
|> update_and_set_cache
{:ok, followed} = update_follower_count(followed)
@ -191,6 +191,17 @@ def get_by_ap_id(ap_id) do
Repo.get_by(User, ap_id: ap_id)
end
def update_and_set_cache(changeset) do
with {:ok, user} <- Repo.update(changeset) do
Cachex.set(:user_cache, "ap_id:#{user.ap_id}", user)
Cachex.set(:user_cache, "nickname:#{user.nickname}", user)
Cachex.set(:user_cache, "user_info:#{user.id}", user_info(user))
{:ok, user}
else
e -> e
end
end
def get_cached_by_ap_id(ap_id) do
key = "ap_id:#{ap_id}"
Cachex.get!(:user_cache, key, fallback: fn(_) -> get_by_ap_id(ap_id) end)
@ -245,7 +256,7 @@ def increase_note_count(%User{} = user) do
cs = info_changeset(user, %{info: new_info})
Repo.update(cs)
update_and_set_cache(cs)
end
def update_note_count(%User{} = user) do
@ -259,7 +270,7 @@ def update_note_count(%User{} = user) do
cs = info_changeset(user, %{info: new_info})
Repo.update(cs)
update_and_set_cache(cs)
end
def update_follower_count(%User{} = user) do
@ -274,7 +285,7 @@ def update_follower_count(%User{} = user) do
cs = info_changeset(user, %{info: new_info})
Repo.update(cs)
update_and_set_cache(cs)
end
def get_notified_from_activity(%Activity{data: %{"to" => to}}) do
@ -312,7 +323,7 @@ def block(user, %{ap_id: ap_id}) do
new_info = Map.put(user.info, "blocks", new_blocks)
cs = User.info_changeset(user, %{info: new_info})
Repo.update(cs)
update_and_set_cache(cs)
end
def unblock(user, %{ap_id: ap_id}) do
@ -321,7 +332,7 @@ def unblock(user, %{ap_id: ap_id}) do
new_info = Map.put(user.info, "blocks", new_blocks)
cs = User.info_changeset(user, %{info: new_info})
Repo.update(cs)
update_and_set_cache(cs)
end
def blocks?(user, %{ap_id: ap_id}) do
@ -334,4 +345,35 @@ def local_user_query() do
where: u.local == true
end
def deactivate (%User{} = user) do
new_info = Map.put(user.info, "deactivated", true)
cs = User.info_changeset(user, %{info: new_info})
update_and_set_cache(cs)
end
def delete (%User{} = user) do
{:ok, user} = User.deactivate(user)
# Remove all relationships
{:ok, followers } = User.get_followers(user)
followers
|> Enum.each(fn (follower) -> User.unfollow(follower, user) end)
{:ok, friends} = User.get_friends(user)
friends
|> Enum.each(fn (followed) -> User.unfollow(user, followed) end)
query = from a in Activity,
where: a.actor == ^user.ap_id
Repo.all(query)
|> Enum.each(fn (activity) ->
case activity.data["type"] do
"Create" -> ActivityPub.delete(Object.get_by_ap_id(activity.data["object"]["id"]))
_ -> "Doing nothing" # TODO: Do something with likes, follows, repeats.
end
end)
:ok
end
end

View file

@ -29,7 +29,12 @@ def generate_id(type) do
Enqueues an activity for federation if it's local
"""
def maybe_federate(%Activity{local: true} = activity) do
Pleroma.Web.Federator.enqueue(:publish, activity)
priority = case activity.data["type"] do
"Delete" -> 10
"Create" -> 1
_ -> 5
end
Pleroma.Web.Federator.enqueue(:publish, activity, priority)
:ok
end
def maybe_federate(_), do: :ok

View file

@ -56,7 +56,7 @@ def post(user, %{"status" => status} = data) do
inReplyTo <- get_replied_to_activity(data["in_reply_to_status_id"]),
to <- to_for_user_and_mentions(user, mentions, inReplyTo),
tags <- Formatter.parse_tags(status, data),
content_html <- make_content_html(status, mentions, attachments, tags),
content_html <- make_content_html(status, mentions, attachments, tags, data["no_attachment_links"]),
context <- make_context(inReplyTo),
cw <- data["spoiler_text"],
object <- make_note_data(user.ap_id, to, context, content_html, attachments, inReplyTo, tags, cw),

View file

@ -38,15 +38,19 @@ def to_for_user_and_mentions(user, mentions, inReplyTo) do
end
end
def make_content_html(status, mentions, attachments, tags) do
def make_content_html(status, mentions, attachments, tags, no_attachment_links \\ false) do
status
|> format_input(mentions, tags)
|> add_attachments(attachments)
|> maybe_add_attachments(attachments, no_attachment_links)
end
def make_context(%Activity{data: %{"context" => context}}), do: context
def make_context(_), do: Utils.generate_context_id
def maybe_add_attachments(text, attachments, _no_links = true), do: text
def maybe_add_attachments(text, attachments, _no_links) do
add_attachments(text, attachments)
end
def add_attachments(text, attachments) do
attachment_text = Enum.map(attachments, fn
(%{"url" => [%{"href" => href} | _]}) ->
@ -58,8 +62,8 @@ def add_attachments(text, attachments) do
end
def format_input(text, mentions, _tags) do
Phoenix.HTML.html_escape(text)
|> elem(1)
text
|> Formatter.html_escape
|> Formatter.linkify
|> String.replace("\n", "<br>")
|> add_user_links(mentions)

View file

@ -15,8 +15,8 @@ def start_link do
enqueue(:refresh_subscriptions, nil)
end)
GenServer.start_link(__MODULE__, %{
in: {:sets.new(), :queue.new()},
out: {:sets.new(), :queue.new()}
in: {:sets.new(), []},
out: {:sets.new(), []}
}, name: __MODULE__)
end
@ -79,17 +79,17 @@ def handle(type, _) do
{:error, "Don't know what do do with this"}
end
def enqueue(type, payload) do
def enqueue(type, payload, priority \\ 1) do
if Mix.env == :test do
handle(type, payload)
else
GenServer.cast(__MODULE__, {:enqueue, type, payload})
GenServer.cast(__MODULE__, {:enqueue, type, payload, priority})
end
end
def maybe_start_job(running_jobs, queue) do
if (:sets.size(running_jobs) < @max_jobs) && !:queue.is_empty(queue) do
{{:value, {type, payload}}, queue} = :queue.out(queue)
if (:sets.size(running_jobs) < @max_jobs) && queue != [] do
{{type, payload}, queue} = queue_pop(queue)
{:ok, pid} = Task.start(fn -> handle(type, payload) end)
mref = Process.monitor(pid)
{:sets.add_element(mref, running_jobs), queue}
@ -98,16 +98,16 @@ def maybe_start_job(running_jobs, queue) do
end
end
def handle_cast({:enqueue, type, payload}, state) when type in [:incoming_doc] do
def handle_cast({:enqueue, type, payload, priority}, state) when type in [:incoming_doc] do
%{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state
i_queue = :queue.in({type, payload}, i_queue)
i_queue = enqueue_sorted(i_queue, {type, payload}, 1)
{i_running_jobs, i_queue} = maybe_start_job(i_running_jobs, i_queue)
{:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}}
end
def handle_cast({:enqueue, type, payload}, state) do
def handle_cast({:enqueue, type, payload, priority}, state) do
%{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state
o_queue = :queue.in({type, payload}, o_queue)
o_queue = enqueue_sorted(o_queue, {type, payload}, 1)
{o_running_jobs, o_queue} = maybe_start_job(o_running_jobs, o_queue)
{:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}}
end
@ -126,4 +126,13 @@ def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do
{:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}}
end
def enqueue_sorted(queue, element, priority) do
[%{item: element, priority: priority} | queue]
|> Enum.sort_by(fn (%{priority: priority}) -> priority end)
end
def queue_pop([%{item: element} | queue]) do
{element, queue}
end
end

View file

@ -212,6 +212,7 @@ def get_context(%{assigns: %{user: user}} = conn, %{"id" => id}) do
def post_status(%{assigns: %{user: user}} = conn, %{"status" => _} = params) do
params = params
|> Map.put("in_reply_to_status_id", params["in_reply_to_id"])
|> Map.put("no_attachment_links", true)
{:ok, activity} = CommonAPI.post(user, params)
render conn, StatusView, "status.json", %{activity: activity, for: user, as: :activity}

View file

@ -25,6 +25,16 @@ test "turning urls into links" do
expected = "<a href='http://www.cs.vu.nl/~ast/intel/'>http://www.cs.vu.nl/~ast/intel/</a>"
assert Formatter.linkify(text) == expected
text = "https://forum.zdoom.org/viewtopic.php?f=44&t=57087"
expected = "<a href='https://forum.zdoom.org/viewtopic.php?f=44&t=57087'>https://forum.zdoom.org/viewtopic.php?f=44&t=57087</a>"
assert Formatter.linkify(text) == expected
text = "https://en.wikipedia.org/wiki/Sophia_(Gnosticism)#Mythos_of_the_soul"
expected = "<a href='https://en.wikipedia.org/wiki/Sophia_(Gnosticism)#Mythos_of_the_soul'>https://en.wikipedia.org/wiki/Sophia_(Gnosticism)#Mythos_of_the_soul</a>"
assert Formatter.linkify(text) == expected
end
end

View file

@ -14,6 +14,13 @@ defp fetch_nil(_name) do
password_hash: Comeonin.Pbkdf2.hashpwsalt("guy")
}
@deactivated %User{
id: 1,
name: "dude",
password_hash: Comeonin.Pbkdf2.hashpwsalt("guy"),
info: %{"deactivated" => true}
}
@session_opts [
store: :cookie,
key: "_test",
@ -131,6 +138,26 @@ test "it assigns the user", %{conn: conn} do
end
end
describe "with a correct authorization header for an deactiviated user" do
test "it halts the appication", %{conn: conn} do
opts = %{
optional: false,
fetcher: fn _ -> @deactivated end
}
header = basic_auth_enc("dude", "guy")
conn = conn
|> Plug.Session.call(Plug.Session.init(@session_opts))
|> fetch_session
|> put_req_header("authorization", header)
|> AuthenticationPlug.call(opts)
assert conn.status == 403
assert conn.halted == true
end
end
describe "with a user_id in the session for an existing user" do
test "it assigns the user", %{conn: conn} do
opts = %{

View file

@ -1,6 +1,6 @@
defmodule Pleroma.UserTest do
alias Pleroma.Builders.UserBuilder
alias Pleroma.{User, Repo}
alias Pleroma.{User, Repo, Activity}
alias Pleroma.Web.OStatus
alias Pleroma.Web.Websub.WebsubClientSubscription
alias Pleroma.Web.CommonAPI
@ -39,6 +39,13 @@ test "follow takes a user and another user" do
assert User.ap_followers(followed) in user.following
end
test "can't follow a deactivated users" do
user = insert(:user)
followed = insert(:user, info: %{"deactivated" => true})
{:error, _} = User.follow(user, followed)
end
test "following a remote user will ensure a websub subscription is present" do
user = insert(:user)
{:ok, followed} = OStatus.make_user("shp@social.heldscal.la")
@ -325,5 +332,42 @@ test "get recipients from activity" do
assert user in recipients
assert addressed in recipients
end
end
test ".deactivate deactivates a user" do
user = insert(:user)
assert false == !!user.info["deactivated"]
{:ok, user} = User.deactivate(user)
assert true == user.info["deactivated"]
end
test ".delete deactivates a user, all follow relationships and all create activities" do
user = insert(:user)
followed = insert(:user)
follower = insert(:user)
{:ok, user} = User.follow(user, followed)
{:ok, follower} = User.follow(follower, user)
{:ok, activity} = CommonAPI.post(user, %{"status" => "2hu"})
{:ok, activity_two} = CommonAPI.post(follower, %{"status" => "3hu"})
{:ok, _, _} = CommonAPI.favorite(activity_two.id, user)
{:ok, _, _} = CommonAPI.favorite(activity.id, follower)
{:ok, _, _} = CommonAPI.repeat(activity.id, follower)
:ok = User.delete(user)
followed = Repo.get(User, followed.id)
follower = Repo.get(User, follower.id)
user = Repo.get(User, user.id)
assert user.info["deactivated"]
refute User.following?(user, followed)
refute User.following?(followed, follower)
# TODO: Remove favorites, repeats, delete activities.
refute Repo.get(Activity, activity.id)
end
end

View file

@ -0,0 +1,20 @@
defmodule Pleroma.Web.FederatorTest do
alias Pleroma.Web.Federator
use Pleroma.DataCase
test "enqueues an element according to priority" do
queue = [%{item: 1, priority: 2}]
new_queue = Federator.enqueue_sorted(queue, 2, 1)
assert new_queue == [%{item: 2, priority: 1}, %{item: 1, priority: 2}]
new_queue = Federator.enqueue_sorted(queue, 2, 3)
assert new_queue == [%{item: 1, priority: 2}, %{item: 2, priority: 3}]
end
test "pop first item" do
queue = [%{item: 2, priority: 1}, %{item: 1, priority: 2}]
assert {2, [%{item: 1, priority: 2}]} = Federator.queue_pop(queue)
end
end