Merge branch 'develop' into rename/pleroma_activity_consistency

lib/pleroma/activity.ex

@@ -36,6 +36,10 @@ def get_by_ap_id(ap_id) do
     )
   end
 
+  def get_by_id(id) do
+    Repo.get(Activity, id)
+  end
+
   def by_object_ap_id(ap_id) do
     from(
       activity in Activity,

lib/pleroma/reverse_proxy.ex

@@ -275,11 +275,24 @@ defp build_resp_headers(headers, opts) do
 
   defp build_resp_cache_headers(headers, _opts) do
     has_cache? = Enum.any?(headers, fn {k, _} -> k in @resp_cache_headers end)
+    has_cache_control? = List.keymember?(headers, "cache-control", 0)
 
-    if has_cache? do
-      headers
-    else
-      List.keystore(headers, "cache-control", 0, {"cache-control", @default_cache_control_header})
+    cond do
+      has_cache? && has_cache_control? ->
+        headers
+
+      has_cache? ->
+        # There's caching header present but no cache-control -- we need to explicitely override it to public
+        # as Plug defaults to "max-age=0, private, must-revalidate"
+        List.keystore(headers, "cache-control", 0, {"cache-control", "public"})
+
+      true ->
+        List.keystore(
+          headers,
+          "cache-control",
+          0,
+          {"cache-control", @default_cache_control_header}
+        )
     end
   end
 

lib/pleroma/uploaders/uploader.ex

@@ -27,18 +27,47 @@ defmodule Pleroma.Uploaders.Uploader do
     This allows to correctly proxy or redirect requests to the backend, while allowing to migrate backends without breaking any URL.
   * `{url, url :: String.t}` to bypass `get_file/2` and use the `url` directly in the activity.
   * `{:error, String.t}` error information if the file failed to be saved to the backend.
+  * `:wait_callback` will wait for an http post request at `/api/pleroma/upload_callback/:upload_path` and call the uploader's `http_callback/3` method.
 
 
   """
+  @type file_spec :: {:file | :url, String.t()}
   @callback put_file(Pleroma.Upload.t()) ::
-              :ok | {:ok, {:file | :url, String.t()}} | {:error, String.t()}
+              :ok | {:ok, file_spec()} | {:error, String.t()} | :wait_callback
+
+  @callback http_callback(Plug.Conn.t(), Map.t()) ::
+              {:ok, Plug.Conn.t()}
+              | {:ok, Plug.Conn.t(), file_spec()}
+              | {:error, Plug.Conn.t(), String.t()}
+  @optional_callbacks http_callback: 2
+
+  @spec put_file(module(), Pleroma.Upload.t()) :: {:ok, file_spec()} | {:error, String.t()}
 
-  @spec put_file(module(), Pleroma.Upload.t()) ::
-          {:ok, {:file | :url, String.t()}} | {:error, String.t()}
   def put_file(uploader, upload) do
     case uploader.put_file(upload) do
       :ok -> {:ok, {:file, upload.path}}
-      other -> other
+      :wait_callback -> handle_callback(uploader, upload)
+      {:ok, _} = ok -> ok
+      {:error, _} = error -> error
+    end
+  end
+
+  defp handle_callback(uploader, upload) do
+    :global.register_name({__MODULE__, upload.path}, self())
+
+    receive do
+      {__MODULE__, pid, conn, params} ->
+        case uploader.http_callback(conn, params) do
+          {:ok, conn, ok} ->
+            send(pid, {__MODULE__, conn})
+            {:ok, ok}
+
+          {:error, conn, error} ->
+            send(pid, {__MODULE__, conn})
+            {:error, error}
+        end
+    after
+      30_000 -> {:error, "Uploader callback timeout"}
     end
   end
 end

lib/pleroma/user.ex

@@ -901,7 +901,7 @@ def local_user_query do
   def active_local_user_query do
     from(
       u in local_user_query(),
-      where: fragment("?->'deactivated' @> 'false'", u.info)
+      where: fragment("not (?->'deactivated' @> 'true')", u.info)
     )
   end
 

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -140,8 +140,9 @@ def create(%{to: to, actor: actor, context: context, object: object} = params) d
              additional
            ),
          {:ok, activity} <- insert(create_data, local),
-         :ok <- maybe_federate(activity),
-         {:ok, _actor} <- User.increase_note_count(actor) do
+         # Changing note count prior to enqueuing federation task in order to avoid race conditions on updating user.info
+         {:ok, _actor} <- User.increase_note_count(actor),
+         :ok <- maybe_federate(activity) do
       {:ok, activity}
     end
   end
@@ -288,8 +289,9 @@ def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, local \\ tru
 
     with {:ok, _} <- Object.delete(object),
          {:ok, activity} <- insert(data, local),
-         :ok <- maybe_federate(activity),
-         {:ok, _actor} <- User.decrease_note_count(user) do
+         # Changing note count prior to enqueuing federation task in order to avoid race conditions on updating user.info
+         {:ok, _actor} <- User.decrease_note_count(user),
+         :ok <- maybe_federate(activity) do
       {:ok, activity}
     end
   end
@@ -802,11 +804,23 @@ def fetch_and_contain_remote_object_from_id(id) do
   def is_public?(%Object{data: %{"type" => "Tombstone"}}), do: false
   def is_public?(%Object{data: data}), do: is_public?(data)
   def is_public?(%Activity{data: data}), do: is_public?(data)
+  def is_public?(%{"directMessage" => true}), do: false
 
   def is_public?(data) do
     "https://www.w3.org/ns/activitystreams#Public" in (data["to"] ++ (data["cc"] || []))
   end
 
+  def is_private?(activity) do
+    !is_public?(activity) && Enum.any?(activity.data["to"], &String.contains?(&1, "/followers"))
+  end
+
+  def is_direct?(%Activity{data: %{"directMessage" => true}}), do: true
+  def is_direct?(%Object{data: %{"directMessage" => true}}), do: true
+
+  def is_direct?(activity) do
+    !is_public?(activity) && !is_private?(activity)
+  end
+
   def visible_for_user?(activity, nil) do
     is_public?(activity)
   end

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -93,12 +93,47 @@ def fix_addressing_list(map, field) do
     end
   end
 
-  def fix_addressing(map) do
-    map
+  def fix_explicit_addressing(%{"to" => to, "cc" => cc} = object, explicit_mentions) do
+    explicit_to =
+      to
+      |> Enum.filter(fn x -> x in explicit_mentions end)
+
+    explicit_cc =
+      to
+      |> Enum.filter(fn x -> x not in explicit_mentions end)
+
+    final_cc =
+      (cc ++ explicit_cc)
+      |> Enum.uniq()
+
+    object
+    |> Map.put("to", explicit_to)
+    |> Map.put("cc", final_cc)
+  end
+
+  def fix_explicit_addressing(object, _explicit_mentions), do: object
+
+  # if directMessage flag is set to true, leave the addressing alone
+  def fix_explicit_addressing(%{"directMessage" => true} = object), do: object
+
+  def fix_explicit_addressing(object) do
+    explicit_mentions =
+      object
+      |> Utils.determine_explicit_mentions()
+
+    explicit_mentions = explicit_mentions ++ ["https://www.w3.org/ns/activitystreams#Public"]
+
+    object
+    |> fix_explicit_addressing(explicit_mentions)
+  end
+
+  def fix_addressing(object) do
+    object
     |> fix_addressing_list("to")
     |> fix_addressing_list("cc")
     |> fix_addressing_list("bto")
     |> fix_addressing_list("bcc")
+    |> fix_explicit_addressing
   end
 
   def fix_actor(%{"attributedTo" => actor} = object) do
@@ -348,6 +383,7 @@ def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = obj
         additional:
           Map.take(data, [
             "cc",
+            "directMessage",
             "id"
           ])
       }

lib/pleroma/web/activity_pub/utils.ex

@@ -25,6 +25,20 @@ def normalize_params(params) do
     Map.put(params, "actor", get_ap_id(params["actor"]))
   end
 
+  def determine_explicit_mentions(%{"tag" => tag} = _object) when is_list(tag) do
+    tag
+    |> Enum.filter(fn x -> is_map(x) end)
+    |> Enum.filter(fn x -> x["type"] == "Mention" end)
+    |> Enum.map(fn x -> x["href"] end)
+  end
+
+  def determine_explicit_mentions(%{"tag" => tag} = object) when is_map(tag) do
+    Map.put(object, "tag", [tag])
+    |> determine_explicit_mentions()
+  end
+
+  def determine_explicit_mentions(_), do: []
+
   defp recipient_in_collection(ap_id, coll) when is_binary(coll), do: ap_id == coll
   defp recipient_in_collection(ap_id, coll) when is_list(coll), do: ap_id in coll
   defp recipient_in_collection(_, _), do: false

lib/pleroma/web/common_api/common_api.ex

@@ -143,7 +143,7 @@ def post(user, %{"status" => status} = data) do
           actor: user,
           context: context,
           object: object,
-          additional: %{"cc" => cc}
+          additional: %{"cc" => cc, "directMessage" => visibility == "direct"}
         })
 
       res

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -231,6 +231,9 @@ def get_visibility(object) do
       Enum.any?(to, &String.contains?(&1, "/followers")) ->
         "private"
 
+      length(cc) > 0 ->
+        "private"
+
       true ->
         "direct"
     end

lib/pleroma/web/router.ex

@@ -107,6 +107,11 @@ defmodule Pleroma.Web.Router do
     get("/captcha", UtilController, :captcha)
   end
 
+  scope "/api/pleroma", Pleroma.Web do
+    pipe_through(:pleroma_api)
+    post("/uploader_callback/:upload_path", UploaderController, :callback)
+  end
+
   scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do
     pipe_through(:admin_api)
     delete("/user", AdminAPIController, :user_delete)

lib/pleroma/web/uploader_controller.ex

@@ -0,0 +1,25 @@
+defmodule Pleroma.Web.UploaderController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Uploaders.Uploader
+
+  def callback(conn, params = %{"upload_path" => upload_path}) do
+    process_callback(conn, :global.whereis_name({Uploader, upload_path}), params)
+  end
+
+  def callbacks(conn, _) do
+    send_resp(conn, 400, "bad request")
+  end
+
+  defp process_callback(conn, pid, params) when is_pid(pid) do
+    send(pid, {Uploader, self(), conn, params})
+
+    receive do
+      {Uploader, conn} -> conn
+    end
+  end
+
+  defp process_callback(conn, _, _) do
+    send_resp(conn, 400, "bad request")
+  end
+end

priv/repo/migrations/20190122153157_update_activity_visibility.exs

@@ -0,0 +1,36 @@
+defmodule Pleroma.Repo.Migrations.UpdateActivityVisibility do
+  use Ecto.Migration
+  @disable_ddl_transaction true
+
+  def up do
+    definition = """
+    create or replace function activity_visibility(actor varchar, recipients varchar[], data jsonb) returns varchar as $$
+    DECLARE
+      fa varchar;
+      public varchar := 'https://www.w3.org/ns/activitystreams#Public';
+    BEGIN
+      SELECT COALESCE(users.follower_address, '') into fa from users where users.ap_id = actor;
+
+      IF data->'to' ? public THEN
+        RETURN 'public';
+      ELSIF data->'cc' ? public THEN
+        RETURN 'unlisted';
+      ELSIF ARRAY[fa] && recipients THEN
+        RETURN 'private';
+      ELSIF not(ARRAY[fa, public] && recipients) THEN
+        RETURN 'direct';
+      ELSE
+        RETURN 'unknown';
+      END IF;
+    END;
+    $$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE SECURITY DEFINER;
+    """
+
+    execute(definition)
+
+  end
+
+  def down do
+
+  end
+end

priv/static/schemas/litepub-0.1.jsonld

@@ -17,7 +17,9 @@
             "toot": "http://joinmastodon.org/ns#",
             "totalItems": "as:totalItems",
             "value": "schema:value",
-            "sensitive": "as:sensitive"
+            "sensitive": "as:sensitive",
+            "litepub": "http://litepub.social/ns#",
+            "directMessage": "litepub:directMessage"
         }
     ]
 }

test/integration/mastodon_websocket_test.exs

@@ -66,13 +66,10 @@ test "receives well formatted events" do
     assert json["payload"]
     assert {:ok, json} = Jason.decode(json["payload"])
 
-    # Note: we remove the "statuses_count" from this result as it changes in the meantime
-
     view_json =
       Pleroma.Web.MastodonAPI.StatusView.render("status.json", activity: activity, for: nil)
       |> Jason.encode!()
       |> Jason.decode!()
-      |> put_in(["account", "statuses_count"], 0)
 
     assert json == view_json
   end

test/web/activity_pub/transmogrifier_test.exs

@@ -162,6 +162,36 @@ test "it works for incoming notices with url not being a string (prismo)" do
       assert data["object"]["url"] == "https://prismo.news/posts/83"
     end
 
+    test "it cleans up incoming notices which are not really DMs" do
+      user = insert(:user)
+      other_user = insert(:user)
+
+      to = [user.ap_id, other_user.ap_id]
+
+      data =
+        File.read!("test/fixtures/mastodon-post-activity.json")
+        |> Poison.decode!()
+        |> Map.put("to", to)
+        |> Map.put("cc", [])
+
+      object =
+        data["object"]
+        |> Map.put("to", to)
+        |> Map.put("cc", [])
+
+      data = Map.put(data, "object", object)
+
+      {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
+
+      assert data["to"] == []
+      assert data["cc"] == to
+
+      object = data["object"]
+
+      assert object["to"] == []
+      assert object["cc"] == to
+    end
+
     test "it works for incoming follow requests" do
       user = insert(:user)
 
@@ -872,6 +902,34 @@ test "it adds like collection to object" do
       assert modified["object"]["likes"]["type"] == "OrderedCollection"
       assert modified["object"]["likes"]["totalItems"] == 0
     end
+
+    test "the directMessage flag is present" do
+      user = insert(:user)
+      other_user = insert(:user)
+
+      {:ok, activity} = CommonAPI.post(user, %{"status" => "2hu :moominmamma:"})
+
+      {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
+
+      assert modified["directMessage"] == false
+
+      {:ok, activity} =
+        CommonAPI.post(user, %{"status" => "@#{other_user.nickname} :moominmamma:"})
+
+      {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
+
+      assert modified["directMessage"] == false
+
+      {:ok, activity} =
+        CommonAPI.post(user, %{
+          "status" => "@#{other_user.nickname} :moominmamma:",
+          "visibility" => "direct"
+        })
+
+      {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
+
+      assert modified["directMessage"] == true
+    end
   end
 
   describe "user upgrade" do

test/web/activity_pub/utils_test.exs

@@ -0,0 +1,57 @@
+defmodule Pleroma.Web.ActivityPub.UtilsTest do
+  use Pleroma.DataCase
+  alias Pleroma.Web.ActivityPub.Utils
+
+  describe "determine_explicit_mentions()" do
+    test "works with an object that has mentions" do
+      object = %{
+        "tag" => [
+          %{
+            "type" => "Mention",
+            "href" => "https://example.com/~alyssa",
+            "name" => "Alyssa P. Hacker"
+          }
+        ]
+      }
+
+      assert Utils.determine_explicit_mentions(object) == ["https://example.com/~alyssa"]
+    end
+
+    test "works with an object that does not have mentions" do
+      object = %{
+        "tag" => [
+          %{"type" => "Hashtag", "href" => "https://example.com/tag/2hu", "name" => "2hu"}
+        ]
+      }
+
+      assert Utils.determine_explicit_mentions(object) == []
+    end
+
+    test "works with an object that has mentions and other tags" do
+      object = %{
+        "tag" => [
+          %{
+            "type" => "Mention",
+            "href" => "https://example.com/~alyssa",
+            "name" => "Alyssa P. Hacker"
+          },
+          %{"type" => "Hashtag", "href" => "https://example.com/tag/2hu", "name" => "2hu"}
+        ]
+      }
+
+      assert Utils.determine_explicit_mentions(object) == ["https://example.com/~alyssa"]
+    end
+
+    test "works with an object that has no tags" do
+      object = %{}
+
+      assert Utils.determine_explicit_mentions(object) == []
+    end
+
+    test "works with an object that has only IR tags" do
+      object = %{"tag" => ["2hu"]}
+
+      assert Utils.determine_explicit_mentions(object) == []
+    end
+  end
+end

test/web/mastodon_api/mastodon_api_controller_test.exs

@@ -10,6 +10,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
   alias Pleroma.Web.{OStatus, CommonAPI}
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.MastodonAPI.FilterView
+  alias Ecto.Changeset
   import Pleroma.Factory
   import ExUnit.CaptureLog
   import Tesla.Mock
@@ -1483,6 +1484,16 @@ test "get instance information", %{conn: conn} do
 
     {:ok, _} = TwitterAPI.create_status(user, %{"status" => "cofe"})
 
+    # Stats should count users with missing or nil `info.deactivated` value
+    user = Repo.get(User, user.id)
+    info_change = Changeset.change(user.info, %{deactivated: nil})
+
+    {:ok, _user} =
+      user
+      |> Changeset.change()
+      |> Changeset.put_embed(:info, info_change)
+      |> User.update_and_set_cache()
+
     Pleroma.Stats.update_stats()
 
     conn = get(conn, "/api/v1/instance")