diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex
index 6dee61dd6..e7613a5c8 100644
--- a/lib/pleroma/web/activity_pub/visibility.ex
+++ b/lib/pleroma/web/activity_pub/visibility.ex
@@ -13,11 +13,12 @@ def is_public?(data) do
   end
 
   def is_private?(activity) do
-    unless is_public?(activity) do
-      follower_address = User.get_cached_by_ap_id(activity.data["actor"]).follower_address
-      Enum.any?(activity.data["to"], &(&1 == follower_address))
+    with false <- is_public?(activity),
+         %User{follower_address: follower_address} <-
+           User.get_cached_by_ap_id(activity.data["actor"]) do
+      follower_address in activity.data["to"]
     else
-      false
+      _ -> false
     end
   end
 
diff --git a/test/web/activity_pub/visibilty_test.exs b/test/web/activity_pub/visibilty_test.exs
index 24b96c4aa..ff0e72401 100644
--- a/test/web/activity_pub/visibilty_test.exs
+++ b/test/web/activity_pub/visibilty_test.exs
@@ -95,4 +95,14 @@ test "visible_for_user?", %{
     refute Visibility.visible_for_user?(private, unrelated)
     refute Visibility.visible_for_user?(direct, unrelated)
   end
+
+  test "doesn't die when the user doesn't exist",
+       %{
+         direct: direct,
+         user: user
+       } do
+    Repo.delete(user)
+    Cachex.clear(:user_cache)
+    refute Visibility.is_private?(direct)
+  end
 end