From 4f3a6337454807f4145bbc1830c3d55dd883d46d Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 2 Sep 2020 20:21:33 +0400 Subject: [PATCH] Add `backups` table --- lib/pleroma/{export.ex => backup.ex} | 110 ++++++++++++++---- .../20200831192323_create_backups.exs | 17 +++ test/{export_test.exs => backup_test.exs} | 48 ++++++-- 3 files changed, 141 insertions(+), 34 deletions(-) rename lib/pleroma/{export.ex => backup.ex} (60%) create mode 100644 priv/repo/migrations/20200831192323_create_backups.exs rename test/{export_test.exs => backup_test.exs} (75%) diff --git a/lib/pleroma/export.ex b/lib/pleroma/backup.ex similarity index 60% rename from lib/pleroma/export.ex rename to lib/pleroma/backup.ex index 8b1bfefe2..4580d8f92 100644 --- a/lib/pleroma/export.ex +++ b/lib/pleroma/backup.ex @@ -2,41 +2,110 @@ # Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Export do +defmodule Pleroma.Backup do + use Ecto.Schema + + import Ecto.Changeset + import Ecto.Query + alias Pleroma.Activity alias Pleroma.Bookmark + alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.UserView - import Ecto.Query + schema "backups" do + field(:content_type, :string) + field(:file_name, :string) + field(:file_size, :integer, default: 0) + field(:processed, :boolean, default: false) + + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) + + timestamps() + end + + def create(user) do + with :ok <- validate_limit(user), + {:ok, backup} <- user |> new() |> Repo.insert() do + {:ok, backup} + end + end + + def new(user) do + rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false) + datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now()) + name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip" + + %__MODULE__{ + user_id: user.id, + content_type: "application/zip", + file_name: name + } + end + + defp validate_limit(user) do + case get_last(user.id) do + %__MODULE__{inserted_at: inserted_at} -> + days = 7 + diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days) + + if diff > days do + :ok + else + {:error, "Last export was less than #{days} days ago"} + end + + nil -> + :ok + end + end + + def get_last(user_id) do + __MODULE__ + |> where(user_id: ^user_id) + |> order_by(desc: :id) + |> limit(1) + |> Repo.one() + end + + def process(%__MODULE__{} = backup) do + with {:ok, zip_file} <- zip(backup), + {:ok, %{size: size}} <- File.stat(zip_file), + {:ok, _upload} <- upload(backup, zip_file) do + backup + |> cast(%{file_size: size, processed: true}, [:file_size, :processed]) + |> Repo.update() + end + end @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json'] + def zip(%__MODULE__{} = backup) do + backup = Repo.preload(backup, :user) + name = String.trim_trailing(backup.file_name, ".zip") + dir = Path.join(System.tmp_dir!(), name) - def run(user) do - with {:ok, path} <- create_dir(user), - :ok <- actor(path, user), - :ok <- statuses(path, user), - :ok <- likes(path, user), - :ok <- bookmarks(path, user), - {:ok, zip_path} <- :zip.create('#{path}.zip', @files, cwd: path), - {:ok, _} <- File.rm_rf(path) do + with :ok <- File.mkdir(dir), + :ok <- actor(dir, backup.user), + :ok <- statuses(dir, backup.user), + :ok <- likes(dir, backup.user), + :ok <- bookmarks(dir, backup.user), + {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir), + {:ok, _} <- File.rm_rf(dir) do {:ok, :binary.list_to_bin(zip_path)} end end - def upload(zip_path) do + def upload(%__MODULE__{} = backup, zip_path) do uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) - file_name = zip_path |> String.split("/") |> List.last() - id = Ecto.UUID.generate() upload = %Pleroma.Upload{ - id: id, - name: file_name, + name: backup.file_name, tempfile: zip_path, - content_type: "application/zip", - path: id <> "/" <> file_name + content_type: backup.content_type, + path: "backups/" <> backup.file_name } with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload), @@ -54,13 +123,6 @@ defp actor(dir, user) do end end - defp create_dir(user) do - datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now()) - dir = Path.join(System.tmp_dir!(), "archive-#{user.id}-#{datetime}") - - with :ok <- File.mkdir(dir), do: {:ok, dir} - end - defp write_header(file, name) do IO.write( file, diff --git a/priv/repo/migrations/20200831192323_create_backups.exs b/priv/repo/migrations/20200831192323_create_backups.exs new file mode 100644 index 000000000..3ac5889e2 --- /dev/null +++ b/priv/repo/migrations/20200831192323_create_backups.exs @@ -0,0 +1,17 @@ +defmodule Pleroma.Repo.Migrations.CreateBackups do + use Ecto.Migration + + def change do + create_if_not_exists table(:backups) do + add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) + add(:file_name, :string, null: false) + add(:content_type, :string, null: false) + add(:processed, :boolean, null: false, default: false) + add(:file_size, :bigint) + + timestamps() + end + + create_if_not_exists(index(:backups, [:user_id])) + end +end diff --git a/test/export_test.exs b/test/backup_test.exs similarity index 75% rename from test/export_test.exs rename to test/backup_test.exs index d7e8f558c..27f5cb7f7 100644 --- a/test/export_test.exs +++ b/test/backup_test.exs @@ -2,15 +2,41 @@ # Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.ExportTest do +defmodule Pleroma.BackupTest do use Pleroma.DataCase import Pleroma.Factory import Mock + alias Pleroma.Backup alias Pleroma.Bookmark alias Pleroma.Web.CommonAPI - test "it exports user data" do + test "it creates a backup record" do + %{id: user_id} = user = insert(:user) + assert {:ok, backup} = Backup.create(user) + + assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup + end + + test "it return an error if the export limit is over" do + %{id: user_id} = user = insert(:user) + limit_days = 7 + + assert {:ok, backup} = Backup.create(user) + assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup + + assert Backup.create(user) == {:error, "Last export was less than #{limit_days} days ago"} + end + + test "it process a backup record" do + %{id: user_id} = user = insert(:user) + assert {:ok, %{id: backup_id} = backup} = Backup.create(user) + assert {:ok, %Backup{} = backup} = Backup.process(backup) + assert backup.file_size > 0 + assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup + end + + test "it creates a zip archive with user data" do user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) {:ok, %{object: %{data: %{"id" => id1}}} = status1} = @@ -28,7 +54,8 @@ test "it exports user data" do Bookmark.create(user.id, status2.id) Bookmark.create(user.id, status3.id) - assert {:ok, path} = Pleroma.Export.run(user) + assert {:ok, backup} = user |> Backup.new() |> Repo.insert() + assert {:ok, path} = Backup.zip(backup) assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory]) assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile) @@ -110,7 +137,7 @@ test "it exports user data" do File.rm!(path) end - describe "it uploads an exported backup archive" do + describe "it uploads a backup archive" do setup do clear_config(Pleroma.Uploaders.S3, bucket: "test_bucket", @@ -129,23 +156,24 @@ test "it exports user data" do Bookmark.create(user.id, status2.id) Bookmark.create(user.id, status3.id) - assert {:ok, path} = Pleroma.Export.run(user) + assert {:ok, backup} = user |> Backup.new() |> Repo.insert() + assert {:ok, path} = Backup.zip(backup) - [path: path] + [path: path, backup: backup] end - test "S3", %{path: path} do + test "S3", %{path: path, backup: backup} do Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3) with_mock ExAws, request: fn _ -> {:ok, :ok} end do - assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) + assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path) end end - test "Local", %{path: path} do + test "Local", %{path: path, backup: backup} do Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) - assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) + assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path) end end end