From cc9985eb1def29e336b25a3877c010710d64c2a9 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Tue, 1 Nov 2022 23:38:54 -0400 Subject: [PATCH 01/30] creating migration pages --- src/BirdsiteLive.Domain/UserService.cs | 6 ++ .../Controllers/MigrationController.cs | 86 +++++++++++++++++++ src/BirdsiteLive/Views/Migration/Index.cshtml | 45 ++++++++++ src/BirdsiteLive/Views/Users/Index.cshtml | 4 + src/BirdsiteLive/wwwroot/css/birdsite.css | 15 ++++ 5 files changed, 156 insertions(+) create mode 100644 src/BirdsiteLive/Controllers/MigrationController.cs create mode 100644 src/BirdsiteLive/Views/Migration/Index.cshtml diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index a080180..42cf6d3 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -113,6 +113,12 @@ namespace BirdsiteLive.Domain type = "PropertyValue", name = "Official", value = $"https://twitter.com/{acct}" + }, + new UserAttachment + { + type = "PropertyValue", + name = "Disclaimer", + value = "This is an automatically created and managed mirror profile from Twitter. While it reflects exactly the content of the original account, it doesn't provide support for interactions and replies. It is an equivalent view from other 3rd party Twitter client apps and uses the same technical means to provide it." } }, endpoints = new EndPoints diff --git a/src/BirdsiteLive/Controllers/MigrationController.cs b/src/BirdsiteLive/Controllers/MigrationController.cs new file mode 100644 index 0000000..a0b84aa --- /dev/null +++ b/src/BirdsiteLive/Controllers/MigrationController.cs @@ -0,0 +1,86 @@ +using Microsoft.AspNetCore.Mvc; +using System.Security.Cryptography; +using System.Text; +using Npgsql.TypeHandlers; + +namespace BirdsiteLive.Controllers +{ + public class MigrationController : Controller + { + [HttpGet] + [Route("/migration/{id}")] + public IActionResult Index(string id) + { + var migrationCode = GetMigrationCode(id); + var data = new MigrationData() + { + Acct = id, + MigrationCode = migrationCode + }; + + return View(data); + } + + [HttpPost] + [Route("/migration/{id}")] + public IActionResult Migrate(string id, string tweetid, string handle) + { + var migrationCode = GetMigrationCode(id); + var data = new MigrationData() + { + Acct = id, + MigrationCode = migrationCode, + + IsAcctProvided = !string.IsNullOrWhiteSpace(handle), + IsTweetProvided = !string.IsNullOrWhiteSpace(tweetid), + + TweetId = tweetid, + FediverseAccount = handle + }; + + return View("Index", data); + } + + public byte[] GetHash(string inputString) + { + using (HashAlgorithm algorithm = SHA256.Create()) + return algorithm.ComputeHash(Encoding.UTF8.GetBytes(inputString)); + } + + public string GetHashString(string inputString) + { + StringBuilder sb = new StringBuilder(); + foreach (byte b in GetHash(inputString)) + sb.Append(b.ToString("X2")); + + return sb.ToString(); + } + + public string GetMigrationCode(string acct) + { + var hash = GetHashString(acct); + return $"[[BirdsiteLIVE-MigrationCode|{hash.Substring(0, 10)}]]"; + } + } + + + + public class MigrationData + { + public string Acct { get; set; } + + public string FediverseAccount { get; set; } + public string TweetId { get; set; } + + public string MigrationCode { get; set; } + + public bool IsTweetProvided { get; set; } + public bool IsAcctProvided { get; set; } + + public bool IsTweetValid { get; set; } + public bool IsAcctValid { get; set; } + + public string ErrorMessage { get; set; } + + } +} diff --git a/src/BirdsiteLive/Views/Migration/Index.cshtml b/src/BirdsiteLive/Views/Migration/Index.cshtml new file mode 100644 index 0000000..dfdf14d --- /dev/null +++ b/src/BirdsiteLive/Views/Migration/Index.cshtml @@ -0,0 +1,45 @@ +@model BirdsiteLive.Controllers.MigrationData +@{ + ViewData["Title"] = "Migration"; +} + +
+

Migrate @@@ViewData.Model.Acct

+ + @if (!ViewData.Model.IsAcctProvided && !ViewData.Model.IsAcctProvided) + { +

What is needed?

+ +

You'll need a Fediverse account and access to the Twitter account to provide proof of ownership.

+ +

What will migration do?

+ +

Migration will transfer the followers to the provided account.
+ After a week, the account will be deleted. It will also be blacklisted so that it can't be recreated.

+ } + +

Start the migration!

+ +

Please copy and post this string in a Tweet (the string must be untampered, but you can write anything you want before or after it):

+ + +
+ +

Provide migration information:

+
+ @*
+ + + We'll never share your email with anyone else. +
*@ +
+ + +
+
+ + +
+ +
+
\ No newline at end of file diff --git a/src/BirdsiteLive/Views/Users/Index.cshtml b/src/BirdsiteLive/Views/Users/Index.cshtml index 945964a..8177d6c 100644 --- a/src/BirdsiteLive/Views/Users/Index.cshtml +++ b/src/BirdsiteLive/Views/Users/Index.cshtml @@ -45,4 +45,8 @@ } + +
+ I'm the owner of this account and I'm now on the Fediverse +
\ No newline at end of file diff --git a/src/BirdsiteLive/wwwroot/css/birdsite.css b/src/BirdsiteLive/wwwroot/css/birdsite.css index 5b6023c..159a50a 100644 --- a/src/BirdsiteLive/wwwroot/css/birdsite.css +++ b/src/BirdsiteLive/wwwroot/css/birdsite.css @@ -71,3 +71,18 @@ margin-left: 60px; /*font-weight: bold;*/ } + +.user-owner { + font-size: .8em; + padding-top: 20px; +} + +/** Migration **/ + +.migration__title { + font-size: 1.8em; +} + +.migration__subtitle { + font-size: 1.4em; +} \ No newline at end of file From ec3234324c376997cc9213f40aa08d3516b71004 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 2 Nov 2022 00:10:46 -0400 Subject: [PATCH 02/30] validating tweet --- src/BirdsiteLive.Domain/MigrationService.cs | 77 ++++++++++++++++++ .../Controllers/MigrationController.cs | 78 ++++++++++++++----- src/BirdsiteLive/Views/Migration/Index.cshtml | 19 +++-- 3 files changed, 147 insertions(+), 27 deletions(-) create mode 100644 src/BirdsiteLive.Domain/MigrationService.cs diff --git a/src/BirdsiteLive.Domain/MigrationService.cs b/src/BirdsiteLive.Domain/MigrationService.cs new file mode 100644 index 0000000..2e4a1d5 --- /dev/null +++ b/src/BirdsiteLive.Domain/MigrationService.cs @@ -0,0 +1,77 @@ +using System; +using System.Linq; +using BirdsiteLive.Twitter; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; + +namespace BirdsiteLive.Domain +{ + public class MigrationService + { + private readonly ITwitterTweetsService _twitterTweetsService; + + #region Ctor + public MigrationService(ITwitterTweetsService twitterTweetsService) + { + _twitterTweetsService = twitterTweetsService; + } + #endregion + + public string GetMigrationCode(string acct) + { + var hash = GetHashString(acct); + return $"[[BirdsiteLIVE-MigrationCode|{hash.Substring(0, 10)}]]"; + } + + public bool ValidateTweet(string acct, string tweetId) + { + var code = GetMigrationCode(acct); + + var castedTweetId = ExtractedTweetId(tweetId); + var tweet = _twitterTweetsService.GetTweet(castedTweetId); + + if (tweet == null) throw new Exception("Tweet not found"); + if (!tweet.MessageContent.Contains(code)) throw new Exception("Tweet don't have migration code"); + + return true; + } + + private long ExtractedTweetId(string tweetId) + { + long castedId; + if (long.TryParse(tweetId, out castedId)) + return castedId; + + var urlPart = tweetId.Split('/').LastOrDefault(); + if (long.TryParse(urlPart, out castedId)) + return castedId; + + throw new ArgumentException("Unvalid Tweet ID"); + } + + public async Task ValidateFediverseAcctAsync(string fediverseAcct) + { + return true; + } + + public async Task MigrateAccountAsync(string acct, string tweetId, string fediverseAcct, bool triggerRemoteMigration) + { + } + + private byte[] GetHash(string inputString) + { + using (HashAlgorithm algorithm = SHA256.Create()) + return algorithm.ComputeHash(Encoding.UTF8.GetBytes(inputString)); + } + + private string GetHashString(string inputString) + { + StringBuilder sb = new StringBuilder(); + foreach (byte b in GetHash(inputString)) + sb.Append(b.ToString("X2")); + + return sb.ToString(); + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Controllers/MigrationController.cs b/src/BirdsiteLive/Controllers/MigrationController.cs index a0b84aa..d0e121f 100644 --- a/src/BirdsiteLive/Controllers/MigrationController.cs +++ b/src/BirdsiteLive/Controllers/MigrationController.cs @@ -1,17 +1,29 @@ -using Microsoft.AspNetCore.Mvc; +using System; +using Microsoft.AspNetCore.Mvc; using System.Security.Cryptography; using System.Text; +using System.Threading.Tasks; using Npgsql.TypeHandlers; +using BirdsiteLive.Domain; namespace BirdsiteLive.Controllers { public class MigrationController : Controller { + private readonly MigrationService _migrationService; + + #region Ctor + public MigrationController(MigrationService migrationService) + { + _migrationService = migrationService; + } + #endregion + [HttpGet] [Route("/migration/{id}")] public IActionResult Index(string id) { - var migrationCode = GetMigrationCode(id); + var migrationCode = _migrationService.GetMigrationCode(id); var data = new MigrationData() { Acct = id, @@ -23,9 +35,10 @@ namespace BirdsiteLive.Controllers [HttpPost] [Route("/migration/{id}")] - public IActionResult Migrate(string id, string tweetid, string handle) + public async Task Migrate(string id, string tweetid, string handle) { - var migrationCode = GetMigrationCode(id); + var migrationCode = _migrationService.GetMigrationCode(id); + var data = new MigrationData() { Acct = id, @@ -38,28 +51,51 @@ namespace BirdsiteLive.Controllers FediverseAccount = handle }; + try + { + var isAcctValid = await _migrationService.ValidateFediverseAcctAsync(handle); + var isTweetValid = _migrationService.ValidateTweet(id, tweetid); + + data.IsAcctValid = isAcctValid; + data.IsTweetValid = isTweetValid; + } + catch (Exception e) + { + data.ErrorMessage = e.Message; + } + + + if (data.IsAcctValid && data.IsTweetValid) + { + try + { + await _migrationService.MigrateAccountAsync(id, tweetid, handle, true); + data.MigrationSuccess = true; + } + catch (Exception e) + { + Console.WriteLine(e); + data.ErrorMessage = e.Message; + } + } + return View("Index", data); } - public byte[] GetHash(string inputString) + [HttpPost] + [Route("/migration/{id}/{tweetid}/{handle}")] + public async Task RemoteMigrate(string id, string tweetid, string handle) { - using (HashAlgorithm algorithm = SHA256.Create()) - return algorithm.ComputeHash(Encoding.UTF8.GetBytes(inputString)); - } + var isAcctValid = await _migrationService.ValidateFediverseAcctAsync(handle); + var isTweetValid = _migrationService.ValidateTweet(id, tweetid); - public string GetHashString(string inputString) - { - StringBuilder sb = new StringBuilder(); - foreach (byte b in GetHash(inputString)) - sb.Append(b.ToString("X2")); + if (isAcctValid && isTweetValid) + { + await _migrationService.MigrateAccountAsync(id, tweetid, handle, false); + return Ok(); + } - return sb.ToString(); - } - - public string GetMigrationCode(string acct) - { - var hash = GetHashString(acct); - return $"[[BirdsiteLIVE-MigrationCode|{hash.Substring(0, 10)}]]"; + return StatusCode(500); } } @@ -81,6 +117,6 @@ namespace BirdsiteLive.Controllers public bool IsAcctValid { get; set; } public string ErrorMessage { get; set; } - + public bool MigrationSuccess { get; set; } } } diff --git a/src/BirdsiteLive/Views/Migration/Index.cshtml b/src/BirdsiteLive/Views/Migration/Index.cshtml index dfdf14d..422b8f7 100644 --- a/src/BirdsiteLive/Views/Migration/Index.cshtml +++ b/src/BirdsiteLive/Views/Migration/Index.cshtml @@ -4,9 +4,16 @@ }
+ @if (!string.IsNullOrWhiteSpace(ViewData.Model.ErrorMessage)) + { + + } +

Migrate @@@ViewData.Model.Acct

- - @if (!ViewData.Model.IsAcctProvided && !ViewData.Model.IsAcctProvided) + + @if (!ViewData.Model.IsAcctProvided && !ViewData.Model.IsTweetProvided) {

What is needed?

@@ -19,12 +26,12 @@ }

Start the migration!

- +

Please copy and post this string in a Tweet (the string must be untampered, but you can write anything you want before or after it):

- - + +
- +

Provide migration information:

@*
From 15f0ad55ae797d0fc81a0d0e2469e5b4a191d278 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 2 Nov 2022 01:15:05 -0400 Subject: [PATCH 03/30] validation of the fediverse user --- src/BirdsiteLive.Domain/ActivityPubService.cs | 25 +++++++++++++++++++ src/BirdsiteLive.Domain/MigrationService.cs | 18 +++++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs index c460a2d..7a9aeeb 100644 --- a/src/BirdsiteLive.Domain/ActivityPubService.cs +++ b/src/BirdsiteLive.Domain/ActivityPubService.cs @@ -16,12 +16,19 @@ namespace BirdsiteLive.Domain { public interface IActivityPubService { + Task GetUserIdAsync(string acct); Task GetUser(string objectId); Task PostDataAsync(T data, string targetHost, string actorUrl, string inbox = null); Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox); } + public class WebFinger + { + public string subject { get; set; } + public string[] aliases { get; set; } + } + public class ActivityPubService : IActivityPubService { private readonly InstanceSettings _instanceSettings; @@ -39,6 +46,24 @@ namespace BirdsiteLive.Domain } #endregion + public async Task GetUserIdAsync(string acct) + { + var splittedAcct = acct.Trim('@').Split('@'); + + var url = $"https://{splittedAcct[1]}/.well-known/webfinger?resource=acct:{splittedAcct[0]}@{splittedAcct[1]}"; + + var httpClient = _httpClientFactory.CreateClient(); + httpClient.DefaultRequestHeaders.Add("Accept", "application/json"); + var result = await httpClient.GetAsync(url); + + result.EnsureSuccessStatusCode(); + + var content = await result.Content.ReadAsStringAsync(); + + var actor = JsonConvert.DeserializeObject(content); + return actor.aliases.FirstOrDefault(); + } + public async Task GetUser(string objectId) { var httpClient = _httpClientFactory.CreateClient(); diff --git a/src/BirdsiteLive.Domain/MigrationService.cs b/src/BirdsiteLive.Domain/MigrationService.cs index 2e4a1d5..d8818dc 100644 --- a/src/BirdsiteLive.Domain/MigrationService.cs +++ b/src/BirdsiteLive.Domain/MigrationService.cs @@ -10,11 +10,13 @@ namespace BirdsiteLive.Domain public class MigrationService { private readonly ITwitterTweetsService _twitterTweetsService; + private readonly IActivityPubService _activityPubService; #region Ctor - public MigrationService(ITwitterTweetsService twitterTweetsService) + public MigrationService(ITwitterTweetsService twitterTweetsService, IActivityPubService activityPubService) { _twitterTweetsService = twitterTweetsService; + _activityPubService = activityPubService; } #endregion @@ -52,11 +54,23 @@ namespace BirdsiteLive.Domain public async Task ValidateFediverseAcctAsync(string fediverseAcct) { - return true; + if (string.IsNullOrWhiteSpace(fediverseAcct)) + throw new ArgumentException("Please provide Fediverse account"); + + if( !fediverseAcct.Contains('@') || fediverseAcct.Trim('@').Split('@').Length != 2) + throw new ArgumentException("Please provide valid Fediverse handle"); + + var objectId = await _activityPubService.GetUserIdAsync(fediverseAcct); + var user = await _activityPubService.GetUser(objectId); + + if(user != null) return true; + + return false; } public async Task MigrateAccountAsync(string acct, string tweetId, string fediverseAcct, bool triggerRemoteMigration) { + throw new NotImplementedException("Migration not implemented"); } private byte[] GetHash(string inputString) From 76b2e659abb664ed94906f0622243dc1924adb3b Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Thu, 3 Nov 2022 20:02:37 -0400 Subject: [PATCH 04/30] added movedTo support in db --- src/BirdsiteLive.ActivityPub/Models/Actor.cs | 1 + src/BirdsiteLive.Domain/UserService.cs | 1 + .../Processors/RetrieveTweetsProcessor.cs | 4 +- .../Processors/SaveProgressionProcessor.cs | 2 +- .../DbInitializerPostgresDal.cs | 13 ++++++- .../TwitterUserPostgresDal.cs | 38 +++++++++++++------ .../Contracts/ITwitterUserDal.cs | 5 ++- .../Models/SyncTwitterUser.cs | 5 ++- .../TwitterUserPostgresDalTests.cs | 16 ++++---- .../ProcessFollowUserTests.cs | 4 +- .../RetrieveTweetsProcessorTests.cs | 4 +- .../SaveProgressionProcessorTests.cs | 16 ++++++-- 12 files changed, 77 insertions(+), 32 deletions(-) diff --git a/src/BirdsiteLive.ActivityPub/Models/Actor.cs b/src/BirdsiteLive.ActivityPub/Models/Actor.cs index 713ea89..ea4f8a3 100644 --- a/src/BirdsiteLive.ActivityPub/Models/Actor.cs +++ b/src/BirdsiteLive.ActivityPub/Models/Actor.cs @@ -17,6 +17,7 @@ namespace BirdsiteLive.ActivityPub public string name { get; set; } public string summary { get; set; } public string url { get; set; } + public string movedTo { get; set; } public bool manuallyApprovesFollowers { get; set; } public string inbox { get; set; } public bool? discoverable { get; set; } = true; diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index 42cf6d3..f5ae123 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -90,6 +90,7 @@ namespace BirdsiteLive.Domain summary = description, url = actorUrl, manuallyApprovesFollowers = twitterUser.Protected, + discoverable = false, publicKey = new PublicKey() { id = $"{actorUrl}#main-key", diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs index 58d35d0..e6608d7 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs @@ -49,12 +49,12 @@ namespace BirdsiteLive.Pipeline.Processors { var tweetId = tweets.Last().Id; var now = DateTime.UtcNow; - await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now); + await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now, user.MovedTo, user.MovedToAcct); } else { var now = DateTime.UtcNow; - await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now); + await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now, user.MovedTo, user.MovedToAcct); } } diff --git a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs index f262f39..109f751 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs @@ -48,7 +48,7 @@ namespace BirdsiteLive.Pipeline.Processors var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max(); var minimumSync = followingSyncStatuses.Min(); var now = DateTime.UtcNow; - await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now); + await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now, userWithTweetsToSync.User.MovedTo, userWithTweetsToSync.User.MovedToAcct); } catch (Exception e) { diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs index 2f9cb54..a66af72 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs @@ -23,7 +23,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public class DbInitializerPostgresDal : PostgresBase, IDbInitializerDal { private readonly PostgresTools _tools; - private readonly Version _currentVersion = new Version(2, 4); + private readonly Version _currentVersion = new Version(2, 5); private const string DbVersionType = "db-version"; #region Ctor @@ -135,7 +135,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers new Tuple(new Version(2,0), new Version(2,1)), new Tuple(new Version(2,1), new Version(2,2)), new Tuple(new Version(2,2), new Version(2,3)), - new Tuple(new Version(2,3), new Version(2,4)) + new Tuple(new Version(2,3), new Version(2,4)), + new Tuple(new Version(2,4), new Version(2,5)) }; } @@ -172,6 +173,14 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers var alterPostingError = $@"ALTER TABLE {_settings.FollowersTableName} ALTER COLUMN postingErrorCount TYPE INTEGER"; await _tools.ExecuteRequestAsync(alterPostingError); } + else if (from == new Version(2, 4) && to == new Version(2, 5)) + { + var addMovedTo = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD movedTo VARCHAR(2048)"; + await _tools.ExecuteRequestAsync(addMovedTo); + + var addMovedToAcct = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD movedToAcct VARCHAR(305)"; + await _tools.ExecuteRequestAsync(addMovedToAcct); + } else { throw new NotImplementedException(); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs index 11214d4..a649c7b 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs @@ -18,7 +18,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } #endregion - public async Task CreateTwitterUserAsync(string acct, long lastTweetPostedId) + public async Task CreateTwitterUserAsync(string acct, long lastTweetPostedId, string movedTo = null, string movedToAcct = null) { acct = acct.ToLowerInvariant(); @@ -27,8 +27,15 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers dbConnection.Open(); await dbConnection.ExecuteAsync( - $"INSERT INTO {_settings.TwitterUserTableName} (acct,lastTweetPostedId,lastTweetSynchronizedForAllFollowersId) VALUES(@acct,@lastTweetPostedId,@lastTweetSynchronizedForAllFollowersId)", - new { acct, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = lastTweetPostedId }); + $"INSERT INTO {_settings.TwitterUserTableName} (acct,lastTweetPostedId,lastTweetSynchronizedForAllFollowersId, movedTo, movedToAcct) VALUES(@acct,@lastTweetPostedId,@lastTweetSynchronizedForAllFollowersId,@movedTo,@movedToAcct)", + new + { + acct, + lastTweetPostedId, + lastTweetSynchronizedForAllFollowersId = lastTweetPostedId, + movedTo, + movedToAcct + }); } } @@ -62,7 +69,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public async Task GetTwitterUsersCountAsync() { - var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName}"; + var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE"; using (var dbConnection = Connection) { @@ -75,7 +82,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public async Task GetFailingTwitterUsersCountAsync() { - var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE fetchingErrorCount > 0"; + var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE fetchingErrorCount > 0 AND (movedTo = '') IS NOT FALSE"; using (var dbConnection = Connection) { @@ -88,7 +95,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public async Task GetAllTwitterUsersAsync(int maxNumber) { - var query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber"; + var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber"; using (var dbConnection = Connection) { @@ -101,7 +108,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public async Task GetAllTwitterUsersAsync() { - var query = $"SELECT * FROM {_settings.TwitterUserTableName}"; + var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE"; using (var dbConnection = Connection) { @@ -112,26 +119,35 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } } - public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync) + public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync, string movedTo, string movedToAcct) { if(id == default) throw new ArgumentException("id"); if(lastTweetPostedId == default) throw new ArgumentException("lastTweetPostedId"); if(lastTweetSynchronizedForAllFollowersId == default) throw new ArgumentException("lastTweetSynchronizedForAllFollowersId"); if(lastSync == default) throw new ArgumentException("lastSync"); - var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId, fetchingErrorCount = @fetchingErrorCount, lastSync = @lastSync WHERE id = @id"; + var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId, fetchingErrorCount = @fetchingErrorCount, lastSync = @lastSync, movedTo = @movedTo, movedToAcct = @movedToAcct WHERE id = @id"; using (var dbConnection = Connection) { dbConnection.Open(); - await dbConnection.QueryAsync(query, new { id, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId, fetchingErrorCount, lastSync = lastSync.ToUniversalTime() }); + await dbConnection.QueryAsync(query, new + { + id, + lastTweetPostedId, + lastTweetSynchronizedForAllFollowersId, + fetchingErrorCount, + lastSync = lastSync.ToUniversalTime(), + movedTo, + movedToAcct + }); } } public async Task UpdateTwitterUserAsync(SyncTwitterUser user) { - await UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, user.LastSync); + await UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, user.LastSync, user.MovedTo, user.MovedToAcct); } public async Task DeleteTwitterUserAsync(string acct) diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs index ef2cc36..a29463b 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs @@ -6,12 +6,13 @@ namespace BirdsiteLive.DAL.Contracts { public interface ITwitterUserDal { - Task CreateTwitterUserAsync(string acct, long lastTweetPostedId); + Task CreateTwitterUserAsync(string acct, long lastTweetPostedId, string movedTo = null, + string movedToAcct = null); Task GetTwitterUserAsync(string acct); Task GetTwitterUserAsync(int id); Task GetAllTwitterUsersAsync(int maxNumber); Task GetAllTwitterUsersAsync(); - Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync); + Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync, string movedTo, string movedToAcct); Task UpdateTwitterUserAsync(SyncTwitterUser user); Task DeleteTwitterUserAsync(string acct); Task DeleteTwitterUserAsync(int id); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs index 8b18ba1..224d981 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs @@ -12,6 +12,9 @@ namespace BirdsiteLive.DAL.Models public DateTime LastSync { get; set; } - public int FetchingErrorCount { get; set; } //TODO: update DAL + public int FetchingErrorCount { get; set; } + + public string MovedTo { get; set; } + public string MovedToAcct { get; set; } } } \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs index c9bc746..c6932ce 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs @@ -87,7 +87,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers var updatedLastSyncId = 1550L; var now = DateTime.Now; var errors = 15; - await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now); + await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now, null, null); result = await dal.GetTwitterUserAsync(acct); @@ -96,6 +96,8 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId); Assert.AreEqual(errors, result.FetchingErrorCount); Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100); + Assert.AreEqual(null, result.MovedTo); + Assert.AreEqual(null, result.MovedToAcct); } [TestMethod] @@ -167,7 +169,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task Update_NoId() { var dal = new TwitterUserPostgresDal(_settings); - await dal.UpdateTwitterUserAsync(default, default, default, default, DateTime.UtcNow); + await dal.UpdateTwitterUserAsync(default, default, default, default, DateTime.UtcNow, null, null); } [TestMethod] @@ -175,7 +177,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task Update_NoLastTweetPostedId() { var dal = new TwitterUserPostgresDal(_settings); - await dal.UpdateTwitterUserAsync(12, default, default, default, DateTime.UtcNow); + await dal.UpdateTwitterUserAsync(12, default, default, default, DateTime.UtcNow, null, null); } [TestMethod] @@ -183,7 +185,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task Update_NoLastTweetSynchronizedForAllFollowersId() { var dal = new TwitterUserPostgresDal(_settings); - await dal.UpdateTwitterUserAsync(12, 9556, default, default, DateTime.UtcNow); + await dal.UpdateTwitterUserAsync(12, 9556, default, default, DateTime.UtcNow, null, null); } [TestMethod] @@ -191,7 +193,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task Update_NoLastSync() { var dal = new TwitterUserPostgresDal(_settings); - await dal.UpdateTwitterUserAsync(12, 9556, 65, default, default); + await dal.UpdateTwitterUserAsync(12, 9556, 65, default, default, null, null); } [TestMethod] @@ -318,7 +320,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers { var user = allUsers[i]; var date = i % 2 == 0 ? oldest : newest; - await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date); + await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date, null, null); } var result = await dal.GetAllTwitterUsersAsync(10); @@ -382,7 +384,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers if (i == 0 || i == 2 || i == 3) { var t = await dal.GetTwitterUserAsync(acct); - await dal.UpdateTwitterUserAsync(t.Id ,1L,2L, 50+i*2, DateTime.Now); + await dal.UpdateTwitterUserAsync(t.Id ,1L,2L, 50+i*2, DateTime.Now, null, null); } } diff --git a/src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessFollowUserTests.cs b/src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessFollowUserTests.cs index 0fb03ae..8f2f393 100644 --- a/src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessFollowUserTests.cs +++ b/src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessFollowUserTests.cs @@ -77,7 +77,9 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases twitterUserDalMock .Setup(x => x.CreateTwitterUserAsync( It.Is(y => y == twitterName), - It.Is(y => y == -1))) + It.Is(y => y == -1), + It.Is(y => y == null), + It.Is(y => y == null))) .Returns(Task.CompletedTask); #endregion diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs index 17a3aa2..b82ca5e 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs @@ -64,7 +64,9 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == tweets.Last().Id), It.Is(y => y == tweets.Last().Id), It.Is(y => y == 0), - It.IsAny() + It.IsAny(), + It.Is(y => y == null), + It.Is(y => y == null) )) .Returns(Task.CompletedTask); diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs index feffbff..23a11ec 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs @@ -66,7 +66,9 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == tweet2.Id), It.Is(y => y == tweet2.Id), It.Is(y => y == 0), - It.IsAny() + It.IsAny(), + It.Is(y => y == null), + It.Is(y => y == null) )) .Returns(Task.CompletedTask); @@ -133,7 +135,9 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == tweet2.Id), It.Is(y => y == tweet2.Id), It.Is(y => y == 0), - It.IsAny() + It.IsAny(), + It.Is(y => y == null), + It.Is(y => y == null) )) .Throws(new ArgumentException()); @@ -202,7 +206,9 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == tweet3.Id), It.Is(y => y == tweet2.Id), It.Is(y => y == 0), - It.IsAny() + It.IsAny(), + It.Is(y => y == null), + It.Is(y => y == null) )) .Returns(Task.CompletedTask); @@ -281,7 +287,9 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == tweet3.Id), It.Is(y => y == tweet2.Id), It.Is(y => y == 0), - It.IsAny() + It.IsAny(), + It.Is(y => y == null), + It.Is(y => y == null) )) .Returns(Task.CompletedTask); From 498134f21563aa9c04bc318ee82268e604bbac18 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Thu, 3 Nov 2022 20:18:45 -0400 Subject: [PATCH 05/30] added migrated tests --- .../TwitterUserPostgresDalTests.cs | 71 ++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs index c6932ce..ef28eb7 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs @@ -71,6 +71,28 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers Assert.AreEqual(result.Id, resultById.Id); } + [TestMethod] + public async Task CreateAndGetMigratedUser_byId() + { + var acct = "myid"; + var lastTweetId = 1548L; + var movedTo = "https://"; + var movedToAcct = "@account@instance"; + + var dal = new TwitterUserPostgresDal(_settings); + + await dal.CreateTwitterUserAsync(acct, lastTweetId, movedTo, movedToAcct); + var result = await dal.GetTwitterUserAsync(acct); + var resultById = await dal.GetTwitterUserAsync(result.Id); + + Assert.AreEqual(acct, resultById.Acct); + Assert.AreEqual(lastTweetId, resultById.LastTweetPostedId); + Assert.AreEqual(lastTweetId, resultById.LastTweetSynchronizedForAllFollowersId); + Assert.AreEqual(result.Id, resultById.Id); + Assert.AreEqual(movedTo, result.MovedTo); + Assert.AreEqual(movedToAcct, result.MovedToAcct); + } + [TestMethod] public async Task CreateUpdateAndGetUser() { @@ -100,6 +122,37 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers Assert.AreEqual(null, result.MovedToAcct); } + [TestMethod] + public async Task CreateUpdateAndGetMigratedUser() + { + var acct = "myid"; + var lastTweetId = 1548L; + + var dal = new TwitterUserPostgresDal(_settings); + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + var result = await dal.GetTwitterUserAsync(acct); + + + var updatedLastTweetId = 1600L; + var updatedLastSyncId = 1550L; + var now = DateTime.Now; + var errors = 15; + var movedTo = "https://"; + var movedToAcct = "@account@instance"; + await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now, movedTo, movedToAcct); + + result = await dal.GetTwitterUserAsync(acct); + + Assert.AreEqual(acct, result.Acct); + Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId); + Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId); + Assert.AreEqual(errors, result.FetchingErrorCount); + Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100); + Assert.AreEqual(movedTo, result.MovedTo); + Assert.AreEqual(movedToAcct, result.MovedToAcct); + } + [TestMethod] public async Task CreateUpdate2AndGetUser() { @@ -258,7 +311,15 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers await dal.CreateTwitterUserAsync(acct, lastTweetId); } - var result = await dal.GetAllTwitterUsersAsync(1000); + for (int i = 0; i < 10; i++) + { + var acct = $"migrated-myid{i}"; + var lastTweetId = 1548L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain"); + } + + var result = await dal.GetAllTwitterUsersAsync(1100); Assert.AreEqual(1000, result.Length); Assert.IsFalse(result[0].Id == default); Assert.IsFalse(result[0].Acct == default); @@ -346,6 +407,14 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers await dal.CreateTwitterUserAsync(acct, lastTweetId); } + for (int i = 0; i < 10; i++) + { + var acct = $"migrated-myid{i}"; + var lastTweetId = 1548L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain"); + } + var result = await dal.GetAllTwitterUsersAsync(); Assert.AreEqual(1000, result.Length); Assert.IsFalse(result[0].Id == default); From df68b9c37020e7c542ce494e4585d10aeeda7299 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 4 Nov 2022 02:07:50 -0400 Subject: [PATCH 06/30] added migration logic --- src/BirdsiteLive.Domain/MigrationService.cs | 99 +++++++++++++++++-- .../Extractors/TweetExtractor.cs | 3 +- .../Models/ExtractedTweet.cs | 1 + .../Controllers/MigrationController.cs | 20 ++-- 4 files changed, 104 insertions(+), 19 deletions(-) diff --git a/src/BirdsiteLive.Domain/MigrationService.cs b/src/BirdsiteLive.Domain/MigrationService.cs index d8818dc..60f6b84 100644 --- a/src/BirdsiteLive.Domain/MigrationService.cs +++ b/src/BirdsiteLive.Domain/MigrationService.cs @@ -4,19 +4,30 @@ using BirdsiteLive.Twitter; using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; +using BirdsiteLive.ActivityPub; +using BirdsiteLive.ActivityPub.Models; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.ActivityPub.Converters; +using BirdsiteLive.Common.Settings; namespace BirdsiteLive.Domain { public class MigrationService { + private readonly InstanceSettings _instanceSettings; private readonly ITwitterTweetsService _twitterTweetsService; private readonly IActivityPubService _activityPubService; + private readonly ITwitterUserDal _twitterUserDal; + private readonly IFollowersDal _followersDal; #region Ctor - public MigrationService(ITwitterTweetsService twitterTweetsService, IActivityPubService activityPubService) + public MigrationService(ITwitterTweetsService twitterTweetsService, IActivityPubService activityPubService, ITwitterUserDal twitterUserDal, IFollowersDal followersDal, InstanceSettings instanceSettings) { _twitterTweetsService = twitterTweetsService; _activityPubService = activityPubService; + _twitterUserDal = twitterUserDal; + _followersDal = followersDal; + _instanceSettings = instanceSettings; } #endregion @@ -33,8 +44,14 @@ namespace BirdsiteLive.Domain var castedTweetId = ExtractedTweetId(tweetId); var tweet = _twitterTweetsService.GetTweet(castedTweetId); - if (tweet == null) throw new Exception("Tweet not found"); - if (!tweet.MessageContent.Contains(code)) throw new Exception("Tweet don't have migration code"); + if (tweet == null) + throw new Exception("Tweet not found"); + + if (tweet.CreatorName.Trim().ToLowerInvariant() != acct.Trim().ToLowerInvariant()) + throw new Exception($"Tweet not published by @{acct}"); + + if (!tweet.MessageContent.Contains(code)) + throw new Exception("Tweet don't have migration code"); return true; } @@ -52,25 +69,81 @@ namespace BirdsiteLive.Domain throw new ArgumentException("Unvalid Tweet ID"); } - public async Task ValidateFediverseAcctAsync(string fediverseAcct) + public async Task ValidateFediverseAcctAsync(string fediverseAcct) { if (string.IsNullOrWhiteSpace(fediverseAcct)) throw new ArgumentException("Please provide Fediverse account"); - if( !fediverseAcct.Contains('@') || fediverseAcct.Trim('@').Split('@').Length != 2) + if( !fediverseAcct.Contains('@') || !fediverseAcct.StartsWith("@") || fediverseAcct.Trim('@').Split('@').Length != 2) throw new ArgumentException("Please provide valid Fediverse handle"); var objectId = await _activityPubService.GetUserIdAsync(fediverseAcct); var user = await _activityPubService.GetUser(objectId); - if(user != null) return true; + var result = new ValidatedFediverseUser + { + FediverseAcct = fediverseAcct, + ObjectId = objectId, + User = user, + IsValid = user != null + }; - return false; + return result; } - public async Task MigrateAccountAsync(string acct, string tweetId, string fediverseAcct, bool triggerRemoteMigration) + public async Task MigrateAccountAsync(ValidatedFediverseUser validatedUser, string acct) { - throw new NotImplementedException("Migration not implemented"); + // Apply moved to + var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct); + twitterAccount.MovedTo = validatedUser.ObjectId; + twitterAccount.MovedToAcct = validatedUser.FediverseAcct; + await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); + + // Notify Followers + var t = Task.Run(async () => + { + var followers = await _followersDal.GetFollowersAsync(twitterAccount.Id); + foreach (var follower in followers) + { + try + { + var noteId = Guid.NewGuid().ToString(); + var actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, acct); + var noteUrl = UrlFactory.GetNoteUrl(_instanceSettings.Domain, acct, noteId); + + var to = validatedUser.ObjectId; + var cc = new string[0]; + + var note = new Note + { + id = noteId, + + published = DateTime.UtcNow.ToString("s") + "Z", + url = noteUrl, + attributedTo = actorUrl, + + to = new[] { to }, + cc = cc, + + content = $@"

[MIRROR SERVICE NOTIFICATION]
+ This bot has been disabled by it's original owner.
+ It has been redirected to {validatedUser.FediverseAcct}. +

" + }; + + await _activityPubService.PostNewNoteActivity(note, acct, Guid.NewGuid().ToString(), follower.Host, follower.InboxRoute); + } + catch (Exception e) + { + Console.WriteLine(e); + } + } + }); + } + + public async Task TriggerRemoteMigrationAsync(string id, string tweetid, string handle) + { + //TODO } private byte[] GetHash(string inputString) @@ -88,4 +161,12 @@ namespace BirdsiteLive.Domain return sb.ToString(); } } + + public class ValidatedFediverseUser + { + public string FediverseAcct { get; set; } + public string ObjectId { get; set; } + public Actor User { get; set; } + public bool IsValid { get; set; } + } } \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs index 75ae645..f8d29c8 100644 --- a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs +++ b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs @@ -28,7 +28,8 @@ namespace BirdsiteLive.Twitter.Extractors IsReply = tweet.InReplyToUserId != null, IsThread = tweet.InReplyToUserId != null && tweet.InReplyToUserId == tweet.CreatedBy.Id, IsRetweet = tweet.IsRetweet || tweet.QuotedStatusId != null, - RetweetUrl = ExtractRetweetUrl(tweet) + RetweetUrl = ExtractRetweetUrl(tweet), + CreatorName = tweet.CreatedBy.Name }; return extractedTweet; diff --git a/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs b/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs index f7f4e59..89a2e23 100644 --- a/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs +++ b/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs @@ -15,5 +15,6 @@ namespace BirdsiteLive.Twitter.Models public bool IsThread { get; set; } public bool IsRetweet { get; set; } public string RetweetUrl { get; set; } + public string CreatorName { get; set; } } } \ No newline at end of file diff --git a/src/BirdsiteLive/Controllers/MigrationController.cs b/src/BirdsiteLive/Controllers/MigrationController.cs index d0e121f..463f5fa 100644 --- a/src/BirdsiteLive/Controllers/MigrationController.cs +++ b/src/BirdsiteLive/Controllers/MigrationController.cs @@ -51,25 +51,27 @@ namespace BirdsiteLive.Controllers FediverseAccount = handle }; + ValidatedFediverseUser fediverseUserValidation = null; + try { - var isAcctValid = await _migrationService.ValidateFediverseAcctAsync(handle); + fediverseUserValidation = await _migrationService.ValidateFediverseAcctAsync(handle); var isTweetValid = _migrationService.ValidateTweet(id, tweetid); - data.IsAcctValid = isAcctValid; + data.IsAcctValid = fediverseUserValidation.IsValid; data.IsTweetValid = isTweetValid; } catch (Exception e) { data.ErrorMessage = e.Message; } - - - if (data.IsAcctValid && data.IsTweetValid) + + if (data.IsAcctValid && data.IsTweetValid && fediverseUserValidation != null) { try { - await _migrationService.MigrateAccountAsync(id, tweetid, handle, true); + await _migrationService.MigrateAccountAsync(fediverseUserValidation, id); + await _migrationService.TriggerRemoteMigrationAsync(id, tweetid, handle); data.MigrationSuccess = true; } catch (Exception e) @@ -86,12 +88,12 @@ namespace BirdsiteLive.Controllers [Route("/migration/{id}/{tweetid}/{handle}")] public async Task RemoteMigrate(string id, string tweetid, string handle) { - var isAcctValid = await _migrationService.ValidateFediverseAcctAsync(handle); + var fediverseUserValidation = await _migrationService.ValidateFediverseAcctAsync(handle); var isTweetValid = _migrationService.ValidateTweet(id, tweetid); - if (isAcctValid && isTweetValid) + if (fediverseUserValidation.IsValid && isTweetValid) { - await _migrationService.MigrateAccountAsync(id, tweetid, handle, false); + await _migrationService.MigrateAccountAsync(fediverseUserValidation, id); return Ok(); } From 4fb04c16b8851abfb25bdab9d52d26d8392b42f5 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 4 Nov 2022 03:14:00 -0400 Subject: [PATCH 07/30] added better blacklisting handling --- src/BirdsiteLive.Domain/MigrationService.cs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/BirdsiteLive.Domain/MigrationService.cs b/src/BirdsiteLive.Domain/MigrationService.cs index 60f6b84..08177f3 100644 --- a/src/BirdsiteLive.Domain/MigrationService.cs +++ b/src/BirdsiteLive.Domain/MigrationService.cs @@ -95,10 +95,17 @@ namespace BirdsiteLive.Domain { // Apply moved to var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct); - twitterAccount.MovedTo = validatedUser.ObjectId; - twitterAccount.MovedToAcct = validatedUser.FediverseAcct; - await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); - + if (twitterAccount == null) + { + await _twitterUserDal.CreateTwitterUserAsync(acct, -1, validatedUser.ObjectId, validatedUser.FediverseAcct); + } + else + { + twitterAccount.MovedTo = validatedUser.ObjectId; + twitterAccount.MovedToAcct = validatedUser.FediverseAcct; + await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); + } + // Notify Followers var t = Task.Run(async () => { From 6f8a2c03730e019f548b753bcbac01949e15ca53 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Tue, 13 Dec 2022 22:55:22 -0500 Subject: [PATCH 08/30] added deletion workflow --- .../BirdsiteLive.Domain.csproj | 4 + .../Enum/MigrationTypeEnum.cs | 9 ++ src/BirdsiteLive.Domain/MigrationService.cs | 81 ++++++++++++++---- .../Processors/RetrieveTweetsProcessor.cs | 4 +- .../Processors/SaveProgressionProcessor.cs | 2 +- .../Controllers/MigrationController.cs | 84 +++++++++++++++++-- .../Views/Migration/Delete.cshtml | 44 ++++++++++ src/BirdsiteLive/Views/Migration/Index.cshtml | 19 +++-- src/BirdsiteLive/Views/Users/Index.cshtml | 2 +- .../TwitterUserPostgresDal.cs | 9 +- .../Contracts/ITwitterUserDal.cs | 2 +- .../Models/SyncTwitterUser.cs | 2 + .../TwitterUserPostgresDalTests.cs | 16 ++-- .../RetrieveTweetsProcessorTests.cs | 3 +- .../SaveProgressionProcessorTests.cs | 12 ++- 15 files changed, 239 insertions(+), 54 deletions(-) create mode 100644 src/BirdsiteLive.Domain/Enum/MigrationTypeEnum.cs create mode 100644 src/BirdsiteLive/Views/Migration/Delete.cshtml diff --git a/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj b/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj index 8c601b4..7bc9873 100644 --- a/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj +++ b/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj @@ -15,4 +15,8 @@ + + + + diff --git a/src/BirdsiteLive.Domain/Enum/MigrationTypeEnum.cs b/src/BirdsiteLive.Domain/Enum/MigrationTypeEnum.cs new file mode 100644 index 0000000..92b1444 --- /dev/null +++ b/src/BirdsiteLive.Domain/Enum/MigrationTypeEnum.cs @@ -0,0 +1,9 @@ +namespace BirdsiteLive.Domain.Enum +{ + public enum MigrationTypeEnum + { + Unknown = 0, + Migration = 1, + Deletion = 2 + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/MigrationService.cs b/src/BirdsiteLive.Domain/MigrationService.cs index 08177f3..f534f3c 100644 --- a/src/BirdsiteLive.Domain/MigrationService.cs +++ b/src/BirdsiteLive.Domain/MigrationService.cs @@ -9,6 +9,8 @@ using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.ActivityPub.Converters; using BirdsiteLive.Common.Settings; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Domain.Enum; namespace BirdsiteLive.Domain { @@ -37,9 +39,21 @@ namespace BirdsiteLive.Domain return $"[[BirdsiteLIVE-MigrationCode|{hash.Substring(0, 10)}]]"; } - public bool ValidateTweet(string acct, string tweetId) + public string GetDeletionCode(string acct) { - var code = GetMigrationCode(acct); + var hash = GetHashString(acct); + return $"[[BirdsiteLIVE-DeletionCode|{hash.Substring(0, 10)}]]"; + } + + public bool ValidateTweet(string acct, string tweetId, MigrationTypeEnum type) + { + string code; + if (type == MigrationTypeEnum.Migration) + code = GetMigrationCode(acct); + else if (type == MigrationTypeEnum.Deletion) + code = GetDeletionCode(acct); + else + throw new NotImplementedException(); var castedTweetId = ExtractedTweetId(tweetId); var tweet = _twitterTweetsService.GetTweet(castedTweetId); @@ -47,10 +61,10 @@ namespace BirdsiteLive.Domain if (tweet == null) throw new Exception("Tweet not found"); - if (tweet.CreatorName.Trim().ToLowerInvariant() != acct.Trim().ToLowerInvariant()) + if (tweet.CreatorName.Trim().ToLowerInvariant() != acct.Trim().ToLowerInvariant()) throw new Exception($"Tweet not published by @{acct}"); - if (!tweet.MessageContent.Contains(code)) + if (!tweet.MessageContent.Contains(code)) throw new Exception("Tweet don't have migration code"); return true; @@ -74,7 +88,7 @@ namespace BirdsiteLive.Domain if (string.IsNullOrWhiteSpace(fediverseAcct)) throw new ArgumentException("Please provide Fediverse account"); - if( !fediverseAcct.Contains('@') || !fediverseAcct.StartsWith("@") || fediverseAcct.Trim('@').Split('@').Length != 2) + if (!fediverseAcct.Contains('@') || !fediverseAcct.StartsWith("@") || fediverseAcct.Trim('@').Split('@').Length != 2) throw new ArgumentException("Please provide valid Fediverse handle"); var objectId = await _activityPubService.GetUserIdAsync(fediverseAcct); @@ -98,15 +112,23 @@ namespace BirdsiteLive.Domain if (twitterAccount == null) { await _twitterUserDal.CreateTwitterUserAsync(acct, -1, validatedUser.ObjectId, validatedUser.FediverseAcct); - } - else - { - twitterAccount.MovedTo = validatedUser.ObjectId; - twitterAccount.MovedToAcct = validatedUser.FediverseAcct; - await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); + twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct); } + twitterAccount.MovedTo = validatedUser.ObjectId; + twitterAccount.MovedToAcct = validatedUser.FediverseAcct; + await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); + // Notify Followers + var message = $@"

[BSL MIRROR SERVICE NOTIFICATION]
+ This bot has been disabled by it's original owner.
+ It has been redirected to {validatedUser.FediverseAcct}. +

"; + NotifyFollowers(acct, twitterAccount, message); + } + + private void NotifyFollowers(string acct, SyncTwitterUser twitterAccount, string message) + { var t = Task.Run(async () => { var followers = await _followersDal.GetFollowersAsync(twitterAccount.Id); @@ -118,7 +140,8 @@ namespace BirdsiteLive.Domain var actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, acct); var noteUrl = UrlFactory.GetNoteUrl(_instanceSettings.Domain, acct, noteId); - var to = validatedUser.ObjectId; + //var to = validatedUser.ObjectId; + var to = follower.ActorId; var cc = new string[0]; var note = new Note @@ -132,13 +155,11 @@ namespace BirdsiteLive.Domain to = new[] { to }, cc = cc, - content = $@"

[MIRROR SERVICE NOTIFICATION]
- This bot has been disabled by it's original owner.
- It has been redirected to {validatedUser.FediverseAcct}. -

" + content = message }; - await _activityPubService.PostNewNoteActivity(note, acct, Guid.NewGuid().ToString(), follower.Host, follower.InboxRoute); + await _activityPubService.PostNewNoteActivity(note, acct, Guid.NewGuid().ToString(), follower.Host, + follower.InboxRoute); } catch (Exception e) { @@ -148,11 +169,37 @@ namespace BirdsiteLive.Domain }); } + public async Task DeleteAccountAsync(string acct) + { + // Apply moved to + var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct); + if (twitterAccount == null) + { + await _twitterUserDal.CreateTwitterUserAsync(acct, -1); + twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct); + } + + twitterAccount.Deleted = true; + await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); + + + // Notify Followers + var message = $@"

[BSL MIRROR SERVICE NOTIFICATION]
+ This bot has been deleted by it's original owner.
+

"; + NotifyFollowers(acct, twitterAccount, message); + } + public async Task TriggerRemoteMigrationAsync(string id, string tweetid, string handle) { //TODO } + public async Task TriggerRemoteDeleteAsync(string id, string tweetid) + { + //TODO + } + private byte[] GetHash(string inputString) { using (HashAlgorithm algorithm = SHA256.Create()) diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs index e6608d7..321fbf0 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs @@ -49,12 +49,12 @@ namespace BirdsiteLive.Pipeline.Processors { var tweetId = tweets.Last().Id; var now = DateTime.UtcNow; - await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now, user.MovedTo, user.MovedToAcct); + await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now, user.MovedTo, user.MovedToAcct, user.Deleted); } else { var now = DateTime.UtcNow; - await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now, user.MovedTo, user.MovedToAcct); + await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now, user.MovedTo, user.MovedToAcct, user.Deleted); } } diff --git a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs index 109f751..1f94871 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs @@ -48,7 +48,7 @@ namespace BirdsiteLive.Pipeline.Processors var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max(); var minimumSync = followingSyncStatuses.Min(); var now = DateTime.UtcNow; - await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now, userWithTweetsToSync.User.MovedTo, userWithTweetsToSync.User.MovedToAcct); + await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now, userWithTweetsToSync.User.MovedTo, userWithTweetsToSync.User.MovedToAcct, userWithTweetsToSync.User.Deleted); } catch (Exception e) { diff --git a/src/BirdsiteLive/Controllers/MigrationController.cs b/src/BirdsiteLive/Controllers/MigrationController.cs index 463f5fa..8593cb2 100644 --- a/src/BirdsiteLive/Controllers/MigrationController.cs +++ b/src/BirdsiteLive/Controllers/MigrationController.cs @@ -5,6 +5,7 @@ using System.Text; using System.Threading.Tasks; using Npgsql.TypeHandlers; using BirdsiteLive.Domain; +using BirdsiteLive.Domain.Enum; namespace BirdsiteLive.Controllers { @@ -20,8 +21,8 @@ namespace BirdsiteLive.Controllers #endregion [HttpGet] - [Route("/migration/{id}")] - public IActionResult Index(string id) + [Route("/migration/move/{id}")] + public IActionResult IndexMove(string id) { var migrationCode = _migrationService.GetMigrationCode(id); var data = new MigrationData() @@ -30,12 +31,26 @@ namespace BirdsiteLive.Controllers MigrationCode = migrationCode }; - return View(data); + return View("Index", data); + } + + [HttpGet] + [Route("/migration/delete/{id}")] + public IActionResult IndexDelete(string id) + { + var migrationCode = _migrationService.GetDeletionCode(id); + var data = new MigrationData() + { + Acct = id, + MigrationCode = migrationCode + }; + + return View("Delete", data); } [HttpPost] - [Route("/migration/{id}")] - public async Task Migrate(string id, string tweetid, string handle) + [Route("/migration/move/{id}")] + public async Task MigrateMove(string id, string tweetid, string handle) { var migrationCode = _migrationService.GetMigrationCode(id); @@ -56,7 +71,7 @@ namespace BirdsiteLive.Controllers try { fediverseUserValidation = await _migrationService.ValidateFediverseAcctAsync(handle); - var isTweetValid = _migrationService.ValidateTweet(id, tweetid); + var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Migration); data.IsAcctValid = fediverseUserValidation.IsValid; data.IsTweetValid = isTweetValid; @@ -85,11 +100,55 @@ namespace BirdsiteLive.Controllers } [HttpPost] - [Route("/migration/{id}/{tweetid}/{handle}")] - public async Task RemoteMigrate(string id, string tweetid, string handle) + [Route("/migration/delete/{id}")] + public async Task MigrateDelete(string id, string tweetid) + { + var migrationCode = _migrationService.GetMigrationCode(id); + + var data = new MigrationData() + { + Acct = id, + MigrationCode = migrationCode, + + IsTweetProvided = !string.IsNullOrWhiteSpace(tweetid), + + TweetId = tweetid + }; + + try + { + var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Migration); + data.IsTweetValid = isTweetValid; + } + catch (Exception e) + { + data.ErrorMessage = e.Message; + } + + if (data.IsTweetValid) + { + try + { + await _migrationService.DeleteAccountAsync(id); + await _migrationService.TriggerRemoteDeleteAsync(id, tweetid); + data.MigrationSuccess = true; + } + catch (Exception e) + { + Console.WriteLine(e); + data.ErrorMessage = e.Message; + } + } + + return View("Index", data); + } + + [HttpPost] + [Route("/migration/move/{id}/{tweetid}/{handle}")] + public async Task RemoteMigrateMove(string id, string tweetid, string handle) { var fediverseUserValidation = await _migrationService.ValidateFediverseAcctAsync(handle); - var isTweetValid = _migrationService.ValidateTweet(id, tweetid); + var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Deletion); if (fediverseUserValidation.IsValid && isTweetValid) { @@ -99,6 +158,13 @@ namespace BirdsiteLive.Controllers return StatusCode(500); } + + [HttpPost] + [Route("/migration/delete/{id}/{tweetid}/{handle}")] + public async Task RemoteDeleteMove(string id, string tweetid, string handle) + { + throw new NotImplementedException(); + } } diff --git a/src/BirdsiteLive/Views/Migration/Delete.cshtml b/src/BirdsiteLive/Views/Migration/Delete.cshtml new file mode 100644 index 0000000..05f9875 --- /dev/null +++ b/src/BirdsiteLive/Views/Migration/Delete.cshtml @@ -0,0 +1,44 @@ +@model BirdsiteLive.Controllers.MigrationData +@{ + ViewData["Title"] = "Migration"; +} + +
+ @if (!string.IsNullOrWhiteSpace(ViewData.Model.ErrorMessage)) + { + + } + +

Delete @@@ViewData.Model.Acct mirror

+ + @if (!ViewData.Model.IsTweetProvided) + { +

What is needed?

+ +

You'll need access to the Twitter account to provide proof of ownership.

+ +

What will deletion do?

+ +

+ Deletion will remove all followers, delete the account and will be blacklisted so that it can't be recreated.
+

+ } + +

Start the deletion!

+ +

Please copy and post this string in a public Tweet (the string must be untampered, but you can write anything you want before or after it):

+ + +
+ +

Provide deletion information:

+ +
+ + +
+ + +
\ No newline at end of file diff --git a/src/BirdsiteLive/Views/Migration/Index.cshtml b/src/BirdsiteLive/Views/Migration/Index.cshtml index 422b8f7..dcd84e1 100644 --- a/src/BirdsiteLive/Views/Migration/Index.cshtml +++ b/src/BirdsiteLive/Views/Migration/Index.cshtml @@ -11,7 +11,7 @@
} -

Migrate @@@ViewData.Model.Acct

+

Migrate @@@ViewData.Model.Acct mirror to my Fediverse account

@if (!ViewData.Model.IsAcctProvided && !ViewData.Model.IsTweetProvided) { @@ -21,16 +21,17 @@

What will migration do?

-

Migration will transfer the followers to the provided account.
- After a week, the account will be deleted. It will also be blacklisted so that it can't be recreated.

+

+ Migration will notify followers of the migration of the mirror account to your fediverse account and will be disabled after that.
+

}

Start the migration!

-

Please copy and post this string in a Tweet (the string must be untampered, but you can write anything you want before or after it):

+

Please copy and post this string in a public Tweet (the string must be untampered, but you can write anything you want before or after it):

- -
+ +

Provide migration information:

@@ -49,4 +50,10 @@
+
+
+
+
+ I don't have a fediverse account and I'd like to delete this mirror. +
\ No newline at end of file diff --git a/src/BirdsiteLive/Views/Users/Index.cshtml b/src/BirdsiteLive/Views/Users/Index.cshtml index 8177d6c..96ae0d4 100644 --- a/src/BirdsiteLive/Views/Users/Index.cshtml +++ b/src/BirdsiteLive/Views/Users/Index.cshtml @@ -47,6 +47,6 @@ }
- I'm the owner of this account and I'm now on the Fediverse + I'm the owner of this account and I would like to take control of this mirror.
\ No newline at end of file diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs index a649c7b..92dd41f 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs @@ -119,14 +119,14 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } } - public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync, string movedTo, string movedToAcct) + public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync, string movedTo, string movedToAcct, bool deleted) { if(id == default) throw new ArgumentException("id"); if(lastTweetPostedId == default) throw new ArgumentException("lastTweetPostedId"); if(lastTweetSynchronizedForAllFollowersId == default) throw new ArgumentException("lastTweetSynchronizedForAllFollowersId"); if(lastSync == default) throw new ArgumentException("lastSync"); - var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId, fetchingErrorCount = @fetchingErrorCount, lastSync = @lastSync, movedTo = @movedTo, movedToAcct = @movedToAcct WHERE id = @id"; + var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId, fetchingErrorCount = @fetchingErrorCount, lastSync = @lastSync, movedTo = @movedTo, movedToAcct = @movedToAcct, deleted = @deleted WHERE id = @id"; using (var dbConnection = Connection) { @@ -140,14 +140,15 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers fetchingErrorCount, lastSync = lastSync.ToUniversalTime(), movedTo, - movedToAcct + movedToAcct, + deleted }); } } public async Task UpdateTwitterUserAsync(SyncTwitterUser user) { - await UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, user.LastSync, user.MovedTo, user.MovedToAcct); + await UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, user.LastSync, user.MovedTo, user.MovedToAcct, user.Deleted); } public async Task DeleteTwitterUserAsync(string acct) diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs index a29463b..46a59e4 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs @@ -12,7 +12,7 @@ namespace BirdsiteLive.DAL.Contracts Task GetTwitterUserAsync(int id); Task GetAllTwitterUsersAsync(int maxNumber); Task GetAllTwitterUsersAsync(); - Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync, string movedTo, string movedToAcct); + Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync, string movedTo, string movedToAcct, bool deleted); Task UpdateTwitterUserAsync(SyncTwitterUser user); Task DeleteTwitterUserAsync(string acct); Task DeleteTwitterUserAsync(int id); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs index 224d981..3841476 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs @@ -16,5 +16,7 @@ namespace BirdsiteLive.DAL.Models public string MovedTo { get; set; } public string MovedToAcct { get; set; } + + public bool Deleted { get; set; } //TODO: update DAL } } \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs index ef28eb7..9dc1b4a 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs @@ -109,7 +109,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers var updatedLastSyncId = 1550L; var now = DateTime.Now; var errors = 15; - await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now, null, null); + await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now, null, null, false); result = await dal.GetTwitterUserAsync(acct); @@ -140,7 +140,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers var errors = 15; var movedTo = "https://"; var movedToAcct = "@account@instance"; - await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now, movedTo, movedToAcct); + await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now, movedTo, movedToAcct, false); result = await dal.GetTwitterUserAsync(acct); @@ -222,7 +222,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task Update_NoId() { var dal = new TwitterUserPostgresDal(_settings); - await dal.UpdateTwitterUserAsync(default, default, default, default, DateTime.UtcNow, null, null); + await dal.UpdateTwitterUserAsync(default, default, default, default, DateTime.UtcNow, null, null, false); } [TestMethod] @@ -230,7 +230,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task Update_NoLastTweetPostedId() { var dal = new TwitterUserPostgresDal(_settings); - await dal.UpdateTwitterUserAsync(12, default, default, default, DateTime.UtcNow, null, null); + await dal.UpdateTwitterUserAsync(12, default, default, default, DateTime.UtcNow, null, null, false); } [TestMethod] @@ -238,7 +238,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task Update_NoLastTweetSynchronizedForAllFollowersId() { var dal = new TwitterUserPostgresDal(_settings); - await dal.UpdateTwitterUserAsync(12, 9556, default, default, DateTime.UtcNow, null, null); + await dal.UpdateTwitterUserAsync(12, 9556, default, default, DateTime.UtcNow, null, null, false); } [TestMethod] @@ -246,7 +246,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task Update_NoLastSync() { var dal = new TwitterUserPostgresDal(_settings); - await dal.UpdateTwitterUserAsync(12, 9556, 65, default, default, null, null); + await dal.UpdateTwitterUserAsync(12, 9556, 65, default, default, null, null, false); } [TestMethod] @@ -381,7 +381,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers { var user = allUsers[i]; var date = i % 2 == 0 ? oldest : newest; - await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date, null, null); + await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date, null, null, false); } var result = await dal.GetAllTwitterUsersAsync(10); @@ -453,7 +453,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers if (i == 0 || i == 2 || i == 3) { var t = await dal.GetTwitterUserAsync(acct); - await dal.UpdateTwitterUserAsync(t.Id ,1L,2L, 50+i*2, DateTime.Now, null, null); + await dal.UpdateTwitterUserAsync(t.Id ,1L,2L, 50+i*2, DateTime.Now, null, null, false); } } diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs index b82ca5e..f95ad82 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs @@ -66,7 +66,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == 0), It.IsAny(), It.Is(y => y == null), - It.Is(y => y == null) + It.Is(y => y == null), + It.Is(y => y == false) )) .Returns(Task.CompletedTask); diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs index 23a11ec..d245713 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs @@ -68,7 +68,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == 0), It.IsAny(), It.Is(y => y == null), - It.Is(y => y == null) + It.Is(y => y == null), + It.Is(y => y == false) )) .Returns(Task.CompletedTask); @@ -137,7 +138,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == 0), It.IsAny(), It.Is(y => y == null), - It.Is(y => y == null) + It.Is(y => y == null), + It.Is(y => y == false) )) .Throws(new ArgumentException()); @@ -208,7 +210,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == 0), It.IsAny(), It.Is(y => y == null), - It.Is(y => y == null) + It.Is(y => y == null), + It.Is(y => y == false) )) .Returns(Task.CompletedTask); @@ -289,7 +292,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == 0), It.IsAny(), It.Is(y => y == null), - It.Is(y => y == null) + It.Is(y => y == null), + It.Is(y => y == false) )) .Returns(Task.CompletedTask); From 4157f613ea5fef3326ecca6096537b750acbf00f Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Tue, 13 Dec 2022 23:02:28 -0500 Subject: [PATCH 09/30] added db migration + test --- .../DbInitializerPostgresDal.cs | 3 ++ .../TwitterUserPostgresDalTests.cs | 29 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs index a66af72..55b38f6 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs @@ -180,6 +180,9 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers var addMovedToAcct = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD movedToAcct VARCHAR(305)"; await _tools.ExecuteRequestAsync(addMovedToAcct); + + var addDeletedToAcct = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD deleted BOOLEAN"; + await _tools.ExecuteRequestAsync(addDeletedToAcct); } else { diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs index 9dc1b4a..3149cc2 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs @@ -153,6 +153,35 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers Assert.AreEqual(movedToAcct, result.MovedToAcct); } + [TestMethod] + public async Task CreateUpdateAndGetDeletedUser() + { + var acct = "myid"; + var lastTweetId = 1548L; + + var dal = new TwitterUserPostgresDal(_settings); + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + var result = await dal.GetTwitterUserAsync(acct); + + var updatedLastTweetId = 1600L; + var updatedLastSyncId = 1550L; + var now = DateTime.Now; + var errors = 15; + await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now, null, null, true); + + result = await dal.GetTwitterUserAsync(acct); + + Assert.AreEqual(acct, result.Acct); + Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId); + Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId); + Assert.AreEqual(errors, result.FetchingErrorCount); + Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100); + Assert.AreEqual(null, result.MovedTo); + Assert.AreEqual(null, result.MovedToAcct); + Assert.AreEqual(true, result.Deleted); + } + [TestMethod] public async Task CreateUpdate2AndGetUser() { From 2e5bb28ff85fb297e6fc89687955389446982c7c Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Tue, 13 Dec 2022 23:42:22 -0500 Subject: [PATCH 10/30] updated mirror account data --- src/BirdsiteLive.Domain/UserService.cs | 16 ++++++++++---- .../Controllers/UsersController.cs | 18 +++++++++++---- src/BirdsiteLive/Models/DisplayTwitterUser.cs | 4 ++++ src/BirdsiteLive/Views/Users/Index.cshtml | 22 ++++++++++++++++--- 4 files changed, 49 insertions(+), 11 deletions(-) diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index f5ae123..d225888 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -11,6 +11,7 @@ using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.Common.Regexes; using BirdsiteLive.Common.Settings; using BirdsiteLive.Cryptography; +using BirdsiteLive.DAL.Models; using BirdsiteLive.Domain.BusinessUseCases; using BirdsiteLive.Domain.Repository; using BirdsiteLive.Domain.Statistics; @@ -24,7 +25,7 @@ namespace BirdsiteLive.Domain { public interface IUserService { - Actor GetUser(TwitterUser twitterUser); + Actor GetUser(TwitterUser twitterUser, SyncTwitterUser dbTwitterUser); Task FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityFollow activity, string body); Task UndoFollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityUndoFollow activity, string body); @@ -64,7 +65,7 @@ namespace BirdsiteLive.Domain } #endregion - public Actor GetUser(TwitterUser twitterUser) + public Actor GetUser(TwitterUser twitterUser, SyncTwitterUser dbTwitterUser) { var actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, twitterUser.Acct); var acct = twitterUser.Acct.ToLowerInvariant(); @@ -112,7 +113,7 @@ namespace BirdsiteLive.Domain new UserAttachment { type = "PropertyValue", - name = "Official", + name = "Official Account", value = $"https://twitter.com/{acct}" }, new UserAttachment @@ -120,12 +121,19 @@ namespace BirdsiteLive.Domain type = "PropertyValue", name = "Disclaimer", value = "This is an automatically created and managed mirror profile from Twitter. While it reflects exactly the content of the original account, it doesn't provide support for interactions and replies. It is an equivalent view from other 3rd party Twitter client apps and uses the same technical means to provide it." + }, + new UserAttachment + { + type = "PropertyValue", + name = "Take control of the account", + value = $"https://{_instanceSettings.Domain}" } }, endpoints = new EndPoints { sharedInbox = $"https://{_instanceSettings.Domain}/inbox" - } + }, + movedTo = dbTwitterUser?.MovedTo }; return user; } diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index 24a9eb5..0097bf9 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -11,6 +11,8 @@ using BirdsiteLive.ActivityPub; using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.Common.Regexes; using BirdsiteLive.Common.Settings; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; using BirdsiteLive.Domain; using BirdsiteLive.Models; using BirdsiteLive.Tools; @@ -28,13 +30,14 @@ namespace BirdsiteLive.Controllers { private readonly ITwitterUserService _twitterUserService; private readonly ITwitterTweetsService _twitterTweetService; + private readonly ITwitterUserDal _twitterUserDal; private readonly IUserService _userService; private readonly IStatusService _statusService; private readonly InstanceSettings _instanceSettings; private readonly ILogger _logger; #region Ctor - public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ITwitterTweetsService twitterTweetService, ILogger logger) + public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ITwitterTweetsService twitterTweetService, ILogger logger, ITwitterUserDal twitterUserDal) { _twitterUserService = twitterUserService; _userService = userService; @@ -42,6 +45,7 @@ namespace BirdsiteLive.Controllers _instanceSettings = instanceSettings; _twitterTweetService = twitterTweetService; _logger = logger; + _twitterUserDal = twitterUserDal; } #endregion @@ -60,7 +64,7 @@ namespace BirdsiteLive.Controllers [Route("/@{id}")] [Route("/users/{id}")] [Route("/users/{id}/remote_follow")] - public IActionResult Index(string id) + public async Task Index(string id) { _logger.LogTrace("User Index: {Id}", id); @@ -102,6 +106,7 @@ namespace BirdsiteLive.Controllers } //var isSaturated = _twitterUserService.IsUserApiRateLimited(); + var dbUser = await _twitterUserDal.GetTwitterUserAsync(id); var acceptHeaders = Request.Headers["Accept"]; if (acceptHeaders.Any()) @@ -111,7 +116,8 @@ namespace BirdsiteLive.Controllers { if (isSaturated) return new ObjectResult("Too Many Requests") { StatusCode = 429 }; if (notFound) return NotFound(); - var apUser = _userService.GetUser(user); + if (dbUser != null && dbUser.Deleted) return NotFound(); + var apUser = _userService.GetUser(user, dbUser); var jsonApUser = JsonConvert.SerializeObject(apUser); return Content(jsonApUser, "application/activity+json; charset=utf-8"); } @@ -128,8 +134,12 @@ namespace BirdsiteLive.Controllers Url = user.Url, ProfileImageUrl = user.ProfileImageUrl, Protected = user.Protected, + + InstanceHandle = $"@{user.Acct.ToLowerInvariant()}@{_instanceSettings.Domain}", - InstanceHandle = $"@{user.Acct.ToLowerInvariant()}@{_instanceSettings.Domain}" + MovedTo = dbUser?.MovedTo, + MovedToAcct = dbUser?.MovedToAcct, + Deleted = dbUser?.Deleted ?? false, }; return View(displayableUser); } diff --git a/src/BirdsiteLive/Models/DisplayTwitterUser.cs b/src/BirdsiteLive/Models/DisplayTwitterUser.cs index 3a93875..0b17174 100644 --- a/src/BirdsiteLive/Models/DisplayTwitterUser.cs +++ b/src/BirdsiteLive/Models/DisplayTwitterUser.cs @@ -10,5 +10,9 @@ public bool Protected { get; set; } public string InstanceHandle { get; set; } + + public string MovedTo { get; set; } + public string MovedToAcct { get; set; } + public bool Deleted { get; set; } } } \ No newline at end of file diff --git a/src/BirdsiteLive/Views/Users/Index.cshtml b/src/BirdsiteLive/Views/Users/Index.cshtml index 96ae0d4..48d626d 100644 --- a/src/BirdsiteLive/Views/Users/Index.cshtml +++ b/src/BirdsiteLive/Views/Users/Index.cshtml @@ -37,6 +37,19 @@ This account is protected, BirdsiteLIVE cannot fetch their tweets and will not provide follow support until it is unprotected again. } + else if (ViewData.Model.Deleted) + { + + } + else if (!string.IsNullOrEmpty(ViewData.Model.MovedTo)) + { + + } else {
@@ -46,7 +59,10 @@
} -
- I'm the owner of this account and I would like to take control of this mirror. -
+ @if (!ViewData.Model.Deleted && string.IsNullOrEmpty(ViewData.Model.MovedTo)) + { +
+ I'm the owner of this account and I would like to take control of this mirror. +
+ } \ No newline at end of file From 9a6971c6bcbde5bb8d93fb0dd89cfde770376fcf Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Tue, 13 Dec 2022 23:48:14 -0500 Subject: [PATCH 11/30] typo --- src/BirdsiteLive/Views/Users/Index.cshtml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BirdsiteLive/Views/Users/Index.cshtml b/src/BirdsiteLive/Views/Users/Index.cshtml index 48d626d..9f6f33d 100644 --- a/src/BirdsiteLive/Views/Users/Index.cshtml +++ b/src/BirdsiteLive/Views/Users/Index.cshtml @@ -40,13 +40,13 @@ else if (ViewData.Model.Deleted) { } else if (!string.IsNullOrEmpty(ViewData.Model.MovedTo)) { } From 8840d1007c46014173416fc0f89ba4fca24d20d0 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 14 Dec 2022 00:17:00 -0500 Subject: [PATCH 12/30] fix notification --- src/BirdsiteLive.Domain/MigrationService.cs | 18 +++++++---- .../Controllers/MigrationController.cs | 32 ++++++++++++------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/src/BirdsiteLive.Domain/MigrationService.cs b/src/BirdsiteLive.Domain/MigrationService.cs index f534f3c..bc1c73e 100644 --- a/src/BirdsiteLive.Domain/MigrationService.cs +++ b/src/BirdsiteLive.Domain/MigrationService.cs @@ -105,7 +105,7 @@ namespace BirdsiteLive.Domain return result; } - public async Task MigrateAccountAsync(ValidatedFediverseUser validatedUser, string acct) + public async Task MigrateAccountAsync(ValidatedFediverseUser validatedUser, string acct, bool notify) { // Apply moved to var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct); @@ -120,11 +120,14 @@ namespace BirdsiteLive.Domain await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); // Notify Followers - var message = $@"

[BSL MIRROR SERVICE NOTIFICATION]
+ if (notify) + { + var message = $@"

[BSL MIRROR SERVICE NOTIFICATION]
This bot has been disabled by it's original owner.
It has been redirected to {validatedUser.FediverseAcct}.

"; - NotifyFollowers(acct, twitterAccount, message); + NotifyFollowers(acct, twitterAccount, message); + } } private void NotifyFollowers(string acct, SyncTwitterUser twitterAccount, string message) @@ -169,7 +172,7 @@ namespace BirdsiteLive.Domain }); } - public async Task DeleteAccountAsync(string acct) + public async Task DeleteAccountAsync(string acct, bool notify) { // Apply moved to var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct); @@ -184,10 +187,13 @@ namespace BirdsiteLive.Domain // Notify Followers - var message = $@"

[BSL MIRROR SERVICE NOTIFICATION]
+ if (notify) + { + var message = $@"

[BSL MIRROR SERVICE NOTIFICATION]
This bot has been deleted by it's original owner.

"; - NotifyFollowers(acct, twitterAccount, message); + NotifyFollowers(acct, twitterAccount, message); + } } public async Task TriggerRemoteMigrationAsync(string id, string tweetid, string handle) diff --git a/src/BirdsiteLive/Controllers/MigrationController.cs b/src/BirdsiteLive/Controllers/MigrationController.cs index 8593cb2..0c1d580 100644 --- a/src/BirdsiteLive/Controllers/MigrationController.cs +++ b/src/BirdsiteLive/Controllers/MigrationController.cs @@ -53,7 +53,7 @@ namespace BirdsiteLive.Controllers public async Task MigrateMove(string id, string tweetid, string handle) { var migrationCode = _migrationService.GetMigrationCode(id); - + var data = new MigrationData() { Acct = id, @@ -80,12 +80,12 @@ namespace BirdsiteLive.Controllers { data.ErrorMessage = e.Message; } - + if (data.IsAcctValid && data.IsTweetValid && fediverseUserValidation != null) { try { - await _migrationService.MigrateAccountAsync(fediverseUserValidation, id); + await _migrationService.MigrateAccountAsync(fediverseUserValidation, id, true); await _migrationService.TriggerRemoteMigrationAsync(id, tweetid, handle); data.MigrationSuccess = true; } @@ -129,7 +129,7 @@ namespace BirdsiteLive.Controllers { try { - await _migrationService.DeleteAccountAsync(id); + await _migrationService.DeleteAccountAsync(id, true); await _migrationService.TriggerRemoteDeleteAsync(id, tweetid); data.MigrationSuccess = true; } @@ -148,34 +148,42 @@ namespace BirdsiteLive.Controllers public async Task RemoteMigrateMove(string id, string tweetid, string handle) { var fediverseUserValidation = await _migrationService.ValidateFediverseAcctAsync(handle); - var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Deletion); + var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Migration); if (fediverseUserValidation.IsValid && isTweetValid) { - await _migrationService.MigrateAccountAsync(fediverseUserValidation, id); + await _migrationService.MigrateAccountAsync(fediverseUserValidation, id, false); return Ok(); } - return StatusCode(500); + return StatusCode(400); } [HttpPost] [Route("/migration/delete/{id}/{tweetid}/{handle}")] - public async Task RemoteDeleteMove(string id, string tweetid, string handle) + public async Task RemoteMigrateDelete(string id, string tweetid) { - throw new NotImplementedException(); + var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Deletion); + + if (isTweetValid) + { + await _migrationService.DeleteAccountAsync(id, true); + return Ok(); + } + + return StatusCode(400); } } - + public class MigrationData { public string Acct { get; set; } - + public string FediverseAccount { get; set; } public string TweetId { get; set; } - + public string MigrationCode { get; set; } public bool IsTweetProvided { get; set; } From 1a939b6147eeaa6147bdcee45c8a555058a05f6c Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 14 Dec 2022 00:35:25 -0500 Subject: [PATCH 13/30] return gone on deleted state --- src/BirdsiteLive/Controllers/UsersController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index 0097bf9..0750dae 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -116,7 +116,7 @@ namespace BirdsiteLive.Controllers { if (isSaturated) return new ObjectResult("Too Many Requests") { StatusCode = 429 }; if (notFound) return NotFound(); - if (dbUser != null && dbUser.Deleted) return NotFound(); + if (dbUser != null && dbUser.Deleted) return new ObjectResult("Gone") { StatusCode = 410 }; var apUser = _userService.GetUser(user, dbUser); var jsonApUser = JsonConvert.SerializeObject(apUser); return Content(jsonApUser, "application/activity+json; charset=utf-8"); From 7658438741586a636d033999fbf95d8e680100f1 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Tue, 20 Dec 2022 18:47:21 -0500 Subject: [PATCH 14/30] always notify --- src/BirdsiteLive.Domain/MigrationService.cs | 21 +++++++------------ .../Controllers/MigrationController.cs | 8 +++---- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/src/BirdsiteLive.Domain/MigrationService.cs b/src/BirdsiteLive.Domain/MigrationService.cs index bc1c73e..fa19fa7 100644 --- a/src/BirdsiteLive.Domain/MigrationService.cs +++ b/src/BirdsiteLive.Domain/MigrationService.cs @@ -105,7 +105,7 @@ namespace BirdsiteLive.Domain return result; } - public async Task MigrateAccountAsync(ValidatedFediverseUser validatedUser, string acct, bool notify) + public async Task MigrateAccountAsync(ValidatedFediverseUser validatedUser, string acct) { // Apply moved to var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct); @@ -120,14 +120,11 @@ namespace BirdsiteLive.Domain await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); // Notify Followers - if (notify) - { - var message = $@"

[BSL MIRROR SERVICE NOTIFICATION]
+ var message = $@"

[BSL MIRROR SERVICE NOTIFICATION]
This bot has been disabled by it's original owner.
It has been redirected to {validatedUser.FediverseAcct}.

"; - NotifyFollowers(acct, twitterAccount, message); - } + NotifyFollowers(acct, twitterAccount, message); } private void NotifyFollowers(string acct, SyncTwitterUser twitterAccount, string message) @@ -172,7 +169,7 @@ namespace BirdsiteLive.Domain }); } - public async Task DeleteAccountAsync(string acct, bool notify) + public async Task DeleteAccountAsync(string acct) { // Apply moved to var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct); @@ -184,16 +181,12 @@ namespace BirdsiteLive.Domain twitterAccount.Deleted = true; await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); - - + // Notify Followers - if (notify) - { - var message = $@"

[BSL MIRROR SERVICE NOTIFICATION]
+ var message = $@"

[BSL MIRROR SERVICE NOTIFICATION]
This bot has been deleted by it's original owner.

"; - NotifyFollowers(acct, twitterAccount, message); - } + NotifyFollowers(acct, twitterAccount, message); } public async Task TriggerRemoteMigrationAsync(string id, string tweetid, string handle) diff --git a/src/BirdsiteLive/Controllers/MigrationController.cs b/src/BirdsiteLive/Controllers/MigrationController.cs index 0c1d580..00d5fb7 100644 --- a/src/BirdsiteLive/Controllers/MigrationController.cs +++ b/src/BirdsiteLive/Controllers/MigrationController.cs @@ -85,7 +85,7 @@ namespace BirdsiteLive.Controllers { try { - await _migrationService.MigrateAccountAsync(fediverseUserValidation, id, true); + await _migrationService.MigrateAccountAsync(fediverseUserValidation, id); await _migrationService.TriggerRemoteMigrationAsync(id, tweetid, handle); data.MigrationSuccess = true; } @@ -129,7 +129,7 @@ namespace BirdsiteLive.Controllers { try { - await _migrationService.DeleteAccountAsync(id, true); + await _migrationService.DeleteAccountAsync(id); await _migrationService.TriggerRemoteDeleteAsync(id, tweetid); data.MigrationSuccess = true; } @@ -152,7 +152,7 @@ namespace BirdsiteLive.Controllers if (fediverseUserValidation.IsValid && isTweetValid) { - await _migrationService.MigrateAccountAsync(fediverseUserValidation, id, false); + await _migrationService.MigrateAccountAsync(fediverseUserValidation, id); return Ok(); } @@ -167,7 +167,7 @@ namespace BirdsiteLive.Controllers if (isTweetValid) { - await _migrationService.DeleteAccountAsync(id, true); + await _migrationService.DeleteAccountAsync(id); return Ok(); } From d543a1d4f9a337a8413d0c24c130a85149476e37 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 21 Dec 2022 00:25:43 -0500 Subject: [PATCH 15/30] added delete action --- .../Models/ActivityDelete.cs | 1 + src/BirdsiteLive.Domain/ActivityPubService.cs | 26 +++++++++++++++++++ .../Controllers/DebugingController.cs | 11 ++++++++ src/BirdsiteLive/Views/Debuging/Index.cshtml | 6 +++++ 4 files changed, 44 insertions(+) diff --git a/src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs b/src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs index deb7e7f..c628a61 100644 --- a/src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs +++ b/src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs @@ -4,6 +4,7 @@ namespace BirdsiteLive.ActivityPub.Models { public class ActivityDelete : Activity { + public string[] to { get; set; } [JsonProperty("object")] public object apObject { get; set; } } diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs index 7a9aeeb..cab0c58 100644 --- a/src/BirdsiteLive.Domain/ActivityPubService.cs +++ b/src/BirdsiteLive.Domain/ActivityPubService.cs @@ -21,6 +21,7 @@ namespace BirdsiteLive.Domain Task PostDataAsync(T data, string targetHost, string actorUrl, string inbox = null); Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox); + Task DeleteUserAsync(string username, string targetHost, string targetInbox); } public class WebFinger @@ -82,6 +83,31 @@ namespace BirdsiteLive.Domain return actor; } + public async Task DeleteUserAsync(string username, string targetHost, string targetInbox) + { + try + { + var actor = UrlFactory.GetActorUrl(_instanceSettings.Domain, username); + + var deleteUser = new ActivityDelete + { + context = "https://www.w3.org/ns/activitystreams", + id = $"{actor}#delete", + type = "Delete", + actor = actor, + to = new [] { "https://www.w3.org/ns/activitystreams#Public" }, + apObject = actor + }; + + await PostDataAsync(deleteUser, targetHost, actor, targetInbox); + } + catch (Exception e) + { + _logger.LogError(e, "Error deleting {Username} to {Host}{Inbox}", username, targetHost, targetInbox); + throw; + } + } + public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox) { try diff --git a/src/BirdsiteLive/Controllers/DebugingController.cs b/src/BirdsiteLive/Controllers/DebugingController.cs index 00accef..7c2c259 100644 --- a/src/BirdsiteLive/Controllers/DebugingController.cs +++ b/src/BirdsiteLive/Controllers/DebugingController.cs @@ -125,6 +125,17 @@ namespace BirdsiteLive.Controllers await _userService.SendRejectFollowAsync(activityFollow, "mastodon.technology"); return View("Index"); } + + [HttpPost] + public async Task PostDeleteUser() + { + var userName = "mastodon"; + var host = "ioc.exchange"; + var inbox = "/inbox"; + + await _activityPubService.DeleteUserAsync(userName, host, inbox); + return View("Index"); + } } #endif diff --git a/src/BirdsiteLive/Views/Debuging/Index.cshtml b/src/BirdsiteLive/Views/Debuging/Index.cshtml index 5bcde75..e343cf2 100644 --- a/src/BirdsiteLive/Views/Debuging/Index.cshtml +++ b/src/BirdsiteLive/Views/Debuging/Index.cshtml @@ -23,4 +23,10 @@ + + +
+ + +
\ No newline at end of file From d219c59cfe337907408461942e62c137c1fbe3d5 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 21 Dec 2022 01:06:45 -0500 Subject: [PATCH 16/30] added delete event when deleting user --- src/BirdsiteLive.Domain/MigrationService.cs | 48 +++++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/src/BirdsiteLive.Domain/MigrationService.cs b/src/BirdsiteLive.Domain/MigrationService.cs index fa19fa7..e7e9ceb 100644 --- a/src/BirdsiteLive.Domain/MigrationService.cs +++ b/src/BirdsiteLive.Domain/MigrationService.cs @@ -121,7 +121,7 @@ namespace BirdsiteLive.Domain // Notify Followers var message = $@"

[BSL MIRROR SERVICE NOTIFICATION]
- This bot has been disabled by it's original owner.
+ This bot has been disabled by its original owner.
It has been redirected to {validatedUser.FediverseAcct}.

"; NotifyFollowers(acct, twitterAccount, message); @@ -171,7 +171,7 @@ namespace BirdsiteLive.Domain public async Task DeleteAccountAsync(string acct) { - // Apply moved to + // Apply deleted state var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct); if (twitterAccount == null) { @@ -181,12 +181,52 @@ namespace BirdsiteLive.Domain twitterAccount.Deleted = true; await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); - + // Notify Followers var message = $@"

[BSL MIRROR SERVICE NOTIFICATION]
- This bot has been deleted by it's original owner.
+ This bot has been deleted by its original owner.

"; NotifyFollowers(acct, twitterAccount, message); + + // Delete remote accounts + DeleteRemoteAccounts(acct); + } + + private void DeleteRemoteAccounts(string acct) + { + var t = Task.Run(async () => + { + var allUsers = await _followersDal.GetAllFollowersAsync(); + + var followersWtSharedInbox = allUsers + .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute)) + .GroupBy(x => x.Host) + .ToList(); + foreach (var followerGroup in followersWtSharedInbox) + { + var host = followerGroup.First().Host; + var sharedInbox = followerGroup.First().SharedInboxRoute; + + var t1 = Task.Run(async () => + { + await _activityPubService.DeleteUserAsync(acct, host, sharedInbox); + }); + } + + var followerWtInbox = allUsers + .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute)) + .ToList(); + foreach (var followerGroup in followerWtInbox) + { + var host = followerGroup.Host; + var sharedInbox = followerGroup.InboxRoute; + + var t1 = Task.Run(async () => + { + await _activityPubService.DeleteUserAsync(acct, host, sharedInbox); + }); + } + }); } public async Task TriggerRemoteMigrationAsync(string id, string tweetid, string handle) From 1c3da007fd3e52335267cdcd053b5c975f7d3bc5 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 24 Dec 2022 18:44:41 -0500 Subject: [PATCH 17/30] don't retrieve deleted users --- src/BirdsiteLive.Domain/MigrationService.cs | 2 + .../TwitterAccountModerationProcessor.cs | 2 +- .../RetrieveTwitterUsersProcessor.cs | 2 +- .../TwitterUserPostgresDal.cs | 14 ++-- .../Contracts/ITwitterUserDal.cs | 4 +- .../TwitterUserPostgresDalTests.cs | 71 +++++++++++++++++-- .../TwitterAccountModerationProcessorTests.cs | 8 +-- .../RetrieveTwitterUsersProcessorTests.cs | 15 ++-- 8 files changed, 93 insertions(+), 25 deletions(-) diff --git a/src/BirdsiteLive.Domain/MigrationService.cs b/src/BirdsiteLive.Domain/MigrationService.cs index e7e9ceb..844cedf 100644 --- a/src/BirdsiteLive.Domain/MigrationService.cs +++ b/src/BirdsiteLive.Domain/MigrationService.cs @@ -117,6 +117,7 @@ namespace BirdsiteLive.Domain twitterAccount.MovedTo = validatedUser.ObjectId; twitterAccount.MovedToAcct = validatedUser.FediverseAcct; + twitterAccount.LastSync = DateTime.UtcNow; await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); // Notify Followers @@ -180,6 +181,7 @@ namespace BirdsiteLive.Domain } twitterAccount.Deleted = true; + twitterAccount.LastSync = DateTime.UtcNow; await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); // Notify Followers diff --git a/src/BirdsiteLive.Moderation/Processors/TwitterAccountModerationProcessor.cs b/src/BirdsiteLive.Moderation/Processors/TwitterAccountModerationProcessor.cs index 91e3931..2f4d50e 100644 --- a/src/BirdsiteLive.Moderation/Processors/TwitterAccountModerationProcessor.cs +++ b/src/BirdsiteLive.Moderation/Processors/TwitterAccountModerationProcessor.cs @@ -29,7 +29,7 @@ namespace BirdsiteLive.Moderation.Processors { if (type == ModerationTypeEnum.None) return; - var twitterUsers = await _twitterUserDal.GetAllTwitterUsersAsync(); + var twitterUsers = await _twitterUserDal.GetAllTwitterUsersAsync(false); foreach (var user in twitterUsers) { diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs index 973b672..d9d0ffb 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs @@ -39,7 +39,7 @@ namespace BirdsiteLive.Pipeline.Processors try { var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync(); - var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber); + var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber, false); var userCount = users.Any() ? users.Length : 1; var splitNumber = (int) Math.Ceiling(userCount / 15d); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs index 92dd41f..d542a76 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs @@ -69,7 +69,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public async Task GetTwitterUsersCountAsync() { - var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE"; + var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE"; using (var dbConnection = Connection) { @@ -82,7 +82,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public async Task GetFailingTwitterUsersCountAsync() { - var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE fetchingErrorCount > 0 AND (movedTo = '') IS NOT FALSE"; + var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE fetchingErrorCount > 0 AND (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE"; using (var dbConnection = Connection) { @@ -93,9 +93,10 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } } - public async Task GetAllTwitterUsersAsync(int maxNumber) + public async Task GetAllTwitterUsersAsync(int maxNumber, bool retrieveDisabledUser) { - var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber"; + var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber"; + if (retrieveDisabledUser) query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber"; using (var dbConnection = Connection) { @@ -106,9 +107,10 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } } - public async Task GetAllTwitterUsersAsync() + public async Task GetAllTwitterUsersAsync(bool retrieveDisabledUser) { - var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE"; + var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE"; + if(retrieveDisabledUser) query = $"SELECT * FROM {_settings.TwitterUserTableName}"; using (var dbConnection = Connection) { diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs index 46a59e4..0c58881 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs @@ -10,8 +10,8 @@ namespace BirdsiteLive.DAL.Contracts string movedToAcct = null); Task GetTwitterUserAsync(string acct); Task GetTwitterUserAsync(int id); - Task GetAllTwitterUsersAsync(int maxNumber); - Task GetAllTwitterUsersAsync(); + Task GetAllTwitterUsersAsync(int maxNumber, bool retrieveDisabledUser); + Task GetAllTwitterUsersAsync(bool retrieveDisabledUser); Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync, string movedTo, string movedToAcct, bool deleted); Task UpdateTwitterUserAsync(SyncTwitterUser user); Task DeleteTwitterUserAsync(string acct); diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs index 3149cc2..936bb73 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs @@ -348,12 +348,71 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain"); } - var result = await dal.GetAllTwitterUsersAsync(1100); + for (int i = 0; i < 10; i++) + { + var acct = $"deleted-myid{i}"; + var lastTweetId = 148L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + var user = await dal.GetTwitterUserAsync(acct); + user.Deleted = true; + user.LastSync = DateTime.UtcNow; + await dal.UpdateTwitterUserAsync(user); + } + + var result = await dal.GetAllTwitterUsersAsync(1100, false); Assert.AreEqual(1000, result.Length); Assert.IsFalse(result[0].Id == default); Assert.IsFalse(result[0].Acct == default); Assert.IsFalse(result[0].LastTweetPostedId == default); Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default); + + foreach (var user in result) + { + Assert.IsTrue(string.IsNullOrWhiteSpace(user.MovedTo)); + Assert.IsTrue(string.IsNullOrWhiteSpace(user.MovedToAcct)); + Assert.IsFalse(user.Deleted); + } + } + + [TestMethod] + public async Task GetAllTwitterUsers_Top_RetrieveDeleted() + { + var dal = new TwitterUserPostgresDal(_settings); + for (var i = 0; i < 1000; i++) + { + var acct = $"myid{i}"; + var lastTweetId = 1548L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + } + + for (int i = 0; i < 10; i++) + { + var acct = $"migrated-myid{i}"; + var lastTweetId = 1548L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain"); + } + + for (int i = 0; i < 10; i++) + { + var acct = $"deleted-myid{i}"; + var lastTweetId = 148L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + var user = await dal.GetTwitterUserAsync(acct); + user.Deleted = true; + user.LastSync = DateTime.UtcNow; + await dal.UpdateTwitterUserAsync(user); + } + + var result = await dal.GetAllTwitterUsersAsync(1100, true); + Assert.AreEqual(1020, result.Length); + Assert.IsFalse(result[0].Id == default); + Assert.IsFalse(result[0].Acct == default); + Assert.IsFalse(result[0].LastTweetPostedId == default); + Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default); } [TestMethod] @@ -371,7 +430,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers // Update accounts var now = DateTime.UtcNow; - var allUsers = await dal.GetAllTwitterUsersAsync(); + var allUsers = await dal.GetAllTwitterUsersAsync(false); foreach (var acc in allUsers) { var lastSync = now.AddDays(acc.LastTweetPostedId); @@ -382,7 +441,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers // Create a not init account await dal.CreateTwitterUserAsync("not_init", -1); - var result = await dal.GetAllTwitterUsersAsync(10); + var result = await dal.GetAllTwitterUsersAsync(10, false); Assert.IsTrue(result.Any(x => x.Acct == "myid0")); Assert.IsTrue(result.Any(x => x.Acct == "myid8")); @@ -405,7 +464,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers await dal.CreateTwitterUserAsync(acct, lastTweetId); } - var allUsers = await dal.GetAllTwitterUsersAsync(100); + var allUsers = await dal.GetAllTwitterUsersAsync(100, false); for (var i = 0; i < 20; i++) { var user = allUsers[i]; @@ -413,7 +472,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date, null, null, false); } - var result = await dal.GetAllTwitterUsersAsync(10); + var result = await dal.GetAllTwitterUsersAsync(10, false); Assert.AreEqual(10, result.Length); Assert.IsFalse(result[0].Id == default); Assert.IsFalse(result[0].Acct == default); @@ -444,7 +503,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain"); } - var result = await dal.GetAllTwitterUsersAsync(); + var result = await dal.GetAllTwitterUsersAsync(false); Assert.AreEqual(1000, result.Length); Assert.IsFalse(result[0].Id == default); Assert.IsFalse(result[0].Acct == default); diff --git a/src/Tests/BirdsiteLive.Moderation.Tests/Processors/TwitterAccountModerationProcessorTests.cs b/src/Tests/BirdsiteLive.Moderation.Tests/Processors/TwitterAccountModerationProcessorTests.cs index 21d1288..8473424 100644 --- a/src/Tests/BirdsiteLive.Moderation.Tests/Processors/TwitterAccountModerationProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Moderation.Tests/Processors/TwitterAccountModerationProcessorTests.cs @@ -48,7 +48,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync(It.Is(y => y == false))) .ReturnsAsync(allUsers.ToArray()); var moderationRepositoryMock = new Mock(MockBehavior.Strict); @@ -87,7 +87,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync(It.Is(y => y == false))) .ReturnsAsync(allUsers.ToArray()); var moderationRepositoryMock = new Mock(MockBehavior.Strict); @@ -130,7 +130,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync(It.Is(y => y == false))) .ReturnsAsync(allUsers.ToArray()); var moderationRepositoryMock = new Mock(MockBehavior.Strict); @@ -173,7 +173,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync(It.Is(y => y == false))) .ReturnsAsync(allUsers.ToArray()); var moderationRepositoryMock = new Mock(MockBehavior.Strict); diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs index 4d0e465..daf0bfa 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs @@ -40,7 +40,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == maxUsers))) + It.Is(y => y == maxUsers), + It.Is(y => y == false))) .ReturnsAsync(users); var loggerMock = new Mock>(); @@ -83,7 +84,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .SetupSequence(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == maxUsers))) + It.Is(y => y == maxUsers), + It.Is(y => y == false))) .ReturnsAsync(users.ToArray()) .ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0]) @@ -130,7 +132,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .SetupSequence(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == maxUsers))) + It.Is(y => y == maxUsers), + It.Is(y => y == false))) .ReturnsAsync(users.ToArray()) .ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0]) @@ -178,7 +181,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == maxUsers))) + It.Is(y => y == maxUsers), + It.Is(y => y == false))) .ReturnsAsync(new SyncTwitterUser[0]); var loggerMock = new Mock>(); @@ -215,7 +219,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == maxUsers))) + It.Is(y => y == maxUsers), + It.Is(y => y == false))) .Returns(async () => await DelayFaultedTask(new Exception())); var loggerMock = new Mock>(); From ac297b815aaa8b4912b643ee17aeaf223e189c11 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sun, 25 Dec 2022 00:09:38 -0500 Subject: [PATCH 18/30] added account status check before migration/deletion --- .../Controllers/MigrationController.cs | 45 +++++++++++++++++-- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/src/BirdsiteLive/Controllers/MigrationController.cs b/src/BirdsiteLive/Controllers/MigrationController.cs index 00d5fb7..09e1337 100644 --- a/src/BirdsiteLive/Controllers/MigrationController.cs +++ b/src/BirdsiteLive/Controllers/MigrationController.cs @@ -6,17 +6,20 @@ using System.Threading.Tasks; using Npgsql.TypeHandlers; using BirdsiteLive.Domain; using BirdsiteLive.Domain.Enum; +using BirdsiteLive.DAL.Contracts; namespace BirdsiteLive.Controllers { public class MigrationController : Controller { private readonly MigrationService _migrationService; - + private readonly ITwitterUserDal _twitterUserDal; + #region Ctor - public MigrationController(MigrationService migrationService) + public MigrationController(MigrationService migrationService, ITwitterUserDal twitterUserDal) { _migrationService = migrationService; + _twitterUserDal = twitterUserDal; } #endregion @@ -53,7 +56,6 @@ namespace BirdsiteLive.Controllers public async Task MigrateMove(string id, string tweetid, string handle) { var migrationCode = _migrationService.GetMigrationCode(id); - var data = new MigrationData() { Acct = id, @@ -65,9 +67,22 @@ namespace BirdsiteLive.Controllers TweetId = tweetid, FediverseAccount = handle }; - ValidatedFediverseUser fediverseUserValidation = null; + //Verify can be migrated + var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id); + if (twitterAccount.Deleted) + { + data.ErrorMessage = "This account has been deleted, it can't be migrated"; + return View("Index", data); + } + if (!string.IsNullOrWhiteSpace(twitterAccount.MovedTo) || !string.IsNullOrWhiteSpace(twitterAccount.MovedToAcct)) + { + data.ErrorMessage = "This account has been moved already, it can't be migrated again"; + return View("Index", data); + } + + // Start migration try { fediverseUserValidation = await _migrationService.ValidateFediverseAcctAsync(handle); @@ -114,7 +129,16 @@ namespace BirdsiteLive.Controllers TweetId = tweetid }; + + //Verify can be deleted + var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id); + if (twitterAccount.Deleted) + { + data.ErrorMessage = "This account has been deleted, it can't be deleted again"; + return View("Index", data); + } + // Start deletion try { var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Migration); @@ -147,6 +171,14 @@ namespace BirdsiteLive.Controllers [Route("/migration/move/{id}/{tweetid}/{handle}")] public async Task RemoteMigrateMove(string id, string tweetid, string handle) { + //Verify can be migrated + var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id); + if (twitterAccount.Deleted + || !string.IsNullOrWhiteSpace(twitterAccount.MovedTo) + || !string.IsNullOrWhiteSpace(twitterAccount.MovedToAcct)) + return Ok(); + + // Start migration var fediverseUserValidation = await _migrationService.ValidateFediverseAcctAsync(handle); var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Migration); @@ -163,6 +195,11 @@ namespace BirdsiteLive.Controllers [Route("/migration/delete/{id}/{tweetid}/{handle}")] public async Task RemoteMigrateDelete(string id, string tweetid) { + //Verify can be deleted + var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id); + if (twitterAccount.Deleted) return Ok(); + + // Start deletion var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Deletion); if (isTweetValid) From d5a71bbaa60ed4a6fc2e9cf2427ed73c1a274ecc Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sun, 25 Dec 2022 00:10:37 -0500 Subject: [PATCH 19/30] fix route --- src/BirdsiteLive/Controllers/MigrationController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive/Controllers/MigrationController.cs b/src/BirdsiteLive/Controllers/MigrationController.cs index 09e1337..1455c07 100644 --- a/src/BirdsiteLive/Controllers/MigrationController.cs +++ b/src/BirdsiteLive/Controllers/MigrationController.cs @@ -192,7 +192,7 @@ namespace BirdsiteLive.Controllers } [HttpPost] - [Route("/migration/delete/{id}/{tweetid}/{handle}")] + [Route("/migration/delete/{id}/{tweetid}")] public async Task RemoteMigrateDelete(string id, string tweetid) { //Verify can be deleted From c910edc6b3fa1d744ccc56f5b553372a518427a8 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sun, 25 Dec 2022 18:15:54 -0500 Subject: [PATCH 20/30] various logic fixes --- src/BirdsiteLive.Domain/MigrationService.cs | 11 ++++++++++- .../Extractors/TweetExtractor.cs | 2 +- .../Controllers/MigrationController.cs | 18 ++++++++++-------- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/BirdsiteLive.Domain/MigrationService.cs b/src/BirdsiteLive.Domain/MigrationService.cs index 844cedf..b4df0a5 100644 --- a/src/BirdsiteLive.Domain/MigrationService.cs +++ b/src/BirdsiteLive.Domain/MigrationService.cs @@ -65,13 +65,22 @@ namespace BirdsiteLive.Domain throw new Exception($"Tweet not published by @{acct}"); if (!tweet.MessageContent.Contains(code)) - throw new Exception("Tweet don't have migration code"); + { + var message = "Tweet don't have migration code"; + if (type == MigrationTypeEnum.Deletion) + message = "Tweet don't have deletion code"; + + throw new Exception(message); + } return true; } private long ExtractedTweetId(string tweetId) { + if (string.IsNullOrWhiteSpace(tweetId)) + throw new ArgumentException("No provided Tweet ID"); + long castedId; if (long.TryParse(tweetId, out castedId)) return castedId; diff --git a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs index f8d29c8..e8e47fb 100644 --- a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs +++ b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs @@ -29,7 +29,7 @@ namespace BirdsiteLive.Twitter.Extractors IsThread = tweet.InReplyToUserId != null && tweet.InReplyToUserId == tweet.CreatedBy.Id, IsRetweet = tweet.IsRetweet || tweet.QuotedStatusId != null, RetweetUrl = ExtractRetweetUrl(tweet), - CreatorName = tweet.CreatedBy.Name + CreatorName = tweet.CreatedBy.UserIdentifier.ScreenName }; return extractedTweet; diff --git a/src/BirdsiteLive/Controllers/MigrationController.cs b/src/BirdsiteLive/Controllers/MigrationController.cs index 1455c07..386644a 100644 --- a/src/BirdsiteLive/Controllers/MigrationController.cs +++ b/src/BirdsiteLive/Controllers/MigrationController.cs @@ -71,12 +71,14 @@ namespace BirdsiteLive.Controllers //Verify can be migrated var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id); - if (twitterAccount.Deleted) + if (twitterAccount != null && twitterAccount.Deleted) { data.ErrorMessage = "This account has been deleted, it can't be migrated"; return View("Index", data); } - if (!string.IsNullOrWhiteSpace(twitterAccount.MovedTo) || !string.IsNullOrWhiteSpace(twitterAccount.MovedToAcct)) + if (twitterAccount != null && + (!string.IsNullOrWhiteSpace(twitterAccount.MovedTo) + || !string.IsNullOrWhiteSpace(twitterAccount.MovedToAcct))) { data.ErrorMessage = "This account has been moved already, it can't be migrated again"; return View("Index", data); @@ -118,12 +120,12 @@ namespace BirdsiteLive.Controllers [Route("/migration/delete/{id}")] public async Task MigrateDelete(string id, string tweetid) { - var migrationCode = _migrationService.GetMigrationCode(id); + var deletionCode = _migrationService.GetDeletionCode(id); var data = new MigrationData() { Acct = id, - MigrationCode = migrationCode, + MigrationCode = deletionCode, IsTweetProvided = !string.IsNullOrWhiteSpace(tweetid), @@ -132,16 +134,16 @@ namespace BirdsiteLive.Controllers //Verify can be deleted var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id); - if (twitterAccount.Deleted) + if (twitterAccount != null && twitterAccount.Deleted) { data.ErrorMessage = "This account has been deleted, it can't be deleted again"; - return View("Index", data); + return View("Delete", data); } // Start deletion try { - var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Migration); + var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Deletion); data.IsTweetValid = isTweetValid; } catch (Exception e) @@ -164,7 +166,7 @@ namespace BirdsiteLive.Controllers } } - return View("Index", data); + return View("Delete", data); } [HttpPost] From 2dd1cc738198c77b00b705a850ace24a0030574b Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sun, 25 Dec 2022 18:37:05 -0500 Subject: [PATCH 21/30] wording --- src/BirdsiteLive/Views/Users/Index.cshtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive/Views/Users/Index.cshtml b/src/BirdsiteLive/Views/Users/Index.cshtml index 9f6f33d..fce4b14 100644 --- a/src/BirdsiteLive/Views/Users/Index.cshtml +++ b/src/BirdsiteLive/Views/Users/Index.cshtml @@ -40,7 +40,7 @@ else if (ViewData.Model.Deleted) { } else if (!string.IsNullOrEmpty(ViewData.Model.MovedTo)) From 0eb0aa3c5d09bc997777ec314a324b8e91e119be Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sun, 25 Dec 2022 18:46:56 -0500 Subject: [PATCH 22/30] added migration message --- src/BirdsiteLive/Views/Migration/Delete.cshtml | 7 +++++++ src/BirdsiteLive/Views/Migration/Index.cshtml | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/src/BirdsiteLive/Views/Migration/Delete.cshtml b/src/BirdsiteLive/Views/Migration/Delete.cshtml index 05f9875..9df9f62 100644 --- a/src/BirdsiteLive/Views/Migration/Delete.cshtml +++ b/src/BirdsiteLive/Views/Migration/Delete.cshtml @@ -10,6 +10,13 @@ @ViewData.Model.ErrorMessage } + + @if (ViewData.Model.MigrationSuccess) + { + + }

Delete @@@ViewData.Model.Acct mirror

diff --git a/src/BirdsiteLive/Views/Migration/Index.cshtml b/src/BirdsiteLive/Views/Migration/Index.cshtml index dcd84e1..670a209 100644 --- a/src/BirdsiteLive/Views/Migration/Index.cshtml +++ b/src/BirdsiteLive/Views/Migration/Index.cshtml @@ -10,6 +10,13 @@ @ViewData.Model.ErrorMessage } + + @if (ViewData.Model.MigrationSuccess) + { + + }

Migrate @@@ViewData.Model.Acct mirror to my Fediverse account

From 27e735ca4d05800118b7c25ee0b3a38e7b58b4be Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sun, 25 Dec 2022 18:57:31 -0500 Subject: [PATCH 23/30] better redirection --- src/BirdsiteLive/Controllers/UsersController.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index 0750dae..e66dac5 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -60,10 +60,9 @@ namespace BirdsiteLive.Controllers } return View("UserNotFound"); } - + [Route("/@{id}")] [Route("/users/{id}")] - [Route("/users/{id}/remote_follow")] public async Task Index(string id) { _logger.LogTrace("User Index: {Id}", id); @@ -143,6 +142,12 @@ namespace BirdsiteLive.Controllers }; return View(displayableUser); } + + [Route("/users/{id}/remote_follow")] + public async Task IndexRemoteFollow(string id) + { + return Redirect($"/users/{id}"); + } [Route("/@{id}/{statusId}")] [Route("/users/{id}/statuses/{statusId}")] From ae42b109e99d5b72dde90cc0e7be1296cc80079c Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sun, 25 Dec 2022 19:20:07 -0500 Subject: [PATCH 24/30] fix profile --- src/BirdsiteLive.Domain/UserService.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index d225888..7f4c55e 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -88,7 +88,7 @@ namespace BirdsiteLive.Domain preferredUsername = acct, name = twitterUser.Name, inbox = $"{actorUrl}/inbox", - summary = description, + summary = "[THIS IS A TWITTER MIRRORED ACCOUNT]

" + description, url = actorUrl, manuallyApprovesFollowers = twitterUser.Protected, discoverable = false, @@ -126,7 +126,7 @@ namespace BirdsiteLive.Domain { type = "PropertyValue", name = "Take control of the account", - value = $"https://{_instanceSettings.Domain}" + value = $"MANAGE" } }, endpoints = new EndPoints From 7c267063f90f1468773fed3f01daa39f7746408e Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Tue, 27 Dec 2022 22:13:08 -0500 Subject: [PATCH 25/30] fix DM notification --- src/BirdsiteLive.Domain/MigrationService.cs | 29 ++++++++++++--------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/BirdsiteLive.Domain/MigrationService.cs b/src/BirdsiteLive.Domain/MigrationService.cs index b4df0a5..1d9f6ae 100644 --- a/src/BirdsiteLive.Domain/MigrationService.cs +++ b/src/BirdsiteLive.Domain/MigrationService.cs @@ -63,7 +63,7 @@ namespace BirdsiteLive.Domain if (tweet.CreatorName.Trim().ToLowerInvariant() != acct.Trim().ToLowerInvariant()) throw new Exception($"Tweet not published by @{acct}"); - + if (!tweet.MessageContent.Contains(code)) { var message = "Tweet don't have migration code"; @@ -130,10 +130,7 @@ namespace BirdsiteLive.Domain await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); // Notify Followers - var message = $@"

[BSL MIRROR SERVICE NOTIFICATION]
- This bot has been disabled by its original owner.
- It has been redirected to {validatedUser.FediverseAcct}. -

"; + var message = $@"

[BSL MIRROR SERVICE NOTIFICATION]
This bot has been disabled by its original owner.
It has been redirected to {validatedUser.FediverseAcct}.

"; NotifyFollowers(acct, twitterAccount, message); } @@ -156,7 +153,7 @@ namespace BirdsiteLive.Domain var note = new Note { - id = noteId, + id = noteUrl, published = DateTime.UtcNow.ToString("s") + "Z", url = noteUrl, @@ -165,11 +162,21 @@ namespace BirdsiteLive.Domain to = new[] { to }, cc = cc, - content = message + content = message, + tag = new Tag[]{ + new Tag() + { + type = "Mention", + href = follower.ActorId, + name = $"@{follower.Acct}@{follower.Host}" + } + }, }; - await _activityPubService.PostNewNoteActivity(note, acct, Guid.NewGuid().ToString(), follower.Host, - follower.InboxRoute); + if (!string.IsNullOrWhiteSpace(follower.SharedInboxRoute)) + await _activityPubService.PostNewNoteActivity(note, acct, noteId, follower.Host, follower.SharedInboxRoute); + else + await _activityPubService.PostNewNoteActivity(note, acct, noteId, follower.Host, follower.InboxRoute); } catch (Exception e) { @@ -194,9 +201,7 @@ namespace BirdsiteLive.Domain await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); // Notify Followers - var message = $@"

[BSL MIRROR SERVICE NOTIFICATION]
- This bot has been deleted by its original owner.
-

"; + var message = $@"

[BSL MIRROR SERVICE NOTIFICATION]
This bot has been deleted by its original owner.

"; NotifyFollowers(acct, twitterAccount, message); // Delete remote accounts From 52da17393becec952a25be14a3702d9425e9dede Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Tue, 27 Dec 2022 22:34:55 -0500 Subject: [PATCH 26/30] debug info --- .../Controllers/DebugingController.cs | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/BirdsiteLive/Controllers/DebugingController.cs b/src/BirdsiteLive/Controllers/DebugingController.cs index 7c2c259..8f37f22 100644 --- a/src/BirdsiteLive/Controllers/DebugingController.cs +++ b/src/BirdsiteLive/Controllers/DebugingController.cs @@ -59,17 +59,22 @@ namespace BirdsiteLive.Controllers [HttpPost] public async Task PostNote() { - var username = "gra"; + var username = "twitter"; var actor = $"https://{_instanceSettings.Domain}/users/{username}"; - var targetHost = "mastodon.technology"; - var target = $"{targetHost}/users/testtest"; - var inbox = $"/users/testtest/inbox"; + var targetHost = "ioc.exchange"; + var target = $"https://{targetHost}/users/test"; + //var inbox = $"/users/testtest/inbox"; + var inbox = $"/inbox"; var noteGuid = Guid.NewGuid(); var noteId = $"https://{_instanceSettings.Domain}/users/{username}/statuses/{noteGuid}"; var noteUrl = $"https://{_instanceSettings.Domain}/@{username}/{noteGuid}"; var to = $"{actor}/followers"; + to = target; + + var cc = new[] { "https://www.w3.org/ns/activitystreams#Public" }; + cc = new string[0]; var now = DateTime.UtcNow; var nowString = now.ToString("s") + "Z"; @@ -82,7 +87,7 @@ namespace BirdsiteLive.Controllers actor = actor, published = nowString, to = new[] { to }, - //cc = new [] { "https://www.w3.org/ns/activitystreams#Public" }, + cc = cc, apObject = new Note() { id = noteId, @@ -94,7 +99,8 @@ namespace BirdsiteLive.Controllers // Unlisted to = new[] { to }, - cc = new[] { "https://www.w3.org/ns/activitystreams#Public" }, + cc = cc, + //cc = new[] { "https://www.w3.org/ns/activitystreams#Public" }, //// Public //to = new[] { "https://www.w3.org/ns/activitystreams#Public" }, @@ -102,8 +108,16 @@ namespace BirdsiteLive.Controllers sensitive = false, content = "

TEST PUBLIC

", + //content = "

@test test

", attachment = new Attachment[0], - tag = new Tag[0] + tag = new Tag[]{ + new Tag() + { + type = "Mention", + href = target, + name = "@test@ioc.exchange" + } + }, } }; @@ -129,7 +143,7 @@ namespace BirdsiteLive.Controllers [HttpPost] public async Task PostDeleteUser() { - var userName = "mastodon"; + var userName = "twitter"; var host = "ioc.exchange"; var inbox = "/inbox"; From 0dfe1f4f9fa8873ea3211feb085ed4f58bed83e7 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Tue, 27 Dec 2022 22:35:11 -0500 Subject: [PATCH 27/30] fixed moved to --- src/BirdsiteLive.Domain/MigrationService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive.Domain/MigrationService.cs b/src/BirdsiteLive.Domain/MigrationService.cs index 1d9f6ae..975272e 100644 --- a/src/BirdsiteLive.Domain/MigrationService.cs +++ b/src/BirdsiteLive.Domain/MigrationService.cs @@ -124,7 +124,7 @@ namespace BirdsiteLive.Domain twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct); } - twitterAccount.MovedTo = validatedUser.ObjectId; + twitterAccount.MovedTo = validatedUser.User.id; twitterAccount.MovedToAcct = validatedUser.FediverseAcct; twitterAccount.LastSync = DateTime.UtcNow; await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); From e804b1929ceaaee3cb75693a94cabf9f9c6fc75f Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Tue, 27 Dec 2022 22:35:58 -0500 Subject: [PATCH 28/30] always show link --- src/BirdsiteLive/Views/Users/Index.cshtml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/BirdsiteLive/Views/Users/Index.cshtml b/src/BirdsiteLive/Views/Users/Index.cshtml index fce4b14..2ff2ce3 100644 --- a/src/BirdsiteLive/Views/Users/Index.cshtml +++ b/src/BirdsiteLive/Views/Users/Index.cshtml @@ -46,7 +46,7 @@ else if (!string.IsNullOrEmpty(ViewData.Model.MovedTo)) { } @@ -58,11 +58,8 @@ } - - @if (!ViewData.Model.Deleted && string.IsNullOrEmpty(ViewData.Model.MovedTo)) - { -
- I'm the owner of this account and I would like to take control of this mirror. -
- } + +
+ I'm the owner of this account and I would like to take control of this mirror. +
\ No newline at end of file From 89289f2c3afbbcb374cd3d473389cd30de4040a8 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Tue, 27 Dec 2022 22:39:11 -0500 Subject: [PATCH 29/30] wording --- src/BirdsiteLive.Domain/UserService.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index 7f4c55e..6f88543 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -88,7 +88,7 @@ namespace BirdsiteLive.Domain preferredUsername = acct, name = twitterUser.Name, inbox = $"{actorUrl}/inbox", - summary = "[THIS IS A TWITTER MIRRORED ACCOUNT]

" + description, + summary = "[UNOFFICIAL MIRROR: This is a view of Twitter using ActivityPub]

" + description, url = actorUrl, manuallyApprovesFollowers = twitterUser.Protected, discoverable = false, @@ -125,7 +125,7 @@ namespace BirdsiteLive.Domain new UserAttachment { type = "PropertyValue", - name = "Take control of the account", + name = "Take control of this account", value = $"MANAGE" } }, From f8a354d90b755b20a3d56598f6c27a1d1fdef44d Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Tue, 27 Dec 2022 23:09:25 -0500 Subject: [PATCH 30/30] clean up --- src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs index 3841476..5089afc 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs @@ -17,6 +17,6 @@ namespace BirdsiteLive.DAL.Models public string MovedTo { get; set; } public string MovedToAcct { get; set; } - public bool Deleted { get; set; } //TODO: update DAL + public bool Deleted { get; set; } } } \ No newline at end of file