sam/BirdsiteLIVE
sam
/
BirdsiteLIVE
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

Merge pull request #175 from NicolasConstant/topic_user-migration 

Topic user migration
This commit is contained in:
Nicolas Constant 2022-12-27 23:16:41 -05:00 committed by GitHub
parent 4c4fc95da3 f8a354d90b
commit 05cbddbf26
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 1090 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs
View File

@ -4,6 +4,7 @@ namespace BirdsiteLive.ActivityPub.Models
{
public class ActivityDelete : Activity
{
public string[] to { get; set; }
[JsonProperty("object")]
public object apObject { get; set; }
}

1
src/BirdsiteLive.ActivityPub/Models/Actor.cs
View File

@ -17,6 +17,7 @@ namespace BirdsiteLive.ActivityPub
public string name { get; set; }
public string summary { get; set; }
public string url { get; set; }
public string movedTo { get; set; }
public bool manuallyApprovesFollowers { get; set; }
public string inbox { get; set; }
public bool? discoverable { get; set; } = true;

51
src/BirdsiteLive.Domain/ActivityPubService.cs
View File

@ -16,10 +16,18 @@ namespace BirdsiteLive.Domain
{
public interface IActivityPubService
{
Task<string> GetUserIdAsync(string acct);
Task<Actor> GetUser(string objectId);
Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null);
Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost,
string targetInbox);
Task DeleteUserAsync(string username, string targetHost, string targetInbox);
}
public class WebFinger
{
public string subject { get; set; }
public string[] aliases { get; set; }
}
public class ActivityPubService : IActivityPubService
@ -39,6 +47,24 @@ namespace BirdsiteLive.Domain
}
#endregion
public async Task<string> GetUserIdAsync(string acct)
{
var splittedAcct = acct.Trim('@').Split('@');
var url = $"https://{splittedAcct[1]}/.well-known/webfinger?resource=acct:{splittedAcct[0]}@{splittedAcct[1]}";
var httpClient = _httpClientFactory.CreateClient();
httpClient.DefaultRequestHeaders.Add("Accept", "application/json");
var result = await httpClient.GetAsync(url);
result.EnsureSuccessStatusCode();
var content = await result.Content.ReadAsStringAsync();
var actor = JsonConvert.DeserializeObject<WebFinger>(content);
return actor.aliases.FirstOrDefault();
}
public async Task<Actor> GetUser(string objectId)
{
var httpClient = _httpClientFactory.CreateClient();
@ -57,6 +83,31 @@ namespace BirdsiteLive.Domain
return actor;
}
public async Task DeleteUserAsync(string username, string targetHost, string targetInbox)
{
try
{
var actor = UrlFactory.GetActorUrl(_instanceSettings.Domain, username);
var deleteUser = new ActivityDelete
{
context = "https://www.w3.org/ns/activitystreams",
id = $"{actor}#delete",
type = "Delete",
actor = actor,
to = new [] { "https://www.w3.org/ns/activitystreams#Public" },
apObject = actor
};
await PostDataAsync(deleteUser, targetHost, actor, targetInbox);
}
catch (Exception e)
{
_logger.LogError(e, "Error deleting {Username} to {Host}{Inbox}", username, targetHost, targetInbox);
throw;
}
}
public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox)
{
try

4
src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj
View File

@ -15,4 +15,8 @@
<ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
</ItemGroup>
<ItemGroup>
<Folder Include="Enum\" />
</ItemGroup>
</Project>

9
src/BirdsiteLive.Domain/Enum/MigrationTypeEnum.cs Normal file
View File

@ -0,0 +1,9 @@
namespace BirdsiteLive.Domain.Enum
{
public enum MigrationTypeEnum
{
Unknown = 0,
Migration = 1,
Deletion = 2
}
}

281
src/BirdsiteLive.Domain/MigrationService.cs Normal file
View File

@ -0,0 +1,281 @@
using System;
using System.Linq;
using BirdsiteLive.Twitter;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using BirdsiteLive.ActivityPub;
using BirdsiteLive.ActivityPub.Models;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.ActivityPub.Converters;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Domain.Enum;
namespace BirdsiteLive.Domain
{
public class MigrationService
{
private readonly InstanceSettings _instanceSettings;
private readonly ITwitterTweetsService _twitterTweetsService;
private readonly IActivityPubService _activityPubService;
private readonly ITwitterUserDal _twitterUserDal;
private readonly IFollowersDal _followersDal;
#region Ctor
public MigrationService(ITwitterTweetsService twitterTweetsService, IActivityPubService activityPubService, ITwitterUserDal twitterUserDal, IFollowersDal followersDal, InstanceSettings instanceSettings)
{
_twitterTweetsService = twitterTweetsService;
_activityPubService = activityPubService;
_twitterUserDal = twitterUserDal;
_followersDal = followersDal;
_instanceSettings = instanceSettings;
}
#endregion
public string GetMigrationCode(string acct)
{
var hash = GetHashString(acct);
return $"[[BirdsiteLIVE-MigrationCode|{hash.Substring(0, 10)}]]";
}
public string GetDeletionCode(string acct)
{
var hash = GetHashString(acct);
return $"[[BirdsiteLIVE-DeletionCode|{hash.Substring(0, 10)}]]";
}
public bool ValidateTweet(string acct, string tweetId, MigrationTypeEnum type)
{
string code;
if (type == MigrationTypeEnum.Migration)
code = GetMigrationCode(acct);
else if (type == MigrationTypeEnum.Deletion)
code = GetDeletionCode(acct);
else
throw new NotImplementedException();
var castedTweetId = ExtractedTweetId(tweetId);
var tweet = _twitterTweetsService.GetTweet(castedTweetId);
if (tweet == null)
throw new Exception("Tweet not found");
if (tweet.CreatorName.Trim().ToLowerInvariant() != acct.Trim().ToLowerInvariant())
throw new Exception($"Tweet not published by @{acct}");
if (!tweet.MessageContent.Contains(code))
{
var message = "Tweet don't have migration code";
if (type == MigrationTypeEnum.Deletion)
message = "Tweet don't have deletion code";
throw new Exception(message);
}
return true;
}
private long ExtractedTweetId(string tweetId)
{
if (string.IsNullOrWhiteSpace(tweetId))
throw new ArgumentException("No provided Tweet ID");
long castedId;
if (long.TryParse(tweetId, out castedId))
return castedId;
var urlPart = tweetId.Split('/').LastOrDefault();
if (long.TryParse(urlPart, out castedId))
return castedId;
throw new ArgumentException("Unvalid Tweet ID");
}
public async Task<ValidatedFediverseUser> ValidateFediverseAcctAsync(string fediverseAcct)
{
if (string.IsNullOrWhiteSpace(fediverseAcct))
throw new ArgumentException("Please provide Fediverse account");
if (!fediverseAcct.Contains('@') || !fediverseAcct.StartsWith("@") || fediverseAcct.Trim('@').Split('@').Length != 2)
throw new ArgumentException("Please provide valid Fediverse handle");
var objectId = await _activityPubService.GetUserIdAsync(fediverseAcct);
var user = await _activityPubService.GetUser(objectId);
var result = new ValidatedFediverseUser
{
FediverseAcct = fediverseAcct,
ObjectId = objectId,
User = user,
IsValid = user != null
};
return result;
}
public async Task MigrateAccountAsync(ValidatedFediverseUser validatedUser, string acct)
{
// Apply moved to
var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct);
if (twitterAccount == null)
{
await _twitterUserDal.CreateTwitterUserAsync(acct, -1, validatedUser.ObjectId, validatedUser.FediverseAcct);
twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct);
}
twitterAccount.MovedTo = validatedUser.User.id;
twitterAccount.MovedToAcct = validatedUser.FediverseAcct;
twitterAccount.LastSync = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount);
// Notify Followers
var message = $@"<p>[BSL MIRROR SERVICE NOTIFICATION]<br/>This bot has been disabled by its original owner.<br/>It has been redirected to {validatedUser.FediverseAcct}.</p>";
NotifyFollowers(acct, twitterAccount, message);
}
private void NotifyFollowers(string acct, SyncTwitterUser twitterAccount, string message)
{
var t = Task.Run(async () =>
{
var followers = await _followersDal.GetFollowersAsync(twitterAccount.Id);
foreach (var follower in followers)
{
try
{
var noteId = Guid.NewGuid().ToString();
var actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, acct);
var noteUrl = UrlFactory.GetNoteUrl(_instanceSettings.Domain, acct, noteId);
//var to = validatedUser.ObjectId;
var to = follower.ActorId;
var cc = new string[0];
var note = new Note
{
id = noteUrl,
published = DateTime.UtcNow.ToString("s") + "Z",
url = noteUrl,
attributedTo = actorUrl,
to = new[] { to },
cc = cc,
content = message,
tag = new Tag[]{
new Tag()
{
type = "Mention",
href = follower.ActorId,
name = $"@{follower.Acct}@{follower.Host}"
}
},
};
if (!string.IsNullOrWhiteSpace(follower.SharedInboxRoute))
await _activityPubService.PostNewNoteActivity(note, acct, noteId, follower.Host, follower.SharedInboxRoute);
else
await _activityPubService.PostNewNoteActivity(note, acct, noteId, follower.Host, follower.InboxRoute);
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
});
}
public async Task DeleteAccountAsync(string acct)
{
// Apply deleted state
var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct);
if (twitterAccount == null)
{
await _twitterUserDal.CreateTwitterUserAsync(acct, -1);
twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct);
}
twitterAccount.Deleted = true;
twitterAccount.LastSync = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount);
// Notify Followers
var message = $@"<p>[BSL MIRROR SERVICE NOTIFICATION]<br/>This bot has been deleted by its original owner.<br/></p>";
NotifyFollowers(acct, twitterAccount, message);
// Delete remote accounts
DeleteRemoteAccounts(acct);
}
private void DeleteRemoteAccounts(string acct)
{
var t = Task.Run(async () =>
{
var allUsers = await _followersDal.GetAllFollowersAsync();
var followersWtSharedInbox = allUsers
.Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
.GroupBy(x => x.Host)
.ToList();
foreach (var followerGroup in followersWtSharedInbox)
{
var host = followerGroup.First().Host;
var sharedInbox = followerGroup.First().SharedInboxRoute;
var t1 = Task.Run(async () =>
{
await _activityPubService.DeleteUserAsync(acct, host, sharedInbox);
});
}
var followerWtInbox = allUsers
.Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
.ToList();
foreach (var followerGroup in followerWtInbox)
{
var host = followerGroup.Host;
var sharedInbox = followerGroup.InboxRoute;
var t1 = Task.Run(async () =>
{
await _activityPubService.DeleteUserAsync(acct, host, sharedInbox);
});
}
});
}
public async Task TriggerRemoteMigrationAsync(string id, string tweetid, string handle)
{
//TODO
}
public async Task TriggerRemoteDeleteAsync(string id, string tweetid)
{
//TODO
}
private byte[] GetHash(string inputString)
{
using (HashAlgorithm algorithm = SHA256.Create())
return algorithm.ComputeHash(Encoding.UTF8.GetBytes(inputString));
}
private string GetHashString(string inputString)
{
StringBuilder sb = new StringBuilder();
foreach (byte b in GetHash(inputString))
sb.Append(b.ToString("X2"));
return sb.ToString();
}
}
public class ValidatedFediverseUser
{
public string FediverseAcct { get; set; }
public string ObjectId { get; set; }
public Actor User { get; set; }
public bool IsValid { get; set; }
}
}

25
src/BirdsiteLive.Domain/UserService.cs
View File

@ -11,6 +11,7 @@ using BirdsiteLive.ActivityPub.Models;
using BirdsiteLive.Common.Regexes;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.Cryptography;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Domain.BusinessUseCases;
using BirdsiteLive.Domain.Repository;
using BirdsiteLive.Domain.Statistics;
@ -24,7 +25,7 @@ namespace BirdsiteLive.Domain
{
public interface IUserService
{
Actor GetUser(TwitterUser twitterUser);
Actor GetUser(TwitterUser twitterUser, SyncTwitterUser dbTwitterUser);
Task<bool> FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders, ActivityFollow activity, string body);
Task<bool> UndoFollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders, ActivityUndoFollow activity, string body);
@ -64,7 +65,7 @@ namespace BirdsiteLive.Domain
}
#endregion
public Actor GetUser(TwitterUser twitterUser)
public Actor GetUser(TwitterUser twitterUser, SyncTwitterUser dbTwitterUser)
{
var actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, twitterUser.Acct);
var acct = twitterUser.Acct.ToLowerInvariant();
@ -87,9 +88,10 @@ namespace BirdsiteLive.Domain
preferredUsername = acct,
name = twitterUser.Name,
inbox = $"{actorUrl}/inbox",
summary = description,
summary = "[UNOFFICIAL MIRROR: This is a view of Twitter using ActivityPub]<br/><br/>" + description,
url = actorUrl,
manuallyApprovesFollowers = twitterUser.Protected,
discoverable = false,
publicKey = new PublicKey()
{
id = $"{actorUrl}#main-key",
@ -111,14 +113,27 @@ namespace BirdsiteLive.Domain
new UserAttachment
{
type = "PropertyValue",
name = "Official",
name = "Official Account",
value = $"<a href=\"https://twitter.com/{acct}\" rel=\"me nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">twitter.com/{acct}</span></a>"
},
new UserAttachment
{
type = "PropertyValue",
name = "Disclaimer",
value = "This is an automatically created and managed mirror profile from Twitter. While it reflects exactly the content of the original account, it doesn't provide support for interactions and replies. It is an equivalent view from other 3rd party Twitter client apps and uses the same technical means to provide it."
},
new UserAttachment
{
type = "PropertyValue",
name = "Take control of this account",
value = $"<a href=\"https://{_instanceSettings.Domain}/migration/move/{acct}\" rel=\"me nofollow noopener noreferrer\" target=\"_blank\">MANAGE</a>"
}
},
endpoints = new EndPoints
{
sharedInbox = $"https://{_instanceSettings.Domain}/inbox"
}
},
movedTo = dbTwitterUser?.MovedTo
};
return user;
}

2
src/BirdsiteLive.Moderation/Processors/TwitterAccountModerationProcessor.cs
View File

@ -29,7 +29,7 @@ namespace BirdsiteLive.Moderation.Processors
{
if (type == ModerationTypeEnum.None) return;
var twitterUsers = await _twitterUserDal.GetAllTwitterUsersAsync();
var twitterUsers = await _twitterUserDal.GetAllTwitterUsersAsync(false);
foreach (var user in twitterUsers)
{

4
src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs
View File

@ -49,12 +49,12 @@ namespace BirdsiteLive.Pipeline.Processors
{
var tweetId = tweets.Last().Id;
var now = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now);
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now, user.MovedTo, user.MovedToAcct, user.Deleted);
}
else
{
var now = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now);
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now, user.MovedTo, user.MovedToAcct, user.Deleted);
}
}

2
src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
View File

@ -39,7 +39,7 @@ namespace BirdsiteLive.Pipeline.Processors
try
{
var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync();
var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber);
var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber, false);
var userCount = users.Any() ? users.Length : 1;
var splitNumber = (int) Math.Ceiling(userCount / 15d);

2
src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs
View File

@ -48,7 +48,7 @@ namespace BirdsiteLive.Pipeline.Processors
var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();
var minimumSync = followingSyncStatuses.Min();
var now = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now);
await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now, userWithTweetsToSync.User.MovedTo, userWithTweetsToSync.User.MovedToAcct, userWithTweetsToSync.User.Deleted);
}
catch (Exception e)
{

3
src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs
View File

@ -28,7 +28,8 @@ namespace BirdsiteLive.Twitter.Extractors
IsReply = tweet.InReplyToUserId != null,
IsThread = tweet.InReplyToUserId != null && tweet.InReplyToUserId == tweet.CreatedBy.Id,
IsRetweet = tweet.IsRetweet || tweet.QuotedStatusId != null,
RetweetUrl = ExtractRetweetUrl(tweet)
RetweetUrl = ExtractRetweetUrl(tweet),
CreatorName = tweet.CreatedBy.UserIdentifier.ScreenName
};
return extractedTweet;

1
src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs
View File

@ -15,5 +15,6 @@ namespace BirdsiteLive.Twitter.Models
public bool IsThread { get; set; }
public bool IsRetweet { get; set; }
public string RetweetUrl { get; set; }
public string CreatorName { get; set; }
}
}

39
src/BirdsiteLive/Controllers/DebugingController.cs
View File

@ -59,17 +59,22 @@ namespace BirdsiteLive.Controllers
[HttpPost]
public async Task<IActionResult> PostNote()
{
var username = "gra";
var username = "twitter";
var actor = $"https://{_instanceSettings.Domain}/users/{username}";
var targetHost = "mastodon.technology";
var target = $"{targetHost}/users/testtest";
var inbox = $"/users/testtest/inbox";
var targetHost = "ioc.exchange";
var target = $"https://{targetHost}/users/test";
//var inbox = $"/users/testtest/inbox";
var inbox = $"/inbox";
var noteGuid = Guid.NewGuid();
var noteId = $"https://{_instanceSettings.Domain}/users/{username}/statuses/{noteGuid}";
var noteUrl = $"https://{_instanceSettings.Domain}/@{username}/{noteGuid}";
var to = $"{actor}/followers";
to = target;
var cc = new[] { "https://www.w3.org/ns/activitystreams#Public" };
cc = new string[0];
var now = DateTime.UtcNow;
var nowString = now.ToString("s") + "Z";
@ -82,7 +87,7 @@ namespace BirdsiteLive.Controllers
actor = actor,
published = nowString,
to = new[] { to },
//cc = new [] { "https://www.w3.org/ns/activitystreams#Public" },
cc = cc,
apObject = new Note()
{
id = noteId,
@ -94,7 +99,8 @@ namespace BirdsiteLive.Controllers
// Unlisted
to = new[] { to },
cc = new[] { "https://www.w3.org/ns/activitystreams#Public" },
cc = cc,
//cc = new[] { "https://www.w3.org/ns/activitystreams#Public" },
//// Public
//to = new[] { "https://www.w3.org/ns/activitystreams#Public" },
@ -102,8 +108,16 @@ namespace BirdsiteLive.Controllers
sensitive = false,
content = "<p>TEST PUBLIC</p>",
//content = "<p><span class=\"h-card\"><a href=\"https://ioc.exchange/users/test\" class=\"u-url mention\">@<span>test</span></a></span> test</p>",
attachment = new Attachment[0],
tag = new Tag[0]
tag = new Tag[]{
new Tag()
{
type = "Mention",
href = target,
name = "@test@ioc.exchange"
}
},
}
};
@ -125,6 +139,17 @@ namespace BirdsiteLive.Controllers
await _userService.SendRejectFollowAsync(activityFollow, "mastodon.technology");
return View("Index");
}
[HttpPost]
public async Task<IActionResult> PostDeleteUser()
{
var userName = "twitter";
var host = "ioc.exchange";
var inbox = "/inbox";
await _activityPubService.DeleteUserAsync(userName, host, inbox);
return View("Index");
}
}
#endif

237
src/BirdsiteLive/Controllers/MigrationController.cs Normal file
View File

@ -0,0 +1,237 @@
using System;
using Microsoft.AspNetCore.Mvc;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using Npgsql.TypeHandlers;
using BirdsiteLive.Domain;
using BirdsiteLive.Domain.Enum;
using BirdsiteLive.DAL.Contracts;
namespace BirdsiteLive.Controllers
{
public class MigrationController : Controller
{
private readonly MigrationService _migrationService;
private readonly ITwitterUserDal _twitterUserDal;
#region Ctor
public MigrationController(MigrationService migrationService, ITwitterUserDal twitterUserDal)
{
_migrationService = migrationService;
_twitterUserDal = twitterUserDal;
}
#endregion
[HttpGet]
[Route("/migration/move/{id}")]
public IActionResult IndexMove(string id)
{
var migrationCode = _migrationService.GetMigrationCode(id);
var data = new MigrationData()
{
Acct = id,
MigrationCode = migrationCode
};
return View("Index", data);
}
[HttpGet]
[Route("/migration/delete/{id}")]
public IActionResult IndexDelete(string id)
{
var migrationCode = _migrationService.GetDeletionCode(id);
var data = new MigrationData()
{
Acct = id,
MigrationCode = migrationCode
};
return View("Delete", data);
}
[HttpPost]
[Route("/migration/move/{id}")]
public async Task<IActionResult> MigrateMove(string id, string tweetid, string handle)
{
var migrationCode = _migrationService.GetMigrationCode(id);
var data = new MigrationData()
{
Acct = id,
MigrationCode = migrationCode,
IsAcctProvided = !string.IsNullOrWhiteSpace(handle),
IsTweetProvided = !string.IsNullOrWhiteSpace(tweetid),
TweetId = tweetid,
FediverseAccount = handle
};
ValidatedFediverseUser fediverseUserValidation = null;
//Verify can be migrated
var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id);
if (twitterAccount != null && twitterAccount.Deleted)
{
data.ErrorMessage = "This account has been deleted, it can't be migrated";
return View("Index", data);
}
if (twitterAccount != null &&
(!string.IsNullOrWhiteSpace(twitterAccount.MovedTo)
|| !string.IsNullOrWhiteSpace(twitterAccount.MovedToAcct)))
{
data.ErrorMessage = "This account has been moved already, it can't be migrated again";
return View("Index", data);
}
// Start migration
try
{
fediverseUserValidation = await _migrationService.ValidateFediverseAcctAsync(handle);
var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Migration);
data.IsAcctValid = fediverseUserValidation.IsValid;
data.IsTweetValid = isTweetValid;
}
catch (Exception e)
{
data.ErrorMessage = e.Message;
}
if (data.IsAcctValid && data.IsTweetValid && fediverseUserValidation != null)
{
try
{
await _migrationService.MigrateAccountAsync(fediverseUserValidation, id);
await _migrationService.TriggerRemoteMigrationAsync(id, tweetid, handle);
data.MigrationSuccess = true;
}
catch (Exception e)
{
Console.WriteLine(e);
data.ErrorMessage = e.Message;
}
}
return View("Index", data);
}
[HttpPost]
[Route("/migration/delete/{id}")]
public async Task<IActionResult> MigrateDelete(string id, string tweetid)
{
var deletionCode = _migrationService.GetDeletionCode(id);
var data = new MigrationData()
{
Acct = id,
MigrationCode = deletionCode,
IsTweetProvided = !string.IsNullOrWhiteSpace(tweetid),
TweetId = tweetid
};
//Verify can be deleted
var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id);
if (twitterAccount != null && twitterAccount.Deleted)
{
data.ErrorMessage = "This account has been deleted, it can't be deleted again";
return View("Delete", data);
}
// Start deletion
try
{
var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Deletion);
data.IsTweetValid = isTweetValid;
}
catch (Exception e)
{
data.ErrorMessage = e.Message;
}
if (data.IsTweetValid)
{
try
{
await _migrationService.DeleteAccountAsync(id);
await _migrationService.TriggerRemoteDeleteAsync(id, tweetid);
data.MigrationSuccess = true;
}
catch (Exception e)
{
Console.WriteLine(e);
data.ErrorMessage = e.Message;
}
}
return View("Delete", data);
}
[HttpPost]
[Route("/migration/move/{id}/{tweetid}/{handle}")]
public async Task<IActionResult> RemoteMigrateMove(string id, string tweetid, string handle)
{
//Verify can be migrated
var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id);
if (twitterAccount.Deleted
|| !string.IsNullOrWhiteSpace(twitterAccount.MovedTo)
|| !string.IsNullOrWhiteSpace(twitterAccount.MovedToAcct))
return Ok();
// Start migration
var fediverseUserValidation = await _migrationService.ValidateFediverseAcctAsync(handle);
var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Migration);
if (fediverseUserValidation.IsValid && isTweetValid)
{
await _migrationService.MigrateAccountAsync(fediverseUserValidation, id);
return Ok();
}
return StatusCode(400);
}
[HttpPost]
[Route("/migration/delete/{id}/{tweetid}")]
public async Task<IActionResult> RemoteMigrateDelete(string id, string tweetid)
{
//Verify can be deleted
var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id);
if (twitterAccount.Deleted) return Ok();
// Start deletion
var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Deletion);
if (isTweetValid)
{
await _migrationService.DeleteAccountAsync(id);
return Ok();
}
return StatusCode(400);
}
}
public class MigrationData
{
public string Acct { get; set; }
public string FediverseAccount { get; set; }
public string TweetId { get; set; }
public string MigrationCode { get; set; }
public bool IsTweetProvided { get; set; }
public bool IsAcctProvided { get; set; }
public bool IsTweetValid { get; set; }
public bool IsAcctValid { get; set; }
public string ErrorMessage { get; set; }
public bool MigrationSuccess { get; set; }
}
}

27
src/BirdsiteLive/Controllers/UsersController.cs
View File

@ -11,6 +11,8 @@ using BirdsiteLive.ActivityPub;
using BirdsiteLive.ActivityPub.Models;
using BirdsiteLive.Common.Regexes;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Domain;
using BirdsiteLive.Models;
using BirdsiteLive.Tools;
@ -28,13 +30,14 @@ namespace BirdsiteLive.Controllers
{
private readonly ITwitterUserService _twitterUserService;
private readonly ITwitterTweetsService _twitterTweetService;
private readonly ITwitterUserDal _twitterUserDal;
private readonly IUserService _userService;
private readonly IStatusService _statusService;
private readonly InstanceSettings _instanceSettings;
private readonly ILogger<UsersController> _logger;
#region Ctor
public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ITwitterTweetsService twitterTweetService, ILogger<UsersController> logger)
public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ITwitterTweetsService twitterTweetService, ILogger<UsersController> logger, ITwitterUserDal twitterUserDal)
{
_twitterUserService = twitterUserService;
_userService = userService;
@ -42,6 +45,7 @@ namespace BirdsiteLive.Controllers
_instanceSettings = instanceSettings;
_twitterTweetService = twitterTweetService;
_logger = logger;
_twitterUserDal = twitterUserDal;
}
#endregion
@ -56,11 +60,10 @@ namespace BirdsiteLive.Controllers
}
return View("UserNotFound");
}
[Route("/@{id}")]
[Route("/users/{id}")]
[Route("/users/{id}/remote_follow")]
public IActionResult Index(string id)
public async Task<IActionResult> Index(string id)
{
_logger.LogTrace("User Index: {Id}", id);
@ -102,6 +105,7 @@ namespace BirdsiteLive.Controllers
}
//var isSaturated = _twitterUserService.IsUserApiRateLimited();
var dbUser = await _twitterUserDal.GetTwitterUserAsync(id);
var acceptHeaders = Request.Headers["Accept"];
if (acceptHeaders.Any())
@ -111,7 +115,8 @@ namespace BirdsiteLive.Controllers
{
if (isSaturated) return new ObjectResult("Too Many Requests") { StatusCode = 429 };
if (notFound) return NotFound();
var apUser = _userService.GetUser(user);
if (dbUser != null && dbUser.Deleted) return new ObjectResult("Gone") { StatusCode = 410 };
var apUser = _userService.GetUser(user, dbUser);
var jsonApUser = JsonConvert.SerializeObject(apUser);
return Content(jsonApUser, "application/activity+json; charset=utf-8");
}
@ -128,11 +133,21 @@ namespace BirdsiteLive.Controllers
Url = user.Url,
ProfileImageUrl = user.ProfileImageUrl,
Protected = user.Protected,
InstanceHandle = $"@{user.Acct.ToLowerInvariant()}@{_instanceSettings.Domain}",
InstanceHandle = $"@{user.Acct.ToLowerInvariant()}@{_instanceSettings.Domain}"
MovedTo = dbUser?.MovedTo,
MovedToAcct = dbUser?.MovedToAcct,
Deleted = dbUser?.Deleted ?? false,
};
return View(displayableUser);
}
[Route("/users/{id}/remote_follow")]
public async Task<IActionResult> IndexRemoteFollow(string id)
{
return Redirect($"/users/{id}");
}
[Route("/@{id}/{statusId}")]
[Route("/users/{id}/statuses/{statusId}")]

4
src/BirdsiteLive/Models/DisplayTwitterUser.cs
View File

@ -10,5 +10,9 @@
public bool Protected { get; set; }
public string InstanceHandle { get; set; }
public string MovedTo { get; set; }
public string MovedToAcct { get; set; }
public bool Deleted { get; set; }
}
}

6
src/BirdsiteLive/Views/Debuging/Index.cshtml
View File

@ -23,4 +23,10 @@
<!-- Input and Submit elements -->
<button type="submit" value="Submit">Reject Follow</button>
</form>
<form asp-controller="Debuging" asp-action="PostDeleteUser" method="post">
<!-- Input and Submit elements -->
<button type="submit" value="Submit">Delete User</button>
</form>

51
src/BirdsiteLive/Views/Migration/Delete.cshtml Normal file
View File

@ -0,0 +1,51 @@
@model BirdsiteLive.Controllers.MigrationData
@{
ViewData["Title"] = "Migration";
}
<div class="col-12 col-sm-10 col-md-8 col-lg-6 mx-auto">
@if (!string.IsNullOrWhiteSpace(ViewData.Model.ErrorMessage))
{
<div class="alert alert-danger" role="alert">
@ViewData.Model.ErrorMessage
</div>
}
@if (ViewData.Model.MigrationSuccess)
{
<div class="alert alert-success" role="alert">
The mirror has been successfully deleted
</div>
}
<h1 class="display-4 migration__title">Delete @@@ViewData.Model.Acct mirror</h1>
@if (!ViewData.Model.IsTweetProvided)
{
<h2 class="display-4 migration__subtitle">What is needed?</h2>
<p>You'll need access to the Twitter account to provide proof of ownership.</p>
<h2 class="display-4 migration__subtitle">What will deletion do?</h2>
<p>
Deletion will remove all followers, delete the account and will be blacklisted so that it can't be recreated.<br />
</p>
}
<h2 class="display-4 migration__subtitle">Start the deletion!</h2>
<p>Please copy and post this string in a public Tweet (the string must be untampered, but you can write anything you want before or after it):</p>
<input type="text" name="textbox" value="@ViewData.Model.MigrationCode" onclick="this.select()" class="form-control" readonly />
<br />
<h2 class="display-4 migration__subtitle">Provide deletion information:</h2>
<form method="POST">
<div class="form-group">
<label for="tweetid">Tweet URL</label>
<input type="text" class="form-control" id="tweetid" name="tweetid" autocomplete="off" placeholder="https://twitter.com/<username>/status/<tweet id>" value="@ViewData.Model.TweetId">
</div>
<button type="submit" class="btn btn-primary">Delete!</button>
</form>
</div>

66
src/BirdsiteLive/Views/Migration/Index.cshtml Normal file
View File

@ -0,0 +1,66 @@
@model BirdsiteLive.Controllers.MigrationData
@{
ViewData["Title"] = "Migration";
}
<div class="col-12 col-sm-10 col-md-8 col-lg-6 mx-auto">
@if (!string.IsNullOrWhiteSpace(ViewData.Model.ErrorMessage))
{
<div class="alert alert-danger" role="alert">
@ViewData.Model.ErrorMessage
</div>
}
@if (ViewData.Model.MigrationSuccess)
{
<div class="alert alert-success" role="alert">
The mirror has been successfully migrated
</div>
}
<h1 class="display-4 migration__title">Migrate @@@ViewData.Model.Acct mirror to my Fediverse account</h1>
@if (!ViewData.Model.IsAcctProvided && !ViewData.Model.IsTweetProvided)
{
<h2 class="display-4 migration__subtitle">What is needed?</h2>
<p>You'll need a Fediverse account and access to the Twitter account to provide proof of ownership.</p>
<h2 class="display-4 migration__subtitle">What will migration do?</h2>
<p>
Migration will notify followers of the migration of the mirror account to your fediverse account and will be disabled after that.<br />
</p>
}
<h2 class="display-4 migration__subtitle">Start the migration!</h2>
<p>Please copy and post this string in a public Tweet (the string must be untampered, but you can write anything you want before or after it):</p>
<input type="text" name="textbox" value="@ViewData.Model.MigrationCode" onclick="this.select()" class="form-control" readonly />
<br />
<h2 class="display-4 migration__subtitle">Provide migration information:</h2>
<form method="POST">
@*<div class="form-group">
<label for="exampleInputEmail1">Email address</label>
<input type="email" class="form-control" id="exampleInputEmail1" aria-describedby="emailHelp" placeholder="Enter email">
<small id="emailHelp" class="form-text text-muted">We'll never share your email with anyone else.</small>
</div>*@
<div class="form-group">
<label for="handle">Fediverse target account</label>
<input type="text" class="form-control" id="handle" name="handle" autocomplete="off" placeholder="@Html.Raw("@username@domain.ext")" value="@ViewData.Model.FediverseAccount">
</div>
<div class="form-group">
<label for="tweetid">Tweet URL</label>
<input type="text" class="form-control" id="tweetid" name="tweetid" autocomplete="off" placeholder="https://twitter.com/<username>/status/<tweet id>" value="@ViewData.Model.TweetId">
</div>
<button type="submit" class="btn btn-primary">Migrate!</button>
</form>
<br />
<br />
<br />
<div class="user-owner">
<a href="/migration/delete/@ViewData.Model.Acct">I don't have a fediverse account and I'd like to delete this mirror.</a>
</div>
</div>

17
src/BirdsiteLive/Views/Users/Index.cshtml
View File

@ -37,6 +37,19 @@
This account is protected, BirdsiteLIVE cannot fetch their tweets and will not provide follow support until it is unprotected again.
</div>
}
else if (ViewData.Model.Deleted)
{
<div class="alert alert-danger" role="alert">
This mirror has been deleted by its Twitter owner.
</div>
}
else if (!string.IsNullOrEmpty(ViewData.Model.MovedTo))
{
<div class="alert alert-danger" role="alert">
This account has been migrated by its Twitter owner and has been disabled.<br />
You can follow this user at <a href="@ViewData.Model.MovedTo">@ViewData.Model.MovedToAcct</a>.
</div>
}
else
{
<div>
@ -45,4 +58,8 @@
<input type="text" name="textbox" value="@ViewData.Model.InstanceHandle" onclick="this.select()" class="form-control" readonly />
</div>
}
<div class="user-owner">
<a href="/migration/move/@ViewData.Model.Acct">I'm the owner of this account and I would like to take control of this mirror.</a>
</div>
</div>

15
src/BirdsiteLive/wwwroot/css/birdsite.css
View File

@ -71,3 +71,18 @@
margin-left: 60px;
/*font-weight: bold;*/
}
.user-owner {
font-size: .8em;
padding-top: 20px;
}
/** Migration **/
.migration__title {
font-size: 1.8em;
}
.migration__subtitle {
font-size: 1.4em;
}

16
src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs
View File

@ -23,7 +23,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
public class DbInitializerPostgresDal : PostgresBase, IDbInitializerDal
{
private readonly PostgresTools _tools;
private readonly Version _currentVersion = new Version(2, 4);
private readonly Version _currentVersion = new Version(2, 5);
private const string DbVersionType = "db-version";
#region Ctor
@ -135,7 +135,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
new Tuple<Version, Version>(new Version(2,0), new Version(2,1)),
new Tuple<Version, Version>(new Version(2,1), new Version(2,2)),
new Tuple<Version, Version>(new Version(2,2), new Version(2,3)),
new Tuple<Version, Version>(new Version(2,3), new Version(2,4))
new Tuple<Version, Version>(new Version(2,3), new Version(2,4)),
new Tuple<Version, Version>(new Version(2,4), new Version(2,5))
};
}
@ -172,6 +173,17 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
var alterPostingError = $@"ALTER TABLE {_settings.FollowersTableName} ALTER COLUMN postingErrorCount TYPE INTEGER";
await _tools.ExecuteRequestAsync(alterPostingError);
}
else if (from == new Version(2, 4) && to == new Version(2, 5))
{
var addMovedTo = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD movedTo VARCHAR(2048)";
await _tools.ExecuteRequestAsync(addMovedTo);
var addMovedToAcct = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD movedToAcct VARCHAR(305)";
await _tools.ExecuteRequestAsync(addMovedToAcct);
var addDeletedToAcct = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD deleted BOOLEAN";
await _tools.ExecuteRequestAsync(addDeletedToAcct);
}
else
{
throw new NotImplementedException();

45
src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs
View File

@ -18,7 +18,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
}
#endregion
public async Task CreateTwitterUserAsync(string acct, long lastTweetPostedId)
public async Task CreateTwitterUserAsync(string acct, long lastTweetPostedId, string movedTo = null, string movedToAcct = null)
{
acct = acct.ToLowerInvariant();
@ -27,8 +27,15 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
dbConnection.Open();
await dbConnection.ExecuteAsync(
$"INSERT INTO {_settings.TwitterUserTableName} (acct,lastTweetPostedId,lastTweetSynchronizedForAllFollowersId) VALUES(@acct,@lastTweetPostedId,@lastTweetSynchronizedForAllFollowersId)",
new { acct, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = lastTweetPostedId });
$"INSERT INTO {_settings.TwitterUserTableName} (acct,lastTweetPostedId,lastTweetSynchronizedForAllFollowersId, movedTo, movedToAcct) VALUES(@acct,@lastTweetPostedId,@lastTweetSynchronizedForAllFollowersId,@movedTo,@movedToAcct)",
new
{
acct,
lastTweetPostedId,
lastTweetSynchronizedForAllFollowersId = lastTweetPostedId,
movedTo,
movedToAcct
});
}
}
@ -62,7 +69,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
public async Task<int> GetTwitterUsersCountAsync()
{
var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName}";
var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE";
using (var dbConnection = Connection)
{
@ -75,7 +82,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
public async Task<int> GetFailingTwitterUsersCountAsync()
{
var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE fetchingErrorCount > 0";
var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE fetchingErrorCount > 0 AND (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE";
using (var dbConnection = Connection)
{
@ -86,9 +93,10 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
}
}
public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber)
public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber, bool retrieveDisabledUser)
{
var query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber";
var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber";
if (retrieveDisabledUser) query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber";
using (var dbConnection = Connection)
{
@ -99,9 +107,10 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
}
}
public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync()
public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(bool retrieveDisabledUser)
{
var query = $"SELECT * FROM {_settings.TwitterUserTableName}";
var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE";
if(retrieveDisabledUser) query = $"SELECT * FROM {_settings.TwitterUserTableName}";
using (var dbConnection = Connection)
{
@ -112,26 +121,36 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
}
}
public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync)
public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync, string movedTo, string movedToAcct, bool deleted)
{
if(id == default) throw new ArgumentException("id");
if(lastTweetPostedId == default) throw new ArgumentException("lastTweetPostedId");
if(lastTweetSynchronizedForAllFollowersId == default) throw new ArgumentException("lastTweetSynchronizedForAllFollowersId");
if(lastSync == default) throw new ArgumentException("lastSync");
var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId, fetchingErrorCount = @fetchingErrorCount, lastSync = @lastSync WHERE id = @id";
var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId, fetchingErrorCount = @fetchingErrorCount, lastSync = @lastSync, movedTo = @movedTo, movedToAcct = @movedToAcct, deleted = @deleted WHERE id = @id";
using (var dbConnection = Connection)
{
dbConnection.Open();
await dbConnection.QueryAsync(query, new { id, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId, fetchingErrorCount, lastSync = lastSync.ToUniversalTime() });
await dbConnection.QueryAsync(query, new
{
id,
lastTweetPostedId,
lastTweetSynchronizedForAllFollowersId,
fetchingErrorCount,
lastSync = lastSync.ToUniversalTime(),
movedTo,
movedToAcct,
deleted
});
}
}
public async Task UpdateTwitterUserAsync(SyncTwitterUser user)
{
await UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, user.LastSync);
await UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, user.LastSync, user.MovedTo, user.MovedToAcct, user.Deleted);
}
public async Task DeleteTwitterUserAsync(string acct)

9
src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs
View File

@ -6,12 +6,13 @@ namespace BirdsiteLive.DAL.Contracts
{
public interface ITwitterUserDal
{
Task CreateTwitterUserAsync(string acct, long lastTweetPostedId);
Task CreateTwitterUserAsync(string acct, long lastTweetPostedId, string movedTo = null,
string movedToAcct = null);
Task<SyncTwitterUser> GetTwitterUserAsync(string acct);
Task<SyncTwitterUser> GetTwitterUserAsync(int id);
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber);
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync();
Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync);
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber, bool retrieveDisabledUser);
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(bool retrieveDisabledUser);
Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync, string movedTo, string movedToAcct, bool deleted);
Task UpdateTwitterUserAsync(SyncTwitterUser user);
Task DeleteTwitterUserAsync(string acct);
Task DeleteTwitterUserAsync(int id);

7
src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs
View File

@ -12,6 +12,11 @@ namespace BirdsiteLive.DAL.Models
public DateTime LastSync { get; set; }
public int FetchingErrorCount { get; set; } //TODO: update DAL
public int FetchingErrorCount { get; set; }
public string MovedTo { get; set; }
public string MovedToAcct { get; set; }
public bool Deleted { get; set; }
}
}

185
src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs
View File

@ -71,6 +71,28 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
Assert.AreEqual(result.Id, resultById.Id);
}
[TestMethod]
public async Task CreateAndGetMigratedUser_byId()
{
var acct = "myid";
var lastTweetId = 1548L;
var movedTo = "https://";
var movedToAcct = "@account@instance";
var dal = new TwitterUserPostgresDal(_settings);
await dal.CreateTwitterUserAsync(acct, lastTweetId, movedTo, movedToAcct);
var result = await dal.GetTwitterUserAsync(acct);
var resultById = await dal.GetTwitterUserAsync(result.Id);
Assert.AreEqual(acct, resultById.Acct);
Assert.AreEqual(lastTweetId, resultById.LastTweetPostedId);
Assert.AreEqual(lastTweetId, resultById.LastTweetSynchronizedForAllFollowersId);
Assert.AreEqual(result.Id, resultById.Id);
Assert.AreEqual(movedTo, result.MovedTo);
Assert.AreEqual(movedToAcct, result.MovedToAcct);
}
[TestMethod]
public async Task CreateUpdateAndGetUser()
{
@ -87,7 +109,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
var updatedLastSyncId = 1550L;
var now = DateTime.Now;
var errors = 15;
await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now);
await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now, null, null, false);
result = await dal.GetTwitterUserAsync(acct);
@ -96,6 +118,68 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId);
Assert.AreEqual(errors, result.FetchingErrorCount);
Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100);
Assert.AreEqual(null, result.MovedTo);
Assert.AreEqual(null, result.MovedToAcct);
}
[TestMethod]
public async Task CreateUpdateAndGetMigratedUser()
{
var acct = "myid";
var lastTweetId = 1548L;
var dal = new TwitterUserPostgresDal(_settings);
await dal.CreateTwitterUserAsync(acct, lastTweetId);
var result = await dal.GetTwitterUserAsync(acct);
var updatedLastTweetId = 1600L;
var updatedLastSyncId = 1550L;
var now = DateTime.Now;
var errors = 15;
var movedTo = "https://";
var movedToAcct = "@account@instance";
await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now, movedTo, movedToAcct, false);
result = await dal.GetTwitterUserAsync(acct);
Assert.AreEqual(acct, result.Acct);
Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId);
Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId);
Assert.AreEqual(errors, result.FetchingErrorCount);
Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100);
Assert.AreEqual(movedTo, result.MovedTo);
Assert.AreEqual(movedToAcct, result.MovedToAcct);
}
[TestMethod]
public async Task CreateUpdateAndGetDeletedUser()
{
var acct = "myid";
var lastTweetId = 1548L;
var dal = new TwitterUserPostgresDal(_settings);
await dal.CreateTwitterUserAsync(acct, lastTweetId);
var result = await dal.GetTwitterUserAsync(acct);
var updatedLastTweetId = 1600L;
var updatedLastSyncId = 1550L;
var now = DateTime.Now;
var errors = 15;
await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now, null, null, true);
result = await dal.GetTwitterUserAsync(acct);
Assert.AreEqual(acct, result.Acct);
Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId);
Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId);
Assert.AreEqual(errors, result.FetchingErrorCount);
Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100);
Assert.AreEqual(null, result.MovedTo);
Assert.AreEqual(null, result.MovedToAcct);
Assert.AreEqual(true, result.Deleted);
}
[TestMethod]
@ -167,7 +251,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
public async Task Update_NoId()
{
var dal = new TwitterUserPostgresDal(_settings);
await dal.UpdateTwitterUserAsync(default, default, default, default, DateTime.UtcNow);
await dal.UpdateTwitterUserAsync(default, default, default, default, DateTime.UtcNow, null, null, false);
}
[TestMethod]
@ -175,7 +259,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
public async Task Update_NoLastTweetPostedId()
{
var dal = new TwitterUserPostgresDal(_settings);
await dal.UpdateTwitterUserAsync(12, default, default, default, DateTime.UtcNow);
await dal.UpdateTwitterUserAsync(12, default, default, default, DateTime.UtcNow, null, null, false);
}
[TestMethod]
@ -183,7 +267,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
public async Task Update_NoLastTweetSynchronizedForAllFollowersId()
{
var dal = new TwitterUserPostgresDal(_settings);
await dal.UpdateTwitterUserAsync(12, 9556, default, default, DateTime.UtcNow);
await dal.UpdateTwitterUserAsync(12, 9556, default, default, DateTime.UtcNow, null, null, false);
}
[TestMethod]
@ -191,7 +275,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
public async Task Update_NoLastSync()
{
var dal = new TwitterUserPostgresDal(_settings);
await dal.UpdateTwitterUserAsync(12, 9556, 65, default, default);
await dal.UpdateTwitterUserAsync(12, 9556, 65, default, default, null, null, false);
}
[TestMethod]
@ -256,12 +340,79 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
await dal.CreateTwitterUserAsync(acct, lastTweetId);
}
var result = await dal.GetAllTwitterUsersAsync(1000);
for (int i = 0; i < 10; i++)
{
var acct = $"migrated-myid{i}";
var lastTweetId = 1548L;
await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain");
}
for (int i = 0; i < 10; i++)
{
var acct = $"deleted-myid{i}";
var lastTweetId = 148L;
await dal.CreateTwitterUserAsync(acct, lastTweetId);
var user = await dal.GetTwitterUserAsync(acct);
user.Deleted = true;
user.LastSync = DateTime.UtcNow;
await dal.UpdateTwitterUserAsync(user);
}
var result = await dal.GetAllTwitterUsersAsync(1100, false);
Assert.AreEqual(1000, result.Length);
Assert.IsFalse(result[0].Id == default);
Assert.IsFalse(result[0].Acct == default);
Assert.IsFalse(result[0].LastTweetPostedId == default);
Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
foreach (var user in result)
{
Assert.IsTrue(string.IsNullOrWhiteSpace(user.MovedTo));
Assert.IsTrue(string.IsNullOrWhiteSpace(user.MovedToAcct));
Assert.IsFalse(user.Deleted);
}
}
[TestMethod]
public async Task GetAllTwitterUsers_Top_RetrieveDeleted()
{
var dal = new TwitterUserPostgresDal(_settings);
for (var i = 0; i < 1000; i++)
{
var acct = $"myid{i}";
var lastTweetId = 1548L;
await dal.CreateTwitterUserAsync(acct, lastTweetId);
}
for (int i = 0; i < 10; i++)
{
var acct = $"migrated-myid{i}";
var lastTweetId = 1548L;
await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain");
}
for (int i = 0; i < 10; i++)
{
var acct = $"deleted-myid{i}";
var lastTweetId = 148L;
await dal.CreateTwitterUserAsync(acct, lastTweetId);
var user = await dal.GetTwitterUserAsync(acct);
user.Deleted = true;
user.LastSync = DateTime.UtcNow;
await dal.UpdateTwitterUserAsync(user);
}
var result = await dal.GetAllTwitterUsersAsync(1100, true);
Assert.AreEqual(1020, result.Length);
Assert.IsFalse(result[0].Id == default);
Assert.IsFalse(result[0].Acct == default);
Assert.IsFalse(result[0].LastTweetPostedId == default);
Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
}
[TestMethod]
@ -279,7 +430,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
// Update accounts
var now = DateTime.UtcNow;
var allUsers = await dal.GetAllTwitterUsersAsync();
var allUsers = await dal.GetAllTwitterUsersAsync(false);
foreach (var acc in allUsers)
{
var lastSync = now.AddDays(acc.LastTweetPostedId);
@ -290,7 +441,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
// Create a not init account
await dal.CreateTwitterUserAsync("not_init", -1);
var result = await dal.GetAllTwitterUsersAsync(10);
var result = await dal.GetAllTwitterUsersAsync(10, false);
Assert.IsTrue(result.Any(x => x.Acct == "myid0"));
Assert.IsTrue(result.Any(x => x.Acct == "myid8"));
@ -313,15 +464,15 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
await dal.CreateTwitterUserAsync(acct, lastTweetId);
}
var allUsers = await dal.GetAllTwitterUsersAsync(100);
var allUsers = await dal.GetAllTwitterUsersAsync(100, false);
for (var i = 0; i < 20; i++)
{
var user = allUsers[i];
var date = i % 2 == 0 ? oldest : newest;
await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date);
await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date, null, null, false);
}
var result = await dal.GetAllTwitterUsersAsync(10);
var result = await dal.GetAllTwitterUsersAsync(10, false);
Assert.AreEqual(10, result.Length);
Assert.IsFalse(result[0].Id == default);
Assert.IsFalse(result[0].Acct == default);
@ -344,7 +495,15 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
await dal.CreateTwitterUserAsync(acct, lastTweetId);
}
var result = await dal.GetAllTwitterUsersAsync();
for (int i = 0; i < 10; i++)
{
var acct = $"migrated-myid{i}";
var lastTweetId = 1548L;
await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain");
}
var result = await dal.GetAllTwitterUsersAsync(false);
Assert.AreEqual(1000, result.Length);
Assert.IsFalse(result[0].Id == default);
Assert.IsFalse(result[0].Acct == default);
@ -382,7 +541,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
if (i == 0 || i == 2 || i == 3)
{
var t = await dal.GetTwitterUserAsync(acct);
await dal.UpdateTwitterUserAsync(t.Id ,1L,2L, 50+i*2, DateTime.Now);
await dal.UpdateTwitterUserAsync(t.Id ,1L,2L, 50+i*2, DateTime.Now, null, null, false);
}
}

4
src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessFollowUserTests.cs
View File

@ -77,7 +77,9 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
twitterUserDalMock
.Setup(x => x.CreateTwitterUserAsync(
It.Is<string>(y => y == twitterName),
It.Is<long>(y => y == -1)))
It.Is<long>(y => y == -1),
It.Is<string>(y => y == null),
It.Is<string>(y => y == null)))
.Returns(Task.CompletedTask);
#endregion

8
src/Tests/BirdsiteLive.Moderation.Tests/Processors/TwitterAccountModerationProcessorTests.cs
View File

@ -48,7 +48,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors
#region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync())
.Setup(x => x.GetAllTwitterUsersAsync(It.Is<bool>(y => y == false)))
.ReturnsAsync(allUsers.ToArray());
var moderationRepositoryMock = new Mock<IModerationRepository>(MockBehavior.Strict);
@ -87,7 +87,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors
#region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync())
.Setup(x => x.GetAllTwitterUsersAsync(It.Is<bool>(y => y == false)))
.ReturnsAsync(allUsers.ToArray());
var moderationRepositoryMock = new Mock<IModerationRepository>(MockBehavior.Strict);
@ -130,7 +130,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors
#region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync())
.Setup(x => x.GetAllTwitterUsersAsync(It.Is<bool>(y => y == false)))
.ReturnsAsync(allUsers.ToArray());
var moderationRepositoryMock = new Mock<IModerationRepository>(MockBehavior.Strict);
@ -173,7 +173,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors
#region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync())
.Setup(x => x.GetAllTwitterUsersAsync(It.Is<bool>(y => y == false)))
.ReturnsAsync(allUsers.ToArray());
var moderationRepositoryMock = new Mock<IModerationRepository>(MockBehavior.Strict);

5
src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs
View File

@ -64,7 +64,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
It.Is<long>(y => y == tweets.Last().Id),
It.Is<long>(y => y == tweets.Last().Id),
It.Is<int>(y => y == 0),
It.IsAny<DateTime>()
It.IsAny<DateTime>(),
It.Is<string>(y => y == null),
It.Is<string>(y => y == null),
It.Is<bool>(y => y == false)
))
.Returns(Task.CompletedTask);

15
src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs
View File

@ -40,7 +40,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == maxUsers)))
It.Is<int>(y => y == maxUsers),
It.Is<bool>(y => y == false)))
.ReturnsAsync(users);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
@ -83,7 +84,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.SetupSequence(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == maxUsers)))
It.Is<int>(y => y == maxUsers),
It.Is<bool>(y => y == false)))
.ReturnsAsync(users.ToArray())
.ReturnsAsync(new SyncTwitterUser[0])
.ReturnsAsync(new SyncTwitterUser[0])
@ -130,7 +132,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.SetupSequence(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == maxUsers)))
It.Is<int>(y => y == maxUsers),
It.Is<bool>(y => y == false)))
.ReturnsAsync(users.ToArray())
.ReturnsAsync(new SyncTwitterUser[0])
.ReturnsAsync(new SyncTwitterUser[0])
@ -178,7 +181,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == maxUsers)))
It.Is<int>(y => y == maxUsers),
It.Is<bool>(y => y == false)))
.ReturnsAsync(new SyncTwitterUser[0]);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
@ -215,7 +219,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == maxUsers)))
It.Is<int>(y => y == maxUsers),
It.Is<bool>(y => y == false)))
.Returns(async () => await DelayFaultedTask<SyncTwitterUser[]>(new Exception()));
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();

20
src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs
View File

@ -66,7 +66,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
It.Is<long>(y => y == tweet2.Id),
It.Is<long>(y => y == tweet2.Id),
It.Is<int>(y => y == 0),
It.IsAny<DateTime>()
It.IsAny<DateTime>(),
It.Is<string>(y => y == null),
It.Is<string>(y => y == null),
It.Is<bool>(y => y == false)
))
.Returns(Task.CompletedTask);
@ -133,7 +136,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
It.Is<long>(y => y == tweet2.Id),
It.Is<long>(y => y == tweet2.Id),
It.Is<int>(y => y == 0),
It.IsAny<DateTime>()
It.IsAny<DateTime>(),
It.Is<string>(y => y == null),
It.Is<string>(y => y == null),
It.Is<bool>(y => y == false)
))
.Throws(new ArgumentException());
@ -202,7 +208,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
It.Is<long>(y => y == tweet3.Id),
It.Is<long>(y => y == tweet2.Id),
It.Is<int>(y => y == 0),
It.IsAny<DateTime>()
It.IsAny<DateTime>(),
It.Is<string>(y => y == null),
It.Is<string>(y => y == null),
It.Is<bool>(y => y == false)
))
.Returns(Task.CompletedTask);
@ -281,7 +290,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
It.Is<long>(y => y == tweet3.Id),
It.Is<long>(y => y == tweet2.Id),
It.Is<int>(y => y == 0),
It.IsAny<DateTime>()
It.IsAny<DateTime>(),
It.Is<string>(y => y == null),
It.Is<string>(y => y == null),
It.Is<bool>(y => y == false)
))
.Returns(Task.CompletedTask);