From 93b43ee4a0f7e56b37897a1f61dfe19f52552010 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Thu, 9 Dec 2021 02:02:30 -0500 Subject: [PATCH 01/19] added achitecture to handle Delete action --- .../ApDeserializer.cs | 3 +++ .../Models/ActivityDelete.cs | 10 +++++++++ src/BirdsiteLive.Domain/UserService.cs | 17 ++++++++++++++- .../Controllers/InboxController.cs | 21 ++++++++++++++++++- .../Controllers/UsersController.cs | 18 +++++++++------- src/BirdsiteLive/Tools/HeaderHandler.cs | 15 +++++++++++++ .../ApDeserializerTests.cs | 17 ++++++++++++++- 7 files changed, 91 insertions(+), 10 deletions(-) create mode 100644 src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs create mode 100644 src/BirdsiteLive/Tools/HeaderHandler.cs diff --git a/src/BirdsiteLive.ActivityPub/ApDeserializer.cs b/src/BirdsiteLive.ActivityPub/ApDeserializer.cs index 17dadbe..169bbe6 100644 --- a/src/BirdsiteLive.ActivityPub/ApDeserializer.cs +++ b/src/BirdsiteLive.ActivityPub/ApDeserializer.cs @@ -1,4 +1,5 @@ using System; +using BirdsiteLive.ActivityPub.Models; using Newtonsoft.Json; namespace BirdsiteLive.ActivityPub @@ -19,6 +20,8 @@ namespace BirdsiteLive.ActivityPub if(a.apObject.type == "Follow") return JsonConvert.DeserializeObject(json); break; + case "Delete": + return JsonConvert.DeserializeObject(json); case "Accept": var accept = JsonConvert.DeserializeObject(json); //var acceptType = JsonConvert.DeserializeObject(accept.apObject); diff --git a/src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs b/src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs new file mode 100644 index 0000000..deb7e7f --- /dev/null +++ b/src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs @@ -0,0 +1,10 @@ +using Newtonsoft.Json; + +namespace BirdsiteLive.ActivityPub.Models +{ + public class ActivityDelete : Activity + { + [JsonProperty("object")] + public object apObject { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index 24c8287..fedfcca 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -7,6 +7,7 @@ using System.Text; using System.Threading.Tasks; using BirdsiteLive.ActivityPub; using BirdsiteLive.ActivityPub.Converters; +using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.Common.Regexes; using BirdsiteLive.Common.Settings; using BirdsiteLive.Cryptography; @@ -28,6 +29,7 @@ namespace BirdsiteLive.Domain Task UndoFollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityUndoFollow activity, string body); Task SendRejectFollowAsync(ActivityFollow activity, string followerHost); + Task DeleteRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityDelete activity, string body); } public class UserService : IUserService @@ -213,7 +215,7 @@ namespace BirdsiteLive.Domain return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; //TODO: revamp this for better error handling } - + private string OnlyKeepRoute(string inbox, string host) { if (string.IsNullOrWhiteSpace(inbox)) @@ -258,6 +260,19 @@ namespace BirdsiteLive.Domain return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; //TODO: revamp this for better error handling } + public async Task DeleteRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, + ActivityDelete activity, string body) + { + // Validate + var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body); + if (!sigValidation.SignatureIsValidated) return false; + + // Remove user and followings + throw new NotImplementedException(); + + return true; + } + private async Task ValidateSignature(string actor, string rawSig, string method, string path, string queryString, Dictionary requestHeaders, string body) { //Check Date Validity diff --git a/src/BirdsiteLive/Controllers/InboxController.cs b/src/BirdsiteLive/Controllers/InboxController.cs index db055cf..f55e22b 100644 --- a/src/BirdsiteLive/Controllers/InboxController.cs +++ b/src/BirdsiteLive/Controllers/InboxController.cs @@ -3,6 +3,10 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading.Tasks; +using BirdsiteLive.ActivityPub; +using BirdsiteLive.ActivityPub.Models; +using BirdsiteLive.Domain; +using BirdsiteLive.Tools; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; @@ -13,11 +17,13 @@ namespace BirdsiteLive.Controllers public class InboxController : ControllerBase { private readonly ILogger _logger; + private readonly IUserService _userService; #region Ctor - public InboxController(ILogger logger) + public InboxController(ILogger logger, IUserService userService) { _logger = logger; + _userService = userService; } #endregion @@ -33,6 +39,19 @@ namespace BirdsiteLive.Controllers _logger.LogTrace("Inbox: {Body}", body); //System.IO.File.WriteAllText($@"C:\apdebug\inbox\{Guid.NewGuid()}.json", body); + var activity = ApDeserializer.ProcessActivity(body); + var signature = r.Headers["Signature"].First(); + + switch (activity?.type) + { + case "Delete": + { + var succeeded = await _userService.DeleteRequestedAsync(signature, r.Method, r.Path, + r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), activity as ActivityDelete, body); + if (succeeded) return Accepted(); + else return Unauthorized(); + } + } } return Accepted(); diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index 0d5f77b..73be8b0 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -13,6 +13,7 @@ using BirdsiteLive.Common.Regexes; using BirdsiteLive.Common.Settings; using BirdsiteLive.Domain; using BirdsiteLive.Models; +using BirdsiteLive.Tools; using BirdsiteLive.Twitter; using BirdsiteLive.Twitter.Models; using Microsoft.AspNetCore.Http; @@ -142,7 +143,6 @@ namespace BirdsiteLive.Controllers //System.IO.File.WriteAllText($@"C:\apdebug\{Guid.NewGuid()}.json", body); var activity = ApDeserializer.ProcessActivity(body); - // Do something var signature = r.Headers["Signature"].First(); switch (activity?.type) @@ -150,7 +150,7 @@ namespace BirdsiteLive.Controllers case "Follow": { var succeeded = await _userService.FollowRequestedAsync(signature, r.Method, r.Path, - r.QueryString.ToString(), RequestHeaders(r.Headers), activity as ActivityFollow, body); + r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), activity as ActivityFollow, body); if (succeeded) return Accepted(); else return Unauthorized(); } @@ -158,11 +158,18 @@ namespace BirdsiteLive.Controllers if (activity is ActivityUndoFollow) { var succeeded = await _userService.UndoFollowRequestedAsync(signature, r.Method, r.Path, - r.QueryString.ToString(), RequestHeaders(r.Headers), activity as ActivityUndoFollow, body); + r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), activity as ActivityUndoFollow, body); if (succeeded) return Accepted(); else return Unauthorized(); } return Accepted(); + case "Delete": + { + var succeeded = await _userService.DeleteRequestedAsync(signature, r.Method, r.Path, + r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), activity as ActivityDelete, body); + if (succeeded) return Accepted(); + else return Unauthorized(); + } default: return Accepted(); } @@ -184,9 +191,6 @@ namespace BirdsiteLive.Controllers return Content(jsonApUser, "application/activity+json; charset=utf-8"); } - private Dictionary RequestHeaders(IHeaderDictionary header) - { - return header.ToDictionary, string, string>(h => h.Key.ToLowerInvariant(), h => h.Value); - } + } } \ No newline at end of file diff --git a/src/BirdsiteLive/Tools/HeaderHandler.cs b/src/BirdsiteLive/Tools/HeaderHandler.cs new file mode 100644 index 0000000..74ecf29 --- /dev/null +++ b/src/BirdsiteLive/Tools/HeaderHandler.cs @@ -0,0 +1,15 @@ +using System.Collections.Generic; +using System.Linq; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Primitives; + +namespace BirdsiteLive.Tools +{ + public class HeaderHandler + { + public static Dictionary RequestHeaders(IHeaderDictionary header) + { + return header.ToDictionary, string, string>(h => h.Key.ToLowerInvariant(), h => h.Value); + } + } +} \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.ActivityPub.Tests/ApDeserializerTests.cs b/src/Tests/BirdsiteLive.ActivityPub.Tests/ApDeserializerTests.cs index 3c85113..3d64e90 100644 --- a/src/Tests/BirdsiteLive.ActivityPub.Tests/ApDeserializerTests.cs +++ b/src/Tests/BirdsiteLive.ActivityPub.Tests/ApDeserializerTests.cs @@ -1,4 +1,5 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; +using BirdsiteLive.ActivityPub.Models; +using Microsoft.VisualStudio.TestTools.UnitTesting; using Newtonsoft.Json; namespace BirdsiteLive.ActivityPub.Tests @@ -48,6 +49,20 @@ namespace BirdsiteLive.ActivityPub.Tests Assert.AreEqual("https://mamot.fr/users/testtest", data.apObject.apObject); } + [TestMethod] + public void DeleteDeserializationTest() + { + var json = + "{\"@context\": \"https://www.w3.org/ns/activitystreams\", \"id\": \"https://mastodon.technology/users/deleteduser#delete\", \"type\": \"Delete\", \"actor\": \"https://mastodon.technology/users/deleteduser\", \"to\": [\"https://www.w3.org/ns/activitystreams#Public\"],\"object\": \"https://mastodon.technology/users/deleteduser\",\"signature\": {\"type\": \"RsaSignature2017\",\"creator\": \"https://mastodon.technology/users/deleteduser#main-key\",\"created\": \"2020-11-19T22:43:01Z\",\"signatureValue\": \"peksQao4v5N+sMZgHXZ6xZnGaZrd0s+LqZimu63cnp7O5NBJM6gY9AAu/vKUgrh4C50r66f9OQdHg5yChQhc4ViE+yLR/3/e59YQimelmXJPpcC99Nt0YLU/iTRLsBehY3cDdC6+ogJKgpkToQvB6tG2KrPdrkreYh4Il4eXLKMfiQhgdKluOvenLnl2erPWfE02hIu/jpuljyxSuvJunMdU4yQVSZHTtk/I8q3jjzIzhgyb7ICWU5Hkx0H/47Q24ztsvOgiTWNgO+v6l9vA7qIhztENiRPhzGP5RCCzUKRAe6bcSu1Wfa3NKWqB9BeJ7s+2y2bD7ubPbiEE1MQV7Q==\"}}"; + + var data = ApDeserializer.ProcessActivity(json) as ActivityDelete; + + Assert.AreEqual("https://mastodon.technology/users/deleteduser#delete", data.id); + Assert.AreEqual("Delete", data.type); + Assert.AreEqual("https://mastodon.technology/users/deleteduser", data.actor); + Assert.AreEqual("https://mastodon.technology/users/deleteduser", data.apObject); + } + //[TestMethod] //public void NoteDeserializationTest() //{ From 7205a09eaa9410399a54935dbe02114e8aa2699b Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Mon, 13 Dec 2021 20:43:57 -0500 Subject: [PATCH 02/19] added Delete logic --- .../BusinessUseCases/ProcessDeleteUser.cs | 51 ++++++++++ .../Tools/SigValidationResultExtractor.cs | 22 +++++ src/BirdsiteLive.Domain/UserService.cs | 15 ++- .../Actions/RemoveFollowerAction.cs | 29 ++---- .../ProcessDeleteUserTests.cs | 97 +++++++++++++++++++ .../Actions/RemoveFollowerActionTests.cs | 54 +++-------- 6 files changed, 199 insertions(+), 69 deletions(-) create mode 100644 src/BirdsiteLive.Domain/BusinessUseCases/ProcessDeleteUser.cs create mode 100644 src/BirdsiteLive.Domain/Tools/SigValidationResultExtractor.cs create mode 100644 src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessDeleteUserTests.cs diff --git a/src/BirdsiteLive.Domain/BusinessUseCases/ProcessDeleteUser.cs b/src/BirdsiteLive.Domain/BusinessUseCases/ProcessDeleteUser.cs new file mode 100644 index 0000000..a35b6c8 --- /dev/null +++ b/src/BirdsiteLive.Domain/BusinessUseCases/ProcessDeleteUser.cs @@ -0,0 +1,51 @@ +using System.Linq; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; + +namespace BirdsiteLive.Domain.BusinessUseCases +{ + public interface IProcessDeleteUser + { + Task ExecuteAsync(Follower follower); + Task ExecuteAsync(string followerUsername, string followerDomain); + } + + public class ProcessDeleteUser : IProcessDeleteUser + { + private readonly IFollowersDal _followersDal; + private readonly ITwitterUserDal _twitterUserDal; + + #region Ctor + public ProcessDeleteUser(IFollowersDal followersDal, ITwitterUserDal twitterUserDal) + { + _followersDal = followersDal; + _twitterUserDal = twitterUserDal; + } + #endregion + + public async Task ExecuteAsync(string followerUsername, string followerDomain) + { + // Get Follower and Twitter Users + var follower = await _followersDal.GetFollowerAsync(followerUsername, followerDomain); + if (follower == null) return; + + await ExecuteAsync(follower); + } + + public async Task ExecuteAsync(Follower follower) + { + // Remove twitter users if no more followers + var followings = follower.Followings; + foreach (var following in followings) + { + var followers = await _followersDal.GetFollowersAsync(following); + if (followers.Length == 1 && followers.First().Id == follower.Id) + await _twitterUserDal.DeleteTwitterUserAsync(following); + } + + // Remove follower from DB + await _followersDal.DeleteFollowerAsync(follower.Id); + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/Tools/SigValidationResultExtractor.cs b/src/BirdsiteLive.Domain/Tools/SigValidationResultExtractor.cs new file mode 100644 index 0000000..1d6890a --- /dev/null +++ b/src/BirdsiteLive.Domain/Tools/SigValidationResultExtractor.cs @@ -0,0 +1,22 @@ +using System.Linq; + +namespace BirdsiteLive.Domain.Tools +{ + public class SigValidationResultExtractor + { + public static string GetUserName(SignatureValidationResult result) + { + return result.User.preferredUsername.ToLowerInvariant().Trim(); + } + + public static string GetHost(SignatureValidationResult result) + { + return result.User.url.Replace("https://", string.Empty).Split('/').First(); + } + + public static string GetSharedInbox(SignatureValidationResult result) + { + return result.User?.endpoints?.sharedInbox; + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index fedfcca..a080180 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -34,6 +34,7 @@ namespace BirdsiteLive.Domain public class UserService : IUserService { + private readonly IProcessDeleteUser _processDeleteUser; private readonly IProcessFollowUser _processFollowUser; private readonly IProcessUndoFollowUser _processUndoFollowUser; @@ -48,7 +49,7 @@ namespace BirdsiteLive.Domain private readonly IModerationRepository _moderationRepository; #region Ctor - public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler, ITwitterUserService twitterUserService, IModerationRepository moderationRepository) + public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler, ITwitterUserService twitterUserService, IModerationRepository moderationRepository, IProcessDeleteUser processDeleteUser) { _instanceSettings = instanceSettings; _cryptoService = cryptoService; @@ -59,6 +60,7 @@ namespace BirdsiteLive.Domain _statisticsHandler = statisticsHandler; _twitterUserService = twitterUserService; _moderationRepository = moderationRepository; + _processDeleteUser = processDeleteUser; } #endregion @@ -128,10 +130,10 @@ namespace BirdsiteLive.Domain if (!sigValidation.SignatureIsValidated) return false; // Prepare data - var followerUserName = sigValidation.User.preferredUsername.ToLowerInvariant().Trim(); - var followerHost = sigValidation.User.url.Replace("https://", string.Empty).Split('/').First(); + var followerUserName = SigValidationResultExtractor.GetUserName(sigValidation); + var followerHost = SigValidationResultExtractor.GetHost(sigValidation); var followerInbox = sigValidation.User.inbox; - var followerSharedInbox = sigValidation.User?.endpoints?.sharedInbox; + var followerSharedInbox = SigValidationResultExtractor.GetSharedInbox(sigValidation); var twitterUser = activity.apObject.Split('/').Last().Replace("@", string.Empty).ToLowerInvariant().Trim(); // Make sure to only keep routes @@ -268,7 +270,10 @@ namespace BirdsiteLive.Domain if (!sigValidation.SignatureIsValidated) return false; // Remove user and followings - throw new NotImplementedException(); + var followerUserName = SigValidationResultExtractor.GetUserName(sigValidation); + var followerHost = SigValidationResultExtractor.GetHost(sigValidation); + + await _processDeleteUser.ExecuteAsync(followerUserName, followerHost); return true; } diff --git a/src/BirdsiteLive.Moderation/Actions/RemoveFollowerAction.cs b/src/BirdsiteLive.Moderation/Actions/RemoveFollowerAction.cs index 8ab3132..4721154 100644 --- a/src/BirdsiteLive.Moderation/Actions/RemoveFollowerAction.cs +++ b/src/BirdsiteLive.Moderation/Actions/RemoveFollowerAction.cs @@ -1,12 +1,6 @@ -using System; -using System.Linq; -using System.Threading.Tasks; -using BirdsiteLive.ActivityPub; -using BirdsiteLive.ActivityPub.Converters; -using BirdsiteLive.Common.Settings; -using BirdsiteLive.DAL.Contracts; +using System.Threading.Tasks; using BirdsiteLive.DAL.Models; -using BirdsiteLive.Domain; +using BirdsiteLive.Domain.BusinessUseCases; namespace BirdsiteLive.Moderation.Actions { @@ -17,16 +11,14 @@ namespace BirdsiteLive.Moderation.Actions public class RemoveFollowerAction : IRemoveFollowerAction { - private readonly IFollowersDal _followersDal; - private readonly ITwitterUserDal _twitterUserDal; private readonly IRejectAllFollowingsAction _rejectAllFollowingsAction; + private readonly IProcessDeleteUser _processDeleteUser; #region Ctor - public RemoveFollowerAction(IFollowersDal followersDal, ITwitterUserDal twitterUserDal, IRejectAllFollowingsAction rejectAllFollowingsAction) + public RemoveFollowerAction(IRejectAllFollowingsAction rejectAllFollowingsAction, IProcessDeleteUser processDeleteUser) { - _followersDal = followersDal; - _twitterUserDal = twitterUserDal; _rejectAllFollowingsAction = rejectAllFollowingsAction; + _processDeleteUser = processDeleteUser; } #endregion @@ -36,16 +28,7 @@ namespace BirdsiteLive.Moderation.Actions await _rejectAllFollowingsAction.ProcessAsync(follower); // Remove twitter users if no more followers - var followings = follower.Followings; - foreach (var following in followings) - { - var followers = await _followersDal.GetFollowersAsync(following); - if (followers.Length == 1 && followers.First().Id == follower.Id) - await _twitterUserDal.DeleteTwitterUserAsync(following); - } - - // Remove follower from DB - await _followersDal.DeleteFollowerAsync(follower.Id); + await _processDeleteUser.ExecuteAsync(follower); } } } \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessDeleteUserTests.cs b/src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessDeleteUserTests.cs new file mode 100644 index 0000000..85900da --- /dev/null +++ b/src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessDeleteUserTests.cs @@ -0,0 +1,97 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Domain.BusinessUseCases; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace BirdsiteLive.Domain.Tests.BusinessUseCases +{ + [TestClass] + public class ProcessDeleteUserTests + { + [TestMethod] + public async Task ExecuteAsync_NoMoreFollowings() + { + #region Stubs + var follower = new Follower + { + Id = 12, + Followings = new List { 1 } + }; + #endregion + + #region Mocks + var followersDalMock = new Mock(MockBehavior.Strict); + followersDalMock + .Setup(x => x.GetFollowersAsync( + It.Is(y => y == 1))) + .ReturnsAsync(new[] { follower }); + + followersDalMock + .Setup(x => x.DeleteFollowerAsync( + It.Is(y => y == 12))) + .Returns(Task.CompletedTask); + + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.DeleteTwitterUserAsync( + It.Is(y => y == 1))) + .Returns(Task.CompletedTask); + #endregion + + var action = new ProcessDeleteUser(followersDalMock.Object, twitterUserDalMock.Object); + await action.ExecuteAsync(follower); + + #region Validations + followersDalMock.VerifyAll(); + twitterUserDalMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ExecuteAsync_HaveFollowings() + { + #region Stubs + var follower = new Follower + { + Id = 12, + Followings = new List { 1 } + }; + + var followers = new List + { + follower, + new Follower + { + Id = 11 + } + }; + #endregion + + #region Mocks + var followersDalMock = new Mock(MockBehavior.Strict); + followersDalMock + .Setup(x => x.GetFollowersAsync( + It.Is(y => y == 1))) + .ReturnsAsync(followers.ToArray()); + + followersDalMock + .Setup(x => x.DeleteFollowerAsync( + It.Is(y => y == 12))) + .Returns(Task.CompletedTask); + + var twitterUserDalMock = new Mock(MockBehavior.Strict); + #endregion + + var action = new ProcessDeleteUser(followersDalMock.Object, twitterUserDalMock.Object); + await action.ExecuteAsync(follower); + + #region Validations + followersDalMock.VerifyAll(); + twitterUserDalMock.VerifyAll(); + #endregion + } + } +} \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Moderation.Tests/Actions/RemoveFollowerActionTests.cs b/src/Tests/BirdsiteLive.Moderation.Tests/Actions/RemoveFollowerActionTests.cs index 3b83739..34f40b2 100644 --- a/src/Tests/BirdsiteLive.Moderation.Tests/Actions/RemoveFollowerActionTests.cs +++ b/src/Tests/BirdsiteLive.Moderation.Tests/Actions/RemoveFollowerActionTests.cs @@ -2,6 +2,7 @@ using System.Threading.Tasks; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; +using BirdsiteLive.Domain.BusinessUseCases; using BirdsiteLive.Moderation.Actions; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -29,31 +30,19 @@ namespace BirdsiteLive.Moderation.Tests.Actions It.Is(y => y.Id == follower.Id))) .Returns(Task.CompletedTask); - var followersDalMock = new Mock(MockBehavior.Strict); - followersDalMock - .Setup(x => x.GetFollowersAsync( - It.Is(y => y == 1))) - .ReturnsAsync(new[] {follower}); - - followersDalMock - .Setup(x => x.DeleteFollowerAsync( - It.Is(y => y == 12))) - .Returns(Task.CompletedTask); - - var twitterUserDalMock = new Mock(MockBehavior.Strict); - twitterUserDalMock - .Setup(x => x.DeleteTwitterUserAsync( - It.Is(y => y == 1))) + var processDeleteUserMock = new Mock(MockBehavior.Strict); + processDeleteUserMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Id == follower.Id))) .Returns(Task.CompletedTask); #endregion - var action = new RemoveFollowerAction(followersDalMock.Object, twitterUserDalMock.Object, rejectAllFollowingsActionMock.Object); + var action = new RemoveFollowerAction(rejectAllFollowingsActionMock.Object, processDeleteUserMock.Object); await action.ProcessAsync(follower); #region Validations - followersDalMock.VerifyAll(); - twitterUserDalMock.VerifyAll(); rejectAllFollowingsActionMock.VerifyAll(); + processDeleteUserMock.VerifyAll(); #endregion } @@ -66,15 +55,6 @@ namespace BirdsiteLive.Moderation.Tests.Actions Id = 12, Followings = new List { 1 } }; - - var followers = new List - { - follower, - new Follower - { - Id = 11 - } - }; #endregion #region Mocks @@ -84,27 +64,19 @@ namespace BirdsiteLive.Moderation.Tests.Actions It.Is(y => y.Id == follower.Id))) .Returns(Task.CompletedTask); - var followersDalMock = new Mock(MockBehavior.Strict); - followersDalMock - .Setup(x => x.GetFollowersAsync( - It.Is(y => y == 1))) - .ReturnsAsync(followers.ToArray()); - - followersDalMock - .Setup(x => x.DeleteFollowerAsync( - It.Is(y => y == 12))) + var processDeleteUserMock = new Mock(MockBehavior.Strict); + processDeleteUserMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Id == follower.Id))) .Returns(Task.CompletedTask); - - var twitterUserDalMock = new Mock(MockBehavior.Strict); #endregion - var action = new RemoveFollowerAction(followersDalMock.Object, twitterUserDalMock.Object, rejectAllFollowingsActionMock.Object); + var action = new RemoveFollowerAction(rejectAllFollowingsActionMock.Object, processDeleteUserMock.Object); await action.ProcessAsync(follower); #region Validations - followersDalMock.VerifyAll(); - twitterUserDalMock.VerifyAll(); rejectAllFollowingsActionMock.VerifyAll(); + processDeleteUserMock.VerifyAll(); #endregion } } From a36171c16352530b7284c3125391ce1e7fe5d497 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Mon, 13 Dec 2021 23:29:40 -0500 Subject: [PATCH 03/19] road to 0.20.0 --- src/BirdsiteLive/BirdsiteLive.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index a30e4e0..d4e7466 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -4,7 +4,7 @@ netcoreapp3.1 d21486de-a812-47eb-a419-05682bb68856 Linux - 0.19.1 + 0.20.0 From 5c4641c6ae15092aac84381a581f6fc48209033c Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Thu, 27 Jan 2022 19:58:35 -0500 Subject: [PATCH 04/19] disable debuging features on release --- src/BirdsiteLive/Controllers/DebugingController.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/BirdsiteLive/Controllers/DebugingController.cs b/src/BirdsiteLive/Controllers/DebugingController.cs index 252486e..00accef 100644 --- a/src/BirdsiteLive/Controllers/DebugingController.cs +++ b/src/BirdsiteLive/Controllers/DebugingController.cs @@ -14,6 +14,7 @@ using Newtonsoft.Json; namespace BirdsiteLive.Controllers { + #if DEBUG public class DebugingController : Controller { private readonly InstanceSettings _instanceSettings; @@ -67,7 +68,7 @@ namespace BirdsiteLive.Controllers var noteGuid = Guid.NewGuid(); var noteId = $"https://{_instanceSettings.Domain}/users/{username}/statuses/{noteGuid}"; var noteUrl = $"https://{_instanceSettings.Domain}/@{username}/{noteGuid}"; - + var to = $"{actor}/followers"; var now = DateTime.UtcNow; @@ -80,12 +81,12 @@ namespace BirdsiteLive.Controllers type = "Create", actor = actor, published = nowString, - to = new []{ to }, + to = new[] { to }, //cc = new [] { "https://www.w3.org/ns/activitystreams#Public" }, apObject = new Note() { id = noteId, - summary = null, + summary = null, inReplyTo = null, published = nowString, url = noteUrl, @@ -93,7 +94,7 @@ namespace BirdsiteLive.Controllers // Unlisted to = new[] { to }, - cc = new [] { "https://www.w3.org/ns/activitystreams#Public" }, + cc = new[] { "https://www.w3.org/ns/activitystreams#Public" }, //// Public //to = new[] { "https://www.w3.org/ns/activitystreams#Public" }, @@ -125,6 +126,7 @@ namespace BirdsiteLive.Controllers return View("Index"); } } + #endif public static class HtmlHelperExtensions { From 26cca6a306f01ee2726223e56cbe41bfe5ea7dbf Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Thu, 27 Jan 2022 22:52:59 -0500 Subject: [PATCH 05/19] upgrade failing counter to integer --- .../DbInitializerPostgresDal.cs | 13 +++++- .../FollowersPostgresDalTests.cs | 42 +++++++++++++++++++ .../TwitterUserPostgresDalTests.cs | 32 ++++++++++++++ 3 files changed, 85 insertions(+), 2 deletions(-) diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs index 2e3acea..2f9cb54 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs @@ -23,7 +23,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public class DbInitializerPostgresDal : PostgresBase, IDbInitializerDal { private readonly PostgresTools _tools; - private readonly Version _currentVersion = new Version(2, 3); + private readonly Version _currentVersion = new Version(2, 4); private const string DbVersionType = "db-version"; #region Ctor @@ -134,7 +134,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers new Tuple(new Version(1,0), new Version(2,0)), new Tuple(new Version(2,0), new Version(2,1)), new Tuple(new Version(2,1), new Version(2,2)), - new Tuple(new Version(2,2), new Version(2,3)) + new Tuple(new Version(2,2), new Version(2,3)), + new Tuple(new Version(2,3), new Version(2,4)) }; } @@ -163,6 +164,14 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers var addPostingError = $@"ALTER TABLE {_settings.FollowersTableName} ADD postingErrorCount SMALLINT"; await _tools.ExecuteRequestAsync(addPostingError); } + else if (from == new Version(2, 3) && to == new Version(2, 4)) + { + var alterLastSync = $@"ALTER TABLE {_settings.TwitterUserTableName} ALTER COLUMN fetchingErrorCount TYPE INTEGER"; + await _tools.ExecuteRequestAsync(alterLastSync); + + var alterPostingError = $@"ALTER TABLE {_settings.FollowersTableName} ALTER COLUMN postingErrorCount TYPE INTEGER"; + await _tools.ExecuteRequestAsync(alterPostingError); + } else { throw new NotImplementedException(); diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs index a22df0f..3927f13 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs @@ -340,6 +340,48 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers Assert.AreEqual(10, result.PostingErrorCount); } + [TestMethod] + public async Task CreateUpdateAndGetFollower_Integer() + { + var acct = "myhandle"; + var host = "domain.ext"; + var following = new[] { 12, 19, 23 }; + var followingSync = new Dictionary() + { + {12, 165L}, + {19, 166L}, + {23, 167L} + }; + var inboxRoute = "/myhandle/inbox"; + var sharedInboxRoute = "/inbox"; + var actorId = $"https://{host}/{acct}"; + + var dal = new FollowersPostgresDal(_settings); + await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync); + var result = await dal.GetFollowerAsync(acct, host); + + var updatedFollowing = new List { 12, 19, 23, 24 }; + var updatedFollowingSync = new Dictionary(){ + {12, 170L}, + {19, 171L}, + {23, 172L}, + {24, 173L} + }; + result.Followings = updatedFollowing.ToList(); + result.FollowingsSyncStatus = updatedFollowingSync; + result.PostingErrorCount = 32768; + + await dal.UpdateFollowerAsync(result); + result = await dal.GetFollowerAsync(acct, host); + + Assert.AreEqual(updatedFollowing.Count, result.Followings.Count); + Assert.AreEqual(updatedFollowing[0], result.Followings[0]); + Assert.AreEqual(updatedFollowingSync.Count, result.FollowingsSyncStatus.Count); + Assert.AreEqual(updatedFollowingSync.First().Key, result.FollowingsSyncStatus.First().Key); + Assert.AreEqual(updatedFollowingSync.First().Value, result.FollowingsSyncStatus.First().Value); + Assert.AreEqual(32768, result.PostingErrorCount); + } + [TestMethod] public async Task CreateUpdateAndGetFollower_Remove() { diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs index 0b007b6..c9bc746 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs @@ -130,6 +130,38 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100); } + [TestMethod] + public async Task CreateUpdate3AndGetUser() + { + var acct = "myid"; + var lastTweetId = 1548L; + + var dal = new TwitterUserPostgresDal(_settings); + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + var result = await dal.GetTwitterUserAsync(acct); + + + var updatedLastTweetId = 1600L; + var updatedLastSyncId = 1550L; + var now = DateTime.Now; + var errors = 32768; + + result.LastTweetPostedId = updatedLastTweetId; + result.LastTweetSynchronizedForAllFollowersId = updatedLastSyncId; + result.FetchingErrorCount = errors; + result.LastSync = now; + await dal.UpdateTwitterUserAsync(result); + + result = await dal.GetTwitterUserAsync(acct); + + Assert.AreEqual(acct, result.Acct); + Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId); + Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId); + Assert.AreEqual(errors, result.FetchingErrorCount); + Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100); + } + [TestMethod] [ExpectedException(typeof(ArgumentException))] public async Task Update_NoId() From 3a998b60acb5fdb576e2c2df09d386e7cfa437d3 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 2 Feb 2022 21:33:45 -0500 Subject: [PATCH 06/19] added auto clean-up on failing follower --- .../Settings/InstanceSettings.cs | 1 + .../SendTweetsToFollowersProcessor.cs | 20 +- src/BirdsiteLive/appsettings.json | 3 +- .../SendTweetsToFollowersProcessorTests.cs | 250 +++++++++++++++++- 4 files changed, 260 insertions(+), 14 deletions(-) diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs index a85d9e6..eb77d1f 100644 --- a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs +++ b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs @@ -13,5 +13,6 @@ public string SensitiveTwitterAccounts { get; set; } public int FailingTwitterUserCleanUpThreshold { get; set; } + public int FailingFollowerCleanUpThreshold { get; set; } = -1; } } diff --git a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs index 65f9610..e210f39 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs @@ -5,9 +5,11 @@ using System.Net; using System.Threading; using System.Threading.Tasks; using System.Xml; +using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Domain; +using BirdsiteLive.Moderation.Actions; using BirdsiteLive.Pipeline.Contracts; using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Pipeline.Processors.SubTasks; @@ -23,14 +25,18 @@ namespace BirdsiteLive.Pipeline.Processors private readonly ISendTweetsToInboxTask _sendTweetsToInboxTask; private readonly ISendTweetsToSharedInboxTask _sendTweetsToSharedInbox; private readonly IFollowersDal _followersDal; + private readonly InstanceSettings _instanceSettings; private readonly ILogger _logger; + private readonly IRemoveFollowerAction _removeFollowerAction; #region Ctor - public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox, IFollowersDal followersDal, ILogger logger) + public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox, IFollowersDal followersDal, ILogger logger, InstanceSettings instanceSettings, IRemoveFollowerAction removeFollowerAction) { _sendTweetsToInboxTask = sendTweetsToInboxTask; _sendTweetsToSharedInbox = sendTweetsToSharedInbox; _logger = logger; + _instanceSettings = instanceSettings; + _removeFollowerAction = removeFollowerAction; _followersDal = followersDal; } #endregion @@ -107,7 +113,17 @@ namespace BirdsiteLive.Pipeline.Processors private async Task ProcessFailingUserAsync(Follower follower) { follower.PostingErrorCount++; - await _followersDal.UpdateFollowerAsync(follower); + + if (follower.PostingErrorCount > _instanceSettings.FailingFollowerCleanUpThreshold + && _instanceSettings.FailingFollowerCleanUpThreshold > 0 + || follower.PostingErrorCount > 2147483600) + { + await _removeFollowerAction.ProcessAsync(follower); + } + else + { + await _followersDal.UpdateFollowerAsync(follower); + } } } } \ No newline at end of file diff --git a/src/BirdsiteLive/appsettings.json b/src/BirdsiteLive/appsettings.json index d510809..b49d25c 100644 --- a/src/BirdsiteLive/appsettings.json +++ b/src/BirdsiteLive/appsettings.json @@ -23,7 +23,8 @@ "MaxUsersCapacity": 1000, "UnlistedTwitterAccounts": null, "SensitiveTwitterAccounts": null, - "FailingTwitterUserCleanUpThreshold": 700 + "FailingTwitterUserCleanUpThreshold": 700, + "FailingFollowerCleanUpThreshold": 30000 }, "Db": { "Type": "postgres", diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs index 53aa12a..8a78038 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs @@ -2,8 +2,10 @@ using System.Threading; using System.Threading.Tasks; using System.Xml; +using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; +using BirdsiteLive.Moderation.Actions; using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Pipeline.Processors; using BirdsiteLive.Pipeline.Processors.SubTasks; @@ -72,17 +74,22 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); var followersDalMock = new Mock(MockBehavior.Strict); - + var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -147,15 +154,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var followersDalMock = new Mock(MockBehavior.Strict); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -229,15 +241,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -312,15 +329,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -400,15 +422,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -471,15 +498,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var followersDalMock = new Mock(MockBehavior.Strict); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -543,15 +575,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var followersDalMock = new Mock(MockBehavior.Strict); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -623,15 +660,196 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_MultiInstances_Inbox_OneTweet_Error_SettingsThreshold_Test() + { + #region Stubs + var tweetId = 1; + var host1 = "domain1.ext"; + var host2 = "domain2.ext"; + var inbox = "/user/inbox"; + var userId1 = 2; + var userId2 = 3; + var userAcct = "user"; + + var userWithTweets = new UserWithDataToSync() + { + Tweets = new[] + { + new ExtractedTweet + { + Id = tweetId + } + }, + User = new SyncTwitterUser + { + Acct = userAcct + }, + Followers = new[] + { + new Follower + { + Id = userId1, + Host = host1, + InboxRoute = inbox + }, + new Follower + { + Id = userId2, + Host = host2, + InboxRoute = inbox, + PostingErrorCount = 42 + }, + } + }; + #endregion + + #region Mocks + var sendTweetsToInboxTaskMock = new Mock(MockBehavior.Strict); + sendTweetsToInboxTaskMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Length == 1), + It.Is(y => y.Id == userId1), + It.Is(y => y.Acct == userAcct))) + .Returns(Task.CompletedTask); + + sendTweetsToInboxTaskMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Length == 1), + It.Is(y => y.Id == userId2), + It.Is(y => y.Acct == userAcct))) + .Throws(new Exception()); + + var sendTweetsToSharedInboxTaskMock = new Mock(MockBehavior.Strict); + + var followersDalMock = new Mock(MockBehavior.Strict); + + var loggerMock = new Mock>(); + + var settings = new InstanceSettings + { + FailingFollowerCleanUpThreshold = 10 + }; + + var removeFollowerMock = new Mock(MockBehavior.Strict); + removeFollowerMock + .Setup(x => x.ProcessAsync(It.Is(y => y.Id == userId2))) + .Returns(Task.CompletedTask); + #endregion + + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); + var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); + + #region Validations + sendTweetsToInboxTaskMock.VerifyAll(); + sendTweetsToSharedInboxTaskMock.VerifyAll(); + followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_MultiInstances_Inbox_OneTweet_Error_MaxThreshold_Test() + { + #region Stubs + var tweetId = 1; + var host1 = "domain1.ext"; + var host2 = "domain2.ext"; + var inbox = "/user/inbox"; + var userId1 = 2; + var userId2 = 3; + var userAcct = "user"; + + var userWithTweets = new UserWithDataToSync() + { + Tweets = new[] + { + new ExtractedTweet + { + Id = tweetId + } + }, + User = new SyncTwitterUser + { + Acct = userAcct + }, + Followers = new[] + { + new Follower + { + Id = userId1, + Host = host1, + InboxRoute = inbox + }, + new Follower + { + Id = userId2, + Host = host2, + InboxRoute = inbox, + PostingErrorCount = 2147483600 + }, + } + }; + #endregion + + #region Mocks + var sendTweetsToInboxTaskMock = new Mock(MockBehavior.Strict); + sendTweetsToInboxTaskMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Length == 1), + It.Is(y => y.Id == userId1), + It.Is(y => y.Acct == userAcct))) + .Returns(Task.CompletedTask); + + sendTweetsToInboxTaskMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Length == 1), + It.Is(y => y.Id == userId2), + It.Is(y => y.Acct == userAcct))) + .Throws(new Exception()); + + var sendTweetsToSharedInboxTaskMock = new Mock(MockBehavior.Strict); + + var followersDalMock = new Mock(MockBehavior.Strict); + + var loggerMock = new Mock>(); + + var settings = new InstanceSettings + { + FailingFollowerCleanUpThreshold = 0 + }; + + var removeFollowerMock = new Mock(MockBehavior.Strict); + removeFollowerMock + .Setup(x => x.ProcessAsync(It.Is(y => y.Id == userId2))) + .Returns(Task.CompletedTask); + #endregion + + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); + var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); + + #region Validations + sendTweetsToInboxTaskMock.VerifyAll(); + sendTweetsToSharedInboxTaskMock.VerifyAll(); + followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -704,15 +922,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -790,15 +1013,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } } From 15d7e87466b3d9bcb57b00363a286abafc754bed Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 2 Feb 2022 21:47:02 -0500 Subject: [PATCH 07/19] added FailingFollowerCleanUpThreshold variable --- VARIABLES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/VARIABLES.md b/VARIABLES.md index e9f6f33..57d854c 100644 --- a/VARIABLES.md +++ b/VARIABLES.md @@ -49,6 +49,7 @@ If both whitelisting and blacklisting are set, only the whitelisting will be act * `Instance:UnlistedTwitterAccounts` (default: null) to enable unlisted publication for selected twitter accounts, separated by `;` (please limit this to brands and other public profiles). * `Instance:SensitiveTwitterAccounts` (default: null) mark all media from given accounts as sensitive by default, separated by `;`. * `Instance:FailingTwitterUserCleanUpThreshold` (default: 700) set the max allowed errors (due to a banned/deleted/private account) from a Twitter Account retrieval before auto-removal. (by default an account is called every 15 mins) +* `Instance:FailingFollowerCleanUpThreshold` (default: 30000) set the max allowed errors from a Follower (Fediverse) Account before auto-removal. (often due to account suppression, instance issues, etc) # Docker Compose full example From c371218672c006489ea629be68cf608ebb0a4a41 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 2 Feb 2022 23:25:03 -0500 Subject: [PATCH 08/19] prevent saturation of the user retrieval API --- .../Statistics/TwitterStatisticsHandler.cs | 9 ++++++++- src/BirdsiteLive.Twitter/TwitterUserService.cs | 9 ++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs b/src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs index 668be76..8063985 100644 --- a/src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs +++ b/src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs @@ -13,6 +13,8 @@ namespace BirdsiteLive.Statistics.Domain void CalledTweetApi(); void CalledTimelineApi(); ApiStatistics GetStatistics(); + + int GetCurrentUserCalls(); } //Rate limits: https://developer.twitter.com/en/docs/twitter-api/v1/rate-limits @@ -60,7 +62,12 @@ namespace BirdsiteLive.Statistics.Domain foreach (var old in oldSnapshots) _snapshots.TryRemove(old, out var data); } - public void CalledUserApi() //GET users/show - 900/15mins + public int GetCurrentUserCalls() + { + return _userCalls; + } + + public void CalledUserApi() //GET users/show - 300/15mins { Interlocked.Increment(ref _userCalls); } diff --git a/src/BirdsiteLive.Twitter/TwitterUserService.cs b/src/BirdsiteLive.Twitter/TwitterUserService.cs index 6a27dc1..df6e5ad 100644 --- a/src/BirdsiteLive.Twitter/TwitterUserService.cs +++ b/src/BirdsiteLive.Twitter/TwitterUserService.cs @@ -32,6 +32,12 @@ namespace BirdsiteLive.Twitter public TwitterUser GetUser(string username) { + //Check if API is saturated + var currentCalls = _statisticsHandler.GetCurrentUserCalls(); + var maxCalls = _statisticsHandler.GetStatistics().UserCallsMax; + if (currentCalls > maxCalls) return null; + + //Proceed to account retrieval _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized(); ExceptionHandler.SwallowWebExceptions = false; @@ -49,9 +55,6 @@ namespace BirdsiteLive.Twitter catch (Exception e) { _logger.LogError(e, "Error retrieving user {Username}", username); - - // TODO keep track of error, see where to remove user if too much errors - return null; } From bf7baba789f592770532ca2dbd3b58877ffb1cb3 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Thu, 3 Feb 2022 01:06:32 -0500 Subject: [PATCH 09/19] added proper return on TooManyRequest case --- src/BirdsiteLive/Controllers/UsersController.cs | 15 +++++++++++---- src/BirdsiteLive/Views/Users/ApiSaturated.cshtml | 13 +++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 src/BirdsiteLive/Views/Users/ApiSaturated.cshtml diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index 73be8b0..aa4f272 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -13,6 +13,7 @@ using BirdsiteLive.Common.Regexes; using BirdsiteLive.Common.Settings; using BirdsiteLive.Domain; using BirdsiteLive.Models; +using BirdsiteLive.Statistics.Domain; using BirdsiteLive.Tools; using BirdsiteLive.Twitter; using BirdsiteLive.Twitter.Models; @@ -32,9 +33,10 @@ namespace BirdsiteLive.Controllers private readonly IStatusService _statusService; private readonly InstanceSettings _instanceSettings; private readonly ILogger _logger; + private readonly ITwitterStatisticsHandler _twitterStatisticsHandler; #region Ctor - public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ITwitterTweetsService twitterTweetService, ILogger logger) + public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ITwitterTweetsService twitterTweetService, ILogger logger, ITwitterStatisticsHandler twitterStatisticsHandler) { _twitterUserService = twitterUserService; _userService = userService; @@ -42,6 +44,7 @@ namespace BirdsiteLive.Controllers _instanceSettings = instanceSettings; _twitterTweetService = twitterTweetService; _logger = logger; + _twitterStatisticsHandler = twitterStatisticsHandler; } #endregion @@ -72,12 +75,17 @@ namespace BirdsiteLive.Controllers if (!string.IsNullOrWhiteSpace(id) && UserRegexes.TwitterAccount.IsMatch(id) && id.Length <= 15) user = _twitterUserService.GetUser(id); + var isSaturated = user == null + && _twitterStatisticsHandler.GetCurrentUserCalls() >= + _twitterStatisticsHandler.GetStatistics().UserCallsMax; + var acceptHeaders = Request.Headers["Accept"]; if (acceptHeaders.Any()) { var r = acceptHeaders.First(); if (r.Contains("application/activity+json")) { + if (user == null && isSaturated) return new ObjectResult("Too Many Requests") { StatusCode = 429 }; if (user == null) return NotFound(); var apUser = _userService.GetUser(user); var jsonApUser = JsonConvert.SerializeObject(apUser); @@ -85,8 +93,9 @@ namespace BirdsiteLive.Controllers } } + if (user == null && isSaturated) return View("ApiSaturated"); if (user == null) return View("UserNotFound"); - + var displayableUser = new DisplayTwitterUser { Name = user.Name, @@ -190,7 +199,5 @@ namespace BirdsiteLive.Controllers var jsonApUser = JsonConvert.SerializeObject(followers); return Content(jsonApUser, "application/activity+json; charset=utf-8"); } - - } } \ No newline at end of file diff --git a/src/BirdsiteLive/Views/Users/ApiSaturated.cshtml b/src/BirdsiteLive/Views/Users/ApiSaturated.cshtml new file mode 100644 index 0000000..1f807aa --- /dev/null +++ b/src/BirdsiteLive/Views/Users/ApiSaturated.cshtml @@ -0,0 +1,13 @@ +@using BirdsiteLive.Controllers; +@{ + ViewData["Title"] = "Api Saturated"; +} + +
+

429 Too Many Requests

+

+
+ The API is saturated.
+ Please consider using another instance. +

+
\ No newline at end of file From 1536880c73cceff11b8b0b62e2d571c29fbdea33 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Thu, 3 Feb 2022 19:01:21 -0500 Subject: [PATCH 10/19] set the cache limits from settings --- VARIABLES.md | 1 + .../Settings/InstanceSettings.cs | 2 ++ src/BirdsiteLive.Twitter/CachedTwitterService.cs | 15 +++++++++------ src/BirdsiteLive/appsettings.json | 3 ++- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/VARIABLES.md b/VARIABLES.md index 57d854c..2ef0cdf 100644 --- a/VARIABLES.md +++ b/VARIABLES.md @@ -50,6 +50,7 @@ If both whitelisting and blacklisting are set, only the whitelisting will be act * `Instance:SensitiveTwitterAccounts` (default: null) mark all media from given accounts as sensitive by default, separated by `;`. * `Instance:FailingTwitterUserCleanUpThreshold` (default: 700) set the max allowed errors (due to a banned/deleted/private account) from a Twitter Account retrieval before auto-removal. (by default an account is called every 15 mins) * `Instance:FailingFollowerCleanUpThreshold` (default: 30000) set the max allowed errors from a Follower (Fediverse) Account before auto-removal. (often due to account suppression, instance issues, etc) +* `Instance:UserCacheCapacity` (default: 10000) set the caching limit of the Twitter User retrieval. Must be higher than the number of synchronized accounts on the instance. # Docker Compose full example diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs index eb77d1f..0f701ff 100644 --- a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs +++ b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs @@ -14,5 +14,7 @@ public int FailingTwitterUserCleanUpThreshold { get; set; } public int FailingFollowerCleanUpThreshold { get; set; } = -1; + + public int UserCacheCapacity { get; set; } } } diff --git a/src/BirdsiteLive.Twitter/CachedTwitterService.cs b/src/BirdsiteLive.Twitter/CachedTwitterService.cs index ac20a62..ef5b0e5 100644 --- a/src/BirdsiteLive.Twitter/CachedTwitterService.cs +++ b/src/BirdsiteLive.Twitter/CachedTwitterService.cs @@ -1,4 +1,5 @@ using System; +using BirdsiteLive.Common.Settings; using BirdsiteLive.Twitter.Models; using Microsoft.Extensions.Caching.Memory; @@ -13,11 +14,8 @@ namespace BirdsiteLive.Twitter { private readonly ITwitterUserService _twitterService; - private MemoryCache _userCache = new MemoryCache(new MemoryCacheOptions() - { - SizeLimit = 5000 - }); - private MemoryCacheEntryOptions _cacheEntryOptions = new MemoryCacheEntryOptions() + private readonly MemoryCache _userCache; + private readonly MemoryCacheEntryOptions _cacheEntryOptions = new MemoryCacheEntryOptions() .SetSize(1)//Size amount //Priority on removing when reaching size limit (memory pressure) .SetPriority(CacheItemPriority.High) @@ -27,9 +25,14 @@ namespace BirdsiteLive.Twitter .SetAbsoluteExpiration(TimeSpan.FromDays(7)); #region Ctor - public CachedTwitterUserService(ITwitterUserService twitterService) + public CachedTwitterUserService(ITwitterUserService twitterService, InstanceSettings settings) { _twitterService = twitterService; + + _userCache = new MemoryCache(new MemoryCacheOptions() + { + SizeLimit = settings.UserCacheCapacity + }); } #endregion diff --git a/src/BirdsiteLive/appsettings.json b/src/BirdsiteLive/appsettings.json index b49d25c..4cee8ba 100644 --- a/src/BirdsiteLive/appsettings.json +++ b/src/BirdsiteLive/appsettings.json @@ -24,7 +24,8 @@ "UnlistedTwitterAccounts": null, "SensitiveTwitterAccounts": null, "FailingTwitterUserCleanUpThreshold": 700, - "FailingFollowerCleanUpThreshold": 30000 + "FailingFollowerCleanUpThreshold": 30000, + "UserCacheCapacity": 10000 }, "Db": { "Type": "postgres", From c043e0b6a07b366ca0b9da83c187efd0935e984f Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Thu, 3 Feb 2022 19:45:25 -0500 Subject: [PATCH 11/19] get rate limit from API --- .../CachedTwitterService.cs | 5 +++ .../TwitterUserService.cs | 33 +++++++++++++++++-- .../Controllers/UsersController.cs | 9 ++--- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/BirdsiteLive.Twitter/CachedTwitterService.cs b/src/BirdsiteLive.Twitter/CachedTwitterService.cs index ac20a62..4ee3e04 100644 --- a/src/BirdsiteLive.Twitter/CachedTwitterService.cs +++ b/src/BirdsiteLive.Twitter/CachedTwitterService.cs @@ -44,6 +44,11 @@ namespace BirdsiteLive.Twitter return user; } + public bool IsUserApiRateLimited() + { + return _twitterService.IsUserApiRateLimited(); + } + public void PurgeUser(string username) { _userCache.Remove(username); diff --git a/src/BirdsiteLive.Twitter/TwitterUserService.cs b/src/BirdsiteLive.Twitter/TwitterUserService.cs index df6e5ad..adc8d6b 100644 --- a/src/BirdsiteLive.Twitter/TwitterUserService.cs +++ b/src/BirdsiteLive.Twitter/TwitterUserService.cs @@ -13,6 +13,7 @@ namespace BirdsiteLive.Twitter public interface ITwitterUserService { TwitterUser GetUser(string username); + bool IsUserApiRateLimited(); } public class TwitterUserService : ITwitterUserService @@ -33,13 +34,12 @@ namespace BirdsiteLive.Twitter public TwitterUser GetUser(string username) { //Check if API is saturated - var currentCalls = _statisticsHandler.GetCurrentUserCalls(); - var maxCalls = _statisticsHandler.GetStatistics().UserCallsMax; - if (currentCalls > maxCalls) return null; + if (IsUserApiRateLimited()) return null; //Proceed to account retrieval _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized(); ExceptionHandler.SwallowWebExceptions = false; + RateLimit.RateLimitTrackerMode = RateLimitTrackerMode.TrackOnly; IUser user; try @@ -76,5 +76,32 @@ namespace BirdsiteLive.Twitter Protected = user.Protected }; } + + public bool IsUserApiRateLimited() + { + // Retrieve limit from tooling + _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized(); + ExceptionHandler.SwallowWebExceptions = false; + RateLimit.RateLimitTrackerMode = RateLimitTrackerMode.TrackOnly; + + try + { + var queryRateLimits = RateLimit.GetQueryRateLimit("https://api.twitter.com/1.1/users/show.json?screen_name=mastodon"); + + if (queryRateLimits != null) + { + return queryRateLimits.Remaining <= 0; + } + } + catch (Exception e) + { + _logger.LogError(e, "Error retrieving rate limits"); + } + + // Fallback + var currentCalls = _statisticsHandler.GetCurrentUserCalls(); + var maxCalls = _statisticsHandler.GetStatistics().UserCallsMax; + return currentCalls >= maxCalls; + } } } \ No newline at end of file diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index aa4f272..965f988 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -13,7 +13,6 @@ using BirdsiteLive.Common.Regexes; using BirdsiteLive.Common.Settings; using BirdsiteLive.Domain; using BirdsiteLive.Models; -using BirdsiteLive.Statistics.Domain; using BirdsiteLive.Tools; using BirdsiteLive.Twitter; using BirdsiteLive.Twitter.Models; @@ -33,10 +32,9 @@ namespace BirdsiteLive.Controllers private readonly IStatusService _statusService; private readonly InstanceSettings _instanceSettings; private readonly ILogger _logger; - private readonly ITwitterStatisticsHandler _twitterStatisticsHandler; #region Ctor - public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ITwitterTweetsService twitterTweetService, ILogger logger, ITwitterStatisticsHandler twitterStatisticsHandler) + public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ITwitterTweetsService twitterTweetService, ILogger logger) { _twitterUserService = twitterUserService; _userService = userService; @@ -44,7 +42,6 @@ namespace BirdsiteLive.Controllers _instanceSettings = instanceSettings; _twitterTweetService = twitterTweetService; _logger = logger; - _twitterStatisticsHandler = twitterStatisticsHandler; } #endregion @@ -75,9 +72,7 @@ namespace BirdsiteLive.Controllers if (!string.IsNullOrWhiteSpace(id) && UserRegexes.TwitterAccount.IsMatch(id) && id.Length <= 15) user = _twitterUserService.GetUser(id); - var isSaturated = user == null - && _twitterStatisticsHandler.GetCurrentUserCalls() >= - _twitterStatisticsHandler.GetStatistics().UserCallsMax; + var isSaturated = _twitterUserService.IsUserApiRateLimited(); var acceptHeaders = Request.Headers["Accept"]; if (acceptHeaders.Any()) From 662f97e53c1efb5595aecc35c2531e49921868d6 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Mon, 7 Feb 2022 18:48:10 -0500 Subject: [PATCH 12/19] added proper exception in user retrieval --- .../RefreshTwitterUserStatusProcessor.cs | 68 ++++++--- .../Exceptions/RateLimitExceededException.cs | 9 ++ .../UserHasBeenSuspendedException.cs | 9 ++ .../Exceptions/UserNotFoundException.cs | 9 ++ .../TwitterUserService.cs | 28 +++- .../Controllers/UsersController.cs | 132 ++++++++++++------ .../Controllers/WellKnownController.cs | 29 +++- 7 files changed, 212 insertions(+), 72 deletions(-) create mode 100644 src/BirdsiteLive.Twitter/Exceptions/RateLimitExceededException.cs create mode 100644 src/BirdsiteLive.Twitter/Exceptions/UserHasBeenSuspendedException.cs create mode 100644 src/BirdsiteLive.Twitter/Exceptions/UserNotFoundException.cs diff --git a/src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs index 3a36be3..3d29d67 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs @@ -9,6 +9,7 @@ using BirdsiteLive.Moderation.Actions; using BirdsiteLive.Pipeline.Contracts; using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Twitter; +using BirdsiteLive.Twitter.Models; namespace BirdsiteLive.Pipeline.Processors { @@ -35,26 +36,58 @@ namespace BirdsiteLive.Pipeline.Processors foreach (var user in syncTwitterUsers) { - var userView = _twitterUserService.GetUser(user.Acct); - if (userView == null) - { - await AnalyseFailingUserAsync(user); - } - else if (!userView.Protected) - { - user.FetchingErrorCount = 0; - var userWtData = new UserWithDataToSync - { - User = user - }; - usersWtData.Add(userWtData); - } - } + TwitterUser userView = null; + try + { + userView = _twitterUserService.GetUser(user.Acct); + } + catch (UserNotFoundException) + { + await ProcessNotFoundUserAsync(user); + } + catch (UserHasBeenSuspendedException) + { + await ProcessNotFoundUserAsync(user); + } + catch (RateLimitExceededException) + { + await ProcessRateLimitExceededAsync(user); + } + catch (Exception) + { + // ignored + } + + if (userView == null || userView.Protected) + { + await ProcessFailingUserAsync(user); + continue; + } + + user.FetchingErrorCount = 0; + var userWtData = new UserWithDataToSync + { + User = user + }; + usersWtData.Add(userWtData); + } return usersWtData.ToArray(); } - private async Task AnalyseFailingUserAsync(SyncTwitterUser user) + private async Task ProcessRateLimitExceededAsync(SyncTwitterUser user) + { + var dbUser = await _twitterUserDal.GetTwitterUserAsync(user.Acct); + dbUser.LastSync = DateTime.UtcNow; + await _twitterUserDal.UpdateTwitterUserAsync(dbUser); + } + + private async Task ProcessNotFoundUserAsync(SyncTwitterUser user) + { + await _removeTwitterAccountAction.ProcessAsync(user); + } + + private async Task ProcessFailingUserAsync(SyncTwitterUser user) { var dbUser = await _twitterUserDal.GetTwitterUserAsync(user.Acct); dbUser.FetchingErrorCount++; @@ -68,9 +101,6 @@ namespace BirdsiteLive.Pipeline.Processors { await _twitterUserDal.UpdateTwitterUserAsync(dbUser); } - - // Purge - _twitterUserService.PurgeUser(user.Acct); } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/Exceptions/RateLimitExceededException.cs b/src/BirdsiteLive.Twitter/Exceptions/RateLimitExceededException.cs new file mode 100644 index 0000000..93a093a --- /dev/null +++ b/src/BirdsiteLive.Twitter/Exceptions/RateLimitExceededException.cs @@ -0,0 +1,9 @@ +using System; + +namespace BirdsiteLive.Twitter +{ + public class RateLimitExceededException : Exception + { + + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/Exceptions/UserHasBeenSuspendedException.cs b/src/BirdsiteLive.Twitter/Exceptions/UserHasBeenSuspendedException.cs new file mode 100644 index 0000000..03bd835 --- /dev/null +++ b/src/BirdsiteLive.Twitter/Exceptions/UserHasBeenSuspendedException.cs @@ -0,0 +1,9 @@ +using System; + +namespace BirdsiteLive.Twitter +{ + public class UserHasBeenSuspendedException : Exception + { + + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/Exceptions/UserNotFoundException.cs b/src/BirdsiteLive.Twitter/Exceptions/UserNotFoundException.cs new file mode 100644 index 0000000..1dffc72 --- /dev/null +++ b/src/BirdsiteLive.Twitter/Exceptions/UserNotFoundException.cs @@ -0,0 +1,9 @@ +using System; + +namespace BirdsiteLive.Twitter +{ + public class UserNotFoundException : Exception + { + + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/TwitterUserService.cs b/src/BirdsiteLive.Twitter/TwitterUserService.cs index adc8d6b..7fb6439 100644 --- a/src/BirdsiteLive.Twitter/TwitterUserService.cs +++ b/src/BirdsiteLive.Twitter/TwitterUserService.cs @@ -6,6 +6,7 @@ using BirdsiteLive.Twitter.Models; using BirdsiteLive.Twitter.Tools; using Microsoft.Extensions.Logging; using Tweetinvi; +using Tweetinvi.Exceptions; using Tweetinvi.Models; namespace BirdsiteLive.Twitter @@ -45,17 +46,34 @@ namespace BirdsiteLive.Twitter try { user = User.GetUserFromScreenName(username); - _statisticsHandler.CalledUserApi(); - if (user == null) + } + catch (TwitterException e) + { + if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("User has been suspended".ToLowerInvariant()))) { - _logger.LogWarning("User {username} not found", username); - return null; + throw new UserHasBeenSuspendedException(); + } + else if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("User not found".ToLowerInvariant()))) + { + throw new UserNotFoundException(); + } + else if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("Rate limit exceeded".ToLowerInvariant()))) + { + throw new RateLimitExceededException(); + } + else + { + throw; } } catch (Exception e) { _logger.LogError(e, "Error retrieving user {Username}", username); - return null; + throw; + } + finally + { + _statisticsHandler.CalledUserApi(); } // Expand URLs diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index 965f988..f59ae21 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -66,13 +66,42 @@ namespace BirdsiteLive.Controllers id = id.Trim(new[] { ' ', '@' }).ToLowerInvariant(); + TwitterUser user = null; + var isSaturated = false; + var notFound = false; + // Ensure valid username // https://help.twitter.com/en/managing-your-account/twitter-username-rules - TwitterUser user = null; if (!string.IsNullOrWhiteSpace(id) && UserRegexes.TwitterAccount.IsMatch(id) && id.Length <= 15) - user = _twitterUserService.GetUser(id); + { + try + { + user = _twitterUserService.GetUser(id); + } + catch (UserNotFoundException) + { + notFound = true; + } + catch (UserHasBeenSuspendedException) + { + notFound = true; + } + catch (RateLimitExceededException) + { + isSaturated = true; + } + catch (Exception e) + { + _logger.LogError(e, "Exception getting {Id}", id); + throw; + } + } + else + { + notFound = true; + } - var isSaturated = _twitterUserService.IsUserApiRateLimited(); + //var isSaturated = _twitterUserService.IsUserApiRateLimited(); var acceptHeaders = Request.Headers["Accept"]; if (acceptHeaders.Any()) @@ -80,17 +109,17 @@ namespace BirdsiteLive.Controllers var r = acceptHeaders.First(); if (r.Contains("application/activity+json")) { - if (user == null && isSaturated) return new ObjectResult("Too Many Requests") { StatusCode = 429 }; - if (user == null) return NotFound(); + if (isSaturated) return new ObjectResult("Too Many Requests") { StatusCode = 429 }; + if (notFound) return NotFound(); var apUser = _userService.GetUser(user); var jsonApUser = JsonConvert.SerializeObject(apUser); return Content(jsonApUser, "application/activity+json; charset=utf-8"); } } - if (user == null && isSaturated) return View("ApiSaturated"); - if (user == null) return View("UserNotFound"); - + if (isSaturated) return View("ApiSaturated"); + if (notFound) return View("UserNotFound"); + var displayableUser = new DisplayTwitterUser { Name = user.Name, @@ -138,46 +167,61 @@ namespace BirdsiteLive.Controllers [HttpPost] public async Task Inbox() { - var r = Request; - using (var reader = new StreamReader(Request.Body)) + try { - var body = await reader.ReadToEndAsync(); - - _logger.LogTrace("User Inbox: {Body}", body); - //System.IO.File.WriteAllText($@"C:\apdebug\{Guid.NewGuid()}.json", body); - - var activity = ApDeserializer.ProcessActivity(body); - var signature = r.Headers["Signature"].First(); - - switch (activity?.type) + var r = Request; + using (var reader = new StreamReader(Request.Body)) { - case "Follow": - { - var succeeded = await _userService.FollowRequestedAsync(signature, r.Method, r.Path, - r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), activity as ActivityFollow, body); - if (succeeded) return Accepted(); - else return Unauthorized(); - } - case "Undo": - if (activity is ActivityUndoFollow) - { - var succeeded = await _userService.UndoFollowRequestedAsync(signature, r.Method, r.Path, - r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), activity as ActivityUndoFollow, body); - if (succeeded) return Accepted(); - else return Unauthorized(); - } - return Accepted(); - case "Delete": - { - var succeeded = await _userService.DeleteRequestedAsync(signature, r.Method, r.Path, - r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), activity as ActivityDelete, body); - if (succeeded) return Accepted(); - else return Unauthorized(); - } - default: - return Accepted(); + var body = await reader.ReadToEndAsync(); + + _logger.LogTrace("User Inbox: {Body}", body); + //System.IO.File.WriteAllText($@"C:\apdebug\{Guid.NewGuid()}.json", body); + + var activity = ApDeserializer.ProcessActivity(body); + var signature = r.Headers["Signature"].First(); + + switch (activity?.type) + { + case "Follow": + { + var succeeded = await _userService.FollowRequestedAsync(signature, r.Method, r.Path, + r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), activity as ActivityFollow, body); + if (succeeded) return Accepted(); + else return Unauthorized(); + } + case "Undo": + if (activity is ActivityUndoFollow) + { + var succeeded = await _userService.UndoFollowRequestedAsync(signature, r.Method, r.Path, + r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), activity as ActivityUndoFollow, body); + if (succeeded) return Accepted(); + else return Unauthorized(); + } + return Accepted(); + case "Delete": + { + var succeeded = await _userService.DeleteRequestedAsync(signature, r.Method, r.Path, + r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), activity as ActivityDelete, body); + if (succeeded) return Accepted(); + else return Unauthorized(); + } + default: + return Accepted(); + } } } + catch (UserNotFoundException) + { + return NotFound(); + } + catch (UserHasBeenSuspendedException) + { + return NotFound(); + } + catch (RateLimitExceededException) + { + return new ObjectResult("Too Many Requests") { StatusCode = 429 }; + } } [Route("/users/{id}/followers")] diff --git a/src/BirdsiteLive/Controllers/WellKnownController.cs b/src/BirdsiteLive/Controllers/WellKnownController.cs index 501f783..272d789 100644 --- a/src/BirdsiteLive/Controllers/WellKnownController.cs +++ b/src/BirdsiteLive/Controllers/WellKnownController.cs @@ -12,6 +12,7 @@ using BirdsiteLive.Models; using BirdsiteLive.Models.WellKnownModels; using BirdsiteLive.Twitter; using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace BirdsiteLive.Controllers @@ -23,13 +24,15 @@ namespace BirdsiteLive.Controllers private readonly ITwitterUserService _twitterUserService; private readonly ITwitterUserDal _twitterUserDal; private readonly InstanceSettings _settings; - + private readonly ILogger _logger; + #region Ctor - public WellKnownController(InstanceSettings settings, ITwitterUserService twitterUserService, ITwitterUserDal twitterUserDal, IModerationRepository moderationRepository) + public WellKnownController(InstanceSettings settings, ITwitterUserService twitterUserService, ITwitterUserDal twitterUserDal, IModerationRepository moderationRepository, ILogger logger) { _twitterUserService = twitterUserService; _twitterUserDal = twitterUserDal; _moderationRepository = moderationRepository; + _logger = logger; _settings = settings; } #endregion @@ -174,9 +177,27 @@ namespace BirdsiteLive.Controllers if (!string.IsNullOrWhiteSpace(domain) && domain != _settings.Domain) return NotFound(); - var user = _twitterUserService.GetUser(name); - if (user == null) + try + { + _twitterUserService.GetUser(name); + } + catch (UserNotFoundException) + { return NotFound(); + } + catch (UserHasBeenSuspendedException) + { + return NotFound(); + } + catch (RateLimitExceededException) + { + return new ObjectResult("Too Many Requests") { StatusCode = 429 }; + } + catch (Exception e) + { + _logger.LogError(e, "Exception getting {Name}", name); + throw; + } var actorUrl = UrlFactory.GetActorUrl(_settings.Domain, name); From d1c5a592471857bbd592000c54762a107cd0804f Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Mon, 7 Feb 2022 19:33:08 -0500 Subject: [PATCH 13/19] fix tests --- .../RefreshTwitterUserStatusProcessor.cs | 3 + .../RefreshTwitterUserStatusProcessorTests.cs | 323 ++++++++++++++++-- 2 files changed, 306 insertions(+), 20 deletions(-) diff --git a/src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs index 3d29d67..739d50b 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs @@ -45,14 +45,17 @@ namespace BirdsiteLive.Pipeline.Processors catch (UserNotFoundException) { await ProcessNotFoundUserAsync(user); + continue; } catch (UserHasBeenSuspendedException) { await ProcessNotFoundUserAsync(user); + continue; } catch (RateLimitExceededException) { await ProcessRateLimitExceededAsync(user); + continue; } catch (Exception) { diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs index d5fbeef..ae4994f 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs @@ -159,25 +159,14 @@ namespace BirdsiteLive.Pipeline.Tests.Processors twitterUserServiceMock .Setup(x => x.GetUser(It.Is(y => y == acct2))) - .Returns((TwitterUser) null); - - twitterUserServiceMock - .Setup(x => x.PurgeUser(It.Is(y => y == acct2))); + .Throws(new UserNotFoundException()); var twitterUserDalMock = new Mock(MockBehavior.Strict); - twitterUserDalMock - .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) - .ReturnsAsync(new SyncTwitterUser - { - Id = userId2, - FetchingErrorCount = 0 - }); - - twitterUserDalMock - .Setup(x => x.UpdateTwitterUserAsync(It.Is(y => y.Id == userId2 && y.FetchingErrorCount == 1))) - .Returns(Task.CompletedTask); var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); + removeTwitterAccountActionMock + .Setup(x => x.ProcessAsync(It.Is(y => y.Acct == acct2))) + .Returns(Task.CompletedTask); #endregion var processor = new RefreshTwitterUserStatusProcessor(twitterUserServiceMock.Object, twitterUserDalMock.Object, removeTwitterAccountActionMock.Object, settings); @@ -194,7 +183,71 @@ namespace BirdsiteLive.Pipeline.Tests.Processors } [TestMethod] - public async Task ProcessAsync_Unfound_OverThreshold_Test() + public async Task ProcessAsync_Suspended_Test() + { + #region Stubs + var userId1 = 1; + var acct1 = "user1"; + + var userId2 = 2; + var acct2 = "user2"; + + var users = new List + { + new SyncTwitterUser + { + Id = userId1, + Acct = acct1 + }, + new SyncTwitterUser + { + Id = userId2, + Acct = acct2 + } + }; + + var settings = new InstanceSettings + { + FailingTwitterUserCleanUpThreshold = 300 + }; + #endregion + + #region Mocks + var twitterUserServiceMock = new Mock(MockBehavior.Strict); + twitterUserServiceMock + .Setup(x => x.GetUser(It.Is(y => y == acct1))) + .Returns(new TwitterUser + { + Protected = false + }); + + twitterUserServiceMock + .Setup(x => x.GetUser(It.Is(y => y == acct2))) + .Throws(new UserHasBeenSuspendedException()); + + var twitterUserDalMock = new Mock(MockBehavior.Strict); + + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); + removeTwitterAccountActionMock + .Setup(x => x.ProcessAsync(It.Is(y => y.Acct == acct2))) + .Returns(Task.CompletedTask); + #endregion + + var processor = new RefreshTwitterUserStatusProcessor(twitterUserServiceMock.Object, twitterUserDalMock.Object, removeTwitterAccountActionMock.Object, settings); + var result = await processor.ProcessAsync(users.ToArray(), CancellationToken.None); + + #region Validations + Assert.AreEqual(1, result.Length); + Assert.IsTrue(result.Any(x => x.User.Id == userId1)); + + twitterUserServiceMock.VerifyAll(); + twitterUserDalMock.VerifyAll(); + removeTwitterAccountActionMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_Error_Test() { #region Stubs var userId1 = 1; @@ -236,8 +289,83 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Setup(x => x.GetUser(It.Is(y => y == acct2))) .Returns((TwitterUser)null); + //twitterUserServiceMock + // .Setup(x => x.PurgeUser(It.Is(y => y == acct2))); + + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) + .ReturnsAsync(new SyncTwitterUser + { + Id = userId2, + FetchingErrorCount = 0 + }); + + twitterUserDalMock + .Setup(x => x.UpdateTwitterUserAsync(It.Is(y => y.Id == userId2 && y.FetchingErrorCount == 1))) + .Returns(Task.CompletedTask); + + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); + #endregion + + var processor = new RefreshTwitterUserStatusProcessor(twitterUserServiceMock.Object, twitterUserDalMock.Object, removeTwitterAccountActionMock.Object, settings); + var result = await processor.ProcessAsync(users.ToArray(), CancellationToken.None); + + #region Validations + Assert.AreEqual(1, result.Length); + Assert.IsTrue(result.Any(x => x.User.Id == userId1)); + + twitterUserServiceMock.VerifyAll(); + twitterUserDalMock.VerifyAll(); + removeTwitterAccountActionMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_Error_OverThreshold_Test() + { + #region Stubs + var userId1 = 1; + var acct1 = "user1"; + + var userId2 = 2; + var acct2 = "user2"; + + var users = new List + { + new SyncTwitterUser + { + Id = userId1, + Acct = acct1 + }, + new SyncTwitterUser + { + Id = userId2, + Acct = acct2 + } + }; + + var settings = new InstanceSettings + { + FailingTwitterUserCleanUpThreshold = 300 + }; + #endregion + + #region Mocks + var twitterUserServiceMock = new Mock(MockBehavior.Strict); twitterUserServiceMock - .Setup(x => x.PurgeUser(It.Is(y => y == acct2))); + .Setup(x => x.GetUser(It.Is(y => y == acct1))) + .Returns(new TwitterUser + { + Protected = false + }); + + twitterUserServiceMock + .Setup(x => x.GetUser(It.Is(y => y == acct2))) + .Returns((TwitterUser)null); + + //twitterUserServiceMock + // .Setup(x => x.PurgeUser(It.Is(y => y == acct2))); var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock @@ -313,7 +441,22 @@ namespace BirdsiteLive.Pipeline.Tests.Processors Protected = true }); + //twitterUserServiceMock + // .Setup(x => x.PurgeUser(It.Is(y => y == acct2))); + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) + .ReturnsAsync(new SyncTwitterUser + { + Id = userId2, + FetchingErrorCount = 0 + }); + + twitterUserDalMock + .Setup(x => x.UpdateTwitterUserAsync(It.Is(y => y.Id == userId2 && y.FetchingErrorCount == 1))) + .Returns(Task.CompletedTask); + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); #endregion @@ -331,7 +474,147 @@ namespace BirdsiteLive.Pipeline.Tests.Processors } [TestMethod] - public async Task ProcessAsync_Unfound_NotInit_Test() + public async Task ProcessAsync_Protected_OverThreshold_Test() + { + #region Stubs + var userId1 = 1; + var acct1 = "user1"; + + var userId2 = 2; + var acct2 = "user2"; + + var users = new List + { + new SyncTwitterUser + { + Id = userId1, + Acct = acct1 + }, + new SyncTwitterUser + { + Id = userId2, + Acct = acct2 + } + }; + + var settings = new InstanceSettings + { + FailingTwitterUserCleanUpThreshold = 300 + }; + #endregion + + #region Mocks + var twitterUserServiceMock = new Mock(MockBehavior.Strict); + twitterUserServiceMock + .Setup(x => x.GetUser(It.Is(y => y == acct1))) + .Returns(new TwitterUser + { + Protected = false + }); + + twitterUserServiceMock + .Setup(x => x.GetUser(It.Is(y => y == acct2))) + .Returns(new TwitterUser + { + Protected = true + }); + + //twitterUserServiceMock + // .Setup(x => x.PurgeUser(It.Is(y => y == acct2))); + + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) + .ReturnsAsync(new SyncTwitterUser + { + Id = userId2, + FetchingErrorCount = 500 + }); + + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); + removeTwitterAccountActionMock + .Setup(x => x.ProcessAsync(It.Is(y => y.Id == userId2))) + .Returns(Task.CompletedTask); + #endregion + + var processor = new RefreshTwitterUserStatusProcessor(twitterUserServiceMock.Object, twitterUserDalMock.Object, removeTwitterAccountActionMock.Object, settings); + var result = await processor.ProcessAsync(users.ToArray(), CancellationToken.None); + + #region Validations + Assert.AreEqual(1, result.Length); + Assert.IsTrue(result.Any(x => x.User.Id == userId1)); + + twitterUserServiceMock.VerifyAll(); + twitterUserDalMock.VerifyAll(); + removeTwitterAccountActionMock.VerifyAll(); + #endregion + } + + //[TestMethod] + //public async Task ProcessAsync_Protected_Test() + //{ + // #region Stubs + // var userId1 = 1; + // var acct1 = "user1"; + + // var userId2 = 2; + // var acct2 = "user2"; + + // var users = new List + // { + // new SyncTwitterUser + // { + // Id = userId1, + // Acct = acct1 + // }, + // new SyncTwitterUser + // { + // Id = userId2, + // Acct = acct2 + // } + // }; + + // var settings = new InstanceSettings + // { + // FailingTwitterUserCleanUpThreshold = 300 + // }; + // #endregion + + // #region Mocks + // var twitterUserServiceMock = new Mock(MockBehavior.Strict); + // twitterUserServiceMock + // .Setup(x => x.GetUser(It.Is(y => y == acct1))) + // .Returns(new TwitterUser + // { + // Protected = false + // }); + + // twitterUserServiceMock + // .Setup(x => x.GetUser(It.Is(y => y == acct2))) + // .Returns(new TwitterUser + // { + // Protected = true + // }); + + // var twitterUserDalMock = new Mock(MockBehavior.Strict); + // var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); + // #endregion + + // var processor = new RefreshTwitterUserStatusProcessor(twitterUserServiceMock.Object, twitterUserDalMock.Object, removeTwitterAccountActionMock.Object, settings); + // var result = await processor.ProcessAsync(users.ToArray(), CancellationToken.None); + + // #region Validations + // Assert.AreEqual(1, result.Length); + // Assert.IsTrue(result.Any(x => x.User.Id == userId1)); + + // twitterUserServiceMock.VerifyAll(); + // twitterUserDalMock.VerifyAll(); + // removeTwitterAccountActionMock.VerifyAll(); + // #endregion + //} + + [TestMethod] + public async Task ProcessAsync_Error_NotInit_Test() { #region Stubs var userId1 = 1; @@ -361,8 +644,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Setup(x => x.GetUser(It.Is(y => y == acct1))) .Returns((TwitterUser)null); - twitterUserServiceMock - .Setup(x => x.PurgeUser(It.Is(y => y == acct1))); + //twitterUserServiceMock + // .Setup(x => x.PurgeUser(It.Is(y => y == acct1))); var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock From 420d8867e7ea8377bcc569e66b08817ba93ee2f6 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Mon, 7 Feb 2022 19:36:19 -0500 Subject: [PATCH 14/19] added test for new behavior --- .../RefreshTwitterUserStatusProcessorTests.cs | 153 +++++++++++++++++- 1 file changed, 152 insertions(+), 1 deletion(-) diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs index ae4994f..be342c0 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -246,6 +247,81 @@ namespace BirdsiteLive.Pipeline.Tests.Processors #endregion } + [TestMethod] + public async Task ProcessAsync_Exception_Test() + { + #region Stubs + var userId1 = 1; + var acct1 = "user1"; + + var userId2 = 2; + var acct2 = "user2"; + + var users = new List + { + new SyncTwitterUser + { + Id = userId1, + Acct = acct1 + }, + new SyncTwitterUser + { + Id = userId2, + Acct = acct2 + } + }; + + var settings = new InstanceSettings + { + FailingTwitterUserCleanUpThreshold = 300 + }; + #endregion + + #region Mocks + var twitterUserServiceMock = new Mock(MockBehavior.Strict); + twitterUserServiceMock + .Setup(x => x.GetUser(It.Is(y => y == acct1))) + .Returns(new TwitterUser + { + Protected = false + }); + + twitterUserServiceMock + .Setup(x => x.GetUser(It.Is(y => y == acct2))) + .Throws(new Exception()); + + //twitterUserServiceMock + // .Setup(x => x.PurgeUser(It.Is(y => y == acct2))); + + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) + .ReturnsAsync(new SyncTwitterUser + { + Id = userId2, + FetchingErrorCount = 0 + }); + + twitterUserDalMock + .Setup(x => x.UpdateTwitterUserAsync(It.Is(y => y.Id == userId2 && y.FetchingErrorCount == 1))) + .Returns(Task.CompletedTask); + + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); + #endregion + + var processor = new RefreshTwitterUserStatusProcessor(twitterUserServiceMock.Object, twitterUserDalMock.Object, removeTwitterAccountActionMock.Object, settings); + var result = await processor.ProcessAsync(users.ToArray(), CancellationToken.None); + + #region Validations + Assert.AreEqual(1, result.Length); + Assert.IsTrue(result.Any(x => x.User.Id == userId1)); + + twitterUserServiceMock.VerifyAll(); + twitterUserDalMock.VerifyAll(); + removeTwitterAccountActionMock.VerifyAll(); + #endregion + } + [TestMethod] public async Task ProcessAsync_Error_Test() { @@ -671,5 +747,80 @@ namespace BirdsiteLive.Pipeline.Tests.Processors removeTwitterAccountActionMock.VerifyAll(); #endregion } + + [TestMethod] + public async Task ProcessAsync_RateLimited_Test() + { + #region Stubs + var userId1 = 1; + var acct1 = "user1"; + + var userId2 = 2; + var acct2 = "user2"; + + var users = new List + { + new SyncTwitterUser + { + Id = userId1, + Acct = acct1 + }, + new SyncTwitterUser + { + Id = userId2, + Acct = acct2 + } + }; + + var settings = new InstanceSettings + { + FailingTwitterUserCleanUpThreshold = 300 + }; + #endregion + + #region Mocks + var twitterUserServiceMock = new Mock(MockBehavior.Strict); + twitterUserServiceMock + .Setup(x => x.GetUser(It.Is(y => y == acct1))) + .Returns(new TwitterUser + { + Protected = false, + }); + + twitterUserServiceMock + .Setup(x => x.GetUser(It.Is(y => y == acct2))) + .Throws(new RateLimitExceededException()); + + //twitterUserServiceMock + // .Setup(x => x.PurgeUser(It.Is(y => y == acct2))); + + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) + .ReturnsAsync(new SyncTwitterUser + { + Id = userId2, + FetchingErrorCount = 20 + }); + + twitterUserDalMock + .Setup(x => x.UpdateTwitterUserAsync(It.Is(y => y.Id == userId2 && y.FetchingErrorCount == 20))) + .Returns(Task.CompletedTask); + + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); + #endregion + + var processor = new RefreshTwitterUserStatusProcessor(twitterUserServiceMock.Object, twitterUserDalMock.Object, removeTwitterAccountActionMock.Object, settings); + var result = await processor.ProcessAsync(users.ToArray(), CancellationToken.None); + + #region Validations + Assert.AreEqual(1, result.Length); + Assert.IsTrue(result.Any(x => x.User.Id == userId1)); + + twitterUserServiceMock.VerifyAll(); + twitterUserDalMock.VerifyAll(); + removeTwitterAccountActionMock.VerifyAll(); + #endregion + } } } \ No newline at end of file From 9f9f88aab78410bc672cfcd04b8814855a7f1cd3 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Mon, 7 Feb 2022 19:37:28 -0500 Subject: [PATCH 15/19] clean up --- .../RefreshTwitterUserStatusProcessorTests.cs | 98 ++----------------- 1 file changed, 7 insertions(+), 91 deletions(-) diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs index be342c0..cd2d116 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs @@ -289,10 +289,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors twitterUserServiceMock .Setup(x => x.GetUser(It.Is(y => y == acct2))) .Throws(new Exception()); - - //twitterUserServiceMock - // .Setup(x => x.PurgeUser(It.Is(y => y == acct2))); - + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) @@ -364,10 +361,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors twitterUserServiceMock .Setup(x => x.GetUser(It.Is(y => y == acct2))) .Returns((TwitterUser)null); - - //twitterUserServiceMock - // .Setup(x => x.PurgeUser(It.Is(y => y == acct2))); - + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) @@ -439,10 +433,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors twitterUserServiceMock .Setup(x => x.GetUser(It.Is(y => y == acct2))) .Returns((TwitterUser)null); - - //twitterUserServiceMock - // .Setup(x => x.PurgeUser(It.Is(y => y == acct2))); - + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) @@ -516,10 +507,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors { Protected = true }); - - //twitterUserServiceMock - // .Setup(x => x.PurgeUser(It.Is(y => y == acct2))); - + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) @@ -594,10 +582,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors { Protected = true }); - - //twitterUserServiceMock - // .Setup(x => x.PurgeUser(It.Is(y => y == acct2))); - + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) @@ -625,70 +610,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors removeTwitterAccountActionMock.VerifyAll(); #endregion } - - //[TestMethod] - //public async Task ProcessAsync_Protected_Test() - //{ - // #region Stubs - // var userId1 = 1; - // var acct1 = "user1"; - - // var userId2 = 2; - // var acct2 = "user2"; - - // var users = new List - // { - // new SyncTwitterUser - // { - // Id = userId1, - // Acct = acct1 - // }, - // new SyncTwitterUser - // { - // Id = userId2, - // Acct = acct2 - // } - // }; - - // var settings = new InstanceSettings - // { - // FailingTwitterUserCleanUpThreshold = 300 - // }; - // #endregion - - // #region Mocks - // var twitterUserServiceMock = new Mock(MockBehavior.Strict); - // twitterUserServiceMock - // .Setup(x => x.GetUser(It.Is(y => y == acct1))) - // .Returns(new TwitterUser - // { - // Protected = false - // }); - - // twitterUserServiceMock - // .Setup(x => x.GetUser(It.Is(y => y == acct2))) - // .Returns(new TwitterUser - // { - // Protected = true - // }); - - // var twitterUserDalMock = new Mock(MockBehavior.Strict); - // var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); - // #endregion - - // var processor = new RefreshTwitterUserStatusProcessor(twitterUserServiceMock.Object, twitterUserDalMock.Object, removeTwitterAccountActionMock.Object, settings); - // var result = await processor.ProcessAsync(users.ToArray(), CancellationToken.None); - - // #region Validations - // Assert.AreEqual(1, result.Length); - // Assert.IsTrue(result.Any(x => x.User.Id == userId1)); - - // twitterUserServiceMock.VerifyAll(); - // twitterUserDalMock.VerifyAll(); - // removeTwitterAccountActionMock.VerifyAll(); - // #endregion - //} - + [TestMethod] public async Task ProcessAsync_Error_NotInit_Test() { @@ -720,9 +642,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Setup(x => x.GetUser(It.Is(y => y == acct1))) .Returns((TwitterUser)null); - //twitterUserServiceMock - // .Setup(x => x.PurgeUser(It.Is(y => y == acct1))); - var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct1))) @@ -790,10 +709,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors twitterUserServiceMock .Setup(x => x.GetUser(It.Is(y => y == acct2))) .Throws(new RateLimitExceededException()); - - //twitterUserServiceMock - // .Setup(x => x.PurgeUser(It.Is(y => y == acct2))); - + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) From a7b4a4978ad6f6a57d61e8ad07227a4edd4aeddd Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Tue, 8 Feb 2022 20:32:48 -0500 Subject: [PATCH 16/19] throw exception instead of returning null --- src/BirdsiteLive.Twitter/TwitterUserService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive.Twitter/TwitterUserService.cs b/src/BirdsiteLive.Twitter/TwitterUserService.cs index 7fb6439..8505ed3 100644 --- a/src/BirdsiteLive.Twitter/TwitterUserService.cs +++ b/src/BirdsiteLive.Twitter/TwitterUserService.cs @@ -35,7 +35,7 @@ namespace BirdsiteLive.Twitter public TwitterUser GetUser(string username) { //Check if API is saturated - if (IsUserApiRateLimited()) return null; + if (IsUserApiRateLimited()) throw new RateLimitExceededException(); //Proceed to account retrieval _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized(); From e78bc262ed965f0d5a284b776996a122c5c07a24 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Tue, 8 Feb 2022 23:40:02 -0500 Subject: [PATCH 17/19] added url support in webfinger --- .../Controllers/WellKnownController.cs | 44 ++++++++++++++----- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/src/BirdsiteLive/Controllers/WellKnownController.cs b/src/BirdsiteLive/Controllers/WellKnownController.cs index 272d789..3e14d7d 100644 --- a/src/BirdsiteLive/Controllers/WellKnownController.cs +++ b/src/BirdsiteLive/Controllers/WellKnownController.cs @@ -144,30 +144,54 @@ namespace BirdsiteLive.Controllers [Route("/.well-known/webfinger")] public IActionResult Webfinger(string resource = null) { - var acct = resource.Split("acct:")[1].Trim(); + if (string.IsNullOrWhiteSpace(resource)) + return BadRequest(); string name = null; string domain = null; - var splitAcct = acct.Split('@', StringSplitOptions.RemoveEmptyEntries); + if (resource.StartsWith("acct:")) + { + var acct = resource.Split("acct:")[1].Trim(); + var splitAcct = acct.Split('@', StringSplitOptions.RemoveEmptyEntries); - var atCount = acct.Count(x => x == '@'); - if (atCount == 1 && acct.StartsWith('@')) - { - name = splitAcct[1]; + var atCount = acct.Count(x => x == '@'); + if (atCount == 1 && acct.StartsWith('@')) + { + name = splitAcct[1]; + } + else if (atCount == 1 || atCount == 2) + { + name = splitAcct[0]; + domain = splitAcct[1]; + } + else + { + return BadRequest(); + } } - else if (atCount == 1 || atCount == 2) + else if (resource.StartsWith("https://")) { - name = splitAcct[0]; - domain = splitAcct[1]; + try + { + name = resource.Split('/').Last().Trim(); + domain = resource.Split("https://", StringSplitOptions.RemoveEmptyEntries)[0].Split('/')[0].Trim(); + } + catch (Exception e) + { + _logger.LogError(e, "Error parsing {Resource}", resource); + throw new NotImplementedException(); + } } else { - return BadRequest(); + _logger.LogError("Error parsing {Resource}", resource); + throw new NotImplementedException(); } // Ensure lowercase name = name.ToLowerInvariant(); + domain = domain?.ToLowerInvariant(); // Ensure valid username // https://help.twitter.com/en/managing-your-account/twitter-username-rules From 0e9938b712e3eb6cb4358933baf7099e53c6c2b7 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 9 Feb 2022 01:15:48 -0500 Subject: [PATCH 18/19] added user is gone exception --- src/BirdsiteLive.Domain/ActivityPubService.cs | 7 +++- .../Exceptions/UserIsGoneException.cs | 8 +++++ .../Controllers/UsersController.cs | 34 ++++++++++++------- 3 files changed, 35 insertions(+), 14 deletions(-) create mode 100644 src/BirdsiteLive.Domain/Exceptions/UserIsGoneException.cs diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs index 37fe0b5..c242a99 100644 --- a/src/BirdsiteLive.Domain/ActivityPubService.cs +++ b/src/BirdsiteLive.Domain/ActivityPubService.cs @@ -11,7 +11,6 @@ using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.Common.Settings; using Microsoft.Extensions.Logging; using Newtonsoft.Json; -using Org.BouncyCastle.Bcpg; namespace BirdsiteLive.Domain { @@ -45,6 +44,12 @@ namespace BirdsiteLive.Domain var httpClient = _httpClientFactory.CreateClient(); httpClient.DefaultRequestHeaders.Add("Accept", "application/activity+json"); var result = await httpClient.GetAsync(objectId); + + if (result.StatusCode == HttpStatusCode.Gone) + throw new UserIsGoneException(); + + result.EnsureSuccessStatusCode(); + var content = await result.Content.ReadAsStringAsync(); var actor = JsonConvert.DeserializeObject(content); diff --git a/src/BirdsiteLive.Domain/Exceptions/UserIsGoneException.cs b/src/BirdsiteLive.Domain/Exceptions/UserIsGoneException.cs new file mode 100644 index 0000000..96476fe --- /dev/null +++ b/src/BirdsiteLive.Domain/Exceptions/UserIsGoneException.cs @@ -0,0 +1,8 @@ +using System; + +namespace BirdsiteLive.Domain +{ + public class UserIsGoneException : Exception + { + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index f59ae21..de63af1 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -183,33 +183,41 @@ namespace BirdsiteLive.Controllers switch (activity?.type) { case "Follow": - { - var succeeded = await _userService.FollowRequestedAsync(signature, r.Method, r.Path, - r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), activity as ActivityFollow, body); - if (succeeded) return Accepted(); - else return Unauthorized(); - } + { + var succeeded = await _userService.FollowRequestedAsync(signature, r.Method, r.Path, + r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), + activity as ActivityFollow, body); + if (succeeded) return Accepted(); + else return Unauthorized(); + } case "Undo": if (activity is ActivityUndoFollow) { var succeeded = await _userService.UndoFollowRequestedAsync(signature, r.Method, r.Path, - r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), activity as ActivityUndoFollow, body); + r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), + activity as ActivityUndoFollow, body); if (succeeded) return Accepted(); else return Unauthorized(); } + return Accepted(); case "Delete": - { - var succeeded = await _userService.DeleteRequestedAsync(signature, r.Method, r.Path, - r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), activity as ActivityDelete, body); - if (succeeded) return Accepted(); - else return Unauthorized(); - } + { + var succeeded = await _userService.DeleteRequestedAsync(signature, r.Method, r.Path, + r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), + activity as ActivityDelete, body); + if (succeeded) return Accepted(); + else return Unauthorized(); + } default: return Accepted(); } } } + catch (UserIsGoneException) + { + return Accepted(); + } catch (UserNotFoundException) { return NotFound(); From 7007b6309adbae2eb4fd00b36308994dbaa0e30e Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 9 Feb 2022 01:54:35 -0500 Subject: [PATCH 19/19] handle exception in sharedInbox --- src/BirdsiteLive.Domain/ActivityPubService.cs | 2 +- ...xception.cs => FollowerIsGoneException.cs} | 2 +- .../Controllers/InboxController.cs | 36 ++++++++++--------- .../Controllers/UsersController.cs | 2 +- 4 files changed, 23 insertions(+), 19 deletions(-) rename src/BirdsiteLive.Domain/Exceptions/{UserIsGoneException.cs => FollowerIsGoneException.cs} (54%) diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs index c242a99..c460a2d 100644 --- a/src/BirdsiteLive.Domain/ActivityPubService.cs +++ b/src/BirdsiteLive.Domain/ActivityPubService.cs @@ -46,7 +46,7 @@ namespace BirdsiteLive.Domain var result = await httpClient.GetAsync(objectId); if (result.StatusCode == HttpStatusCode.Gone) - throw new UserIsGoneException(); + throw new FollowerIsGoneException(); result.EnsureSuccessStatusCode(); diff --git a/src/BirdsiteLive.Domain/Exceptions/UserIsGoneException.cs b/src/BirdsiteLive.Domain/Exceptions/FollowerIsGoneException.cs similarity index 54% rename from src/BirdsiteLive.Domain/Exceptions/UserIsGoneException.cs rename to src/BirdsiteLive.Domain/Exceptions/FollowerIsGoneException.cs index 96476fe..3063cab 100644 --- a/src/BirdsiteLive.Domain/Exceptions/UserIsGoneException.cs +++ b/src/BirdsiteLive.Domain/Exceptions/FollowerIsGoneException.cs @@ -2,7 +2,7 @@ namespace BirdsiteLive.Domain { - public class UserIsGoneException : Exception + public class FollowerIsGoneException : Exception { } } \ No newline at end of file diff --git a/src/BirdsiteLive/Controllers/InboxController.cs b/src/BirdsiteLive/Controllers/InboxController.cs index f55e22b..f92a0a6 100644 --- a/src/BirdsiteLive/Controllers/InboxController.cs +++ b/src/BirdsiteLive/Controllers/InboxController.cs @@ -31,28 +31,32 @@ namespace BirdsiteLive.Controllers [HttpPost] public async Task Inbox() { - var r = Request; - using (var reader = new StreamReader(Request.Body)) + try { - var body = await reader.ReadToEndAsync(); - - _logger.LogTrace("Inbox: {Body}", body); - //System.IO.File.WriteAllText($@"C:\apdebug\inbox\{Guid.NewGuid()}.json", body); - - var activity = ApDeserializer.ProcessActivity(body); - var signature = r.Headers["Signature"].First(); - - switch (activity?.type) + var r = Request; + using (var reader = new StreamReader(Request.Body)) { - case "Delete": + var body = await reader.ReadToEndAsync(); + + _logger.LogTrace("Inbox: {Body}", body); + //System.IO.File.WriteAllText($@"C:\apdebug\inbox\{Guid.NewGuid()}.json", body); + + var activity = ApDeserializer.ProcessActivity(body); + var signature = r.Headers["Signature"].First(); + + switch (activity?.type) { - var succeeded = await _userService.DeleteRequestedAsync(signature, r.Method, r.Path, - r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), activity as ActivityDelete, body); - if (succeeded) return Accepted(); - else return Unauthorized(); + case "Delete": + { + var succeeded = await _userService.DeleteRequestedAsync(signature, r.Method, r.Path, + r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), activity as ActivityDelete, body); + if (succeeded) return Accepted(); + else return Unauthorized(); + } } } } + catch (FollowerIsGoneException) { } //TODO: check if user in DB return Accepted(); } diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index de63af1..24a9eb5 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -214,7 +214,7 @@ namespace BirdsiteLive.Controllers } } } - catch (UserIsGoneException) + catch (FollowerIsGoneException) //TODO: check if user in DB { return Accepted(); }