From 3a998b60acb5fdb576e2c2df09d386e7cfa437d3 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 2 Feb 2022 21:33:45 -0500 Subject: [PATCH] added auto clean-up on failing follower --- .../Settings/InstanceSettings.cs | 1 + .../SendTweetsToFollowersProcessor.cs | 20 +- src/BirdsiteLive/appsettings.json | 3 +- .../SendTweetsToFollowersProcessorTests.cs | 250 +++++++++++++++++- 4 files changed, 260 insertions(+), 14 deletions(-) diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs index a85d9e6..eb77d1f 100644 --- a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs +++ b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs @@ -13,5 +13,6 @@ public string SensitiveTwitterAccounts { get; set; } public int FailingTwitterUserCleanUpThreshold { get; set; } + public int FailingFollowerCleanUpThreshold { get; set; } = -1; } } diff --git a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs index 65f9610..e210f39 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs @@ -5,9 +5,11 @@ using System.Net; using System.Threading; using System.Threading.Tasks; using System.Xml; +using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Domain; +using BirdsiteLive.Moderation.Actions; using BirdsiteLive.Pipeline.Contracts; using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Pipeline.Processors.SubTasks; @@ -23,14 +25,18 @@ namespace BirdsiteLive.Pipeline.Processors private readonly ISendTweetsToInboxTask _sendTweetsToInboxTask; private readonly ISendTweetsToSharedInboxTask _sendTweetsToSharedInbox; private readonly IFollowersDal _followersDal; + private readonly InstanceSettings _instanceSettings; private readonly ILogger _logger; + private readonly IRemoveFollowerAction _removeFollowerAction; #region Ctor - public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox, IFollowersDal followersDal, ILogger logger) + public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox, IFollowersDal followersDal, ILogger logger, InstanceSettings instanceSettings, IRemoveFollowerAction removeFollowerAction) { _sendTweetsToInboxTask = sendTweetsToInboxTask; _sendTweetsToSharedInbox = sendTweetsToSharedInbox; _logger = logger; + _instanceSettings = instanceSettings; + _removeFollowerAction = removeFollowerAction; _followersDal = followersDal; } #endregion @@ -107,7 +113,17 @@ namespace BirdsiteLive.Pipeline.Processors private async Task ProcessFailingUserAsync(Follower follower) { follower.PostingErrorCount++; - await _followersDal.UpdateFollowerAsync(follower); + + if (follower.PostingErrorCount > _instanceSettings.FailingFollowerCleanUpThreshold + && _instanceSettings.FailingFollowerCleanUpThreshold > 0 + || follower.PostingErrorCount > 2147483600) + { + await _removeFollowerAction.ProcessAsync(follower); + } + else + { + await _followersDal.UpdateFollowerAsync(follower); + } } } } \ No newline at end of file diff --git a/src/BirdsiteLive/appsettings.json b/src/BirdsiteLive/appsettings.json index d510809..b49d25c 100644 --- a/src/BirdsiteLive/appsettings.json +++ b/src/BirdsiteLive/appsettings.json @@ -23,7 +23,8 @@ "MaxUsersCapacity": 1000, "UnlistedTwitterAccounts": null, "SensitiveTwitterAccounts": null, - "FailingTwitterUserCleanUpThreshold": 700 + "FailingTwitterUserCleanUpThreshold": 700, + "FailingFollowerCleanUpThreshold": 30000 }, "Db": { "Type": "postgres", diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs index 53aa12a..8a78038 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs @@ -2,8 +2,10 @@ using System.Threading; using System.Threading.Tasks; using System.Xml; +using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; +using BirdsiteLive.Moderation.Actions; using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Pipeline.Processors; using BirdsiteLive.Pipeline.Processors.SubTasks; @@ -72,17 +74,22 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); var followersDalMock = new Mock(MockBehavior.Strict); - + var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -147,15 +154,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var followersDalMock = new Mock(MockBehavior.Strict); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -229,15 +241,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -312,15 +329,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -400,15 +422,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -471,15 +498,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var followersDalMock = new Mock(MockBehavior.Strict); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -543,15 +575,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var followersDalMock = new Mock(MockBehavior.Strict); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -623,15 +660,196 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_MultiInstances_Inbox_OneTweet_Error_SettingsThreshold_Test() + { + #region Stubs + var tweetId = 1; + var host1 = "domain1.ext"; + var host2 = "domain2.ext"; + var inbox = "/user/inbox"; + var userId1 = 2; + var userId2 = 3; + var userAcct = "user"; + + var userWithTweets = new UserWithDataToSync() + { + Tweets = new[] + { + new ExtractedTweet + { + Id = tweetId + } + }, + User = new SyncTwitterUser + { + Acct = userAcct + }, + Followers = new[] + { + new Follower + { + Id = userId1, + Host = host1, + InboxRoute = inbox + }, + new Follower + { + Id = userId2, + Host = host2, + InboxRoute = inbox, + PostingErrorCount = 42 + }, + } + }; + #endregion + + #region Mocks + var sendTweetsToInboxTaskMock = new Mock(MockBehavior.Strict); + sendTweetsToInboxTaskMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Length == 1), + It.Is(y => y.Id == userId1), + It.Is(y => y.Acct == userAcct))) + .Returns(Task.CompletedTask); + + sendTweetsToInboxTaskMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Length == 1), + It.Is(y => y.Id == userId2), + It.Is(y => y.Acct == userAcct))) + .Throws(new Exception()); + + var sendTweetsToSharedInboxTaskMock = new Mock(MockBehavior.Strict); + + var followersDalMock = new Mock(MockBehavior.Strict); + + var loggerMock = new Mock>(); + + var settings = new InstanceSettings + { + FailingFollowerCleanUpThreshold = 10 + }; + + var removeFollowerMock = new Mock(MockBehavior.Strict); + removeFollowerMock + .Setup(x => x.ProcessAsync(It.Is(y => y.Id == userId2))) + .Returns(Task.CompletedTask); + #endregion + + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); + var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); + + #region Validations + sendTweetsToInboxTaskMock.VerifyAll(); + sendTweetsToSharedInboxTaskMock.VerifyAll(); + followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_MultiInstances_Inbox_OneTweet_Error_MaxThreshold_Test() + { + #region Stubs + var tweetId = 1; + var host1 = "domain1.ext"; + var host2 = "domain2.ext"; + var inbox = "/user/inbox"; + var userId1 = 2; + var userId2 = 3; + var userAcct = "user"; + + var userWithTweets = new UserWithDataToSync() + { + Tweets = new[] + { + new ExtractedTweet + { + Id = tweetId + } + }, + User = new SyncTwitterUser + { + Acct = userAcct + }, + Followers = new[] + { + new Follower + { + Id = userId1, + Host = host1, + InboxRoute = inbox + }, + new Follower + { + Id = userId2, + Host = host2, + InboxRoute = inbox, + PostingErrorCount = 2147483600 + }, + } + }; + #endregion + + #region Mocks + var sendTweetsToInboxTaskMock = new Mock(MockBehavior.Strict); + sendTweetsToInboxTaskMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Length == 1), + It.Is(y => y.Id == userId1), + It.Is(y => y.Acct == userAcct))) + .Returns(Task.CompletedTask); + + sendTweetsToInboxTaskMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Length == 1), + It.Is(y => y.Id == userId2), + It.Is(y => y.Acct == userAcct))) + .Throws(new Exception()); + + var sendTweetsToSharedInboxTaskMock = new Mock(MockBehavior.Strict); + + var followersDalMock = new Mock(MockBehavior.Strict); + + var loggerMock = new Mock>(); + + var settings = new InstanceSettings + { + FailingFollowerCleanUpThreshold = 0 + }; + + var removeFollowerMock = new Mock(MockBehavior.Strict); + removeFollowerMock + .Setup(x => x.ProcessAsync(It.Is(y => y.Id == userId2))) + .Returns(Task.CompletedTask); + #endregion + + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); + var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); + + #region Validations + sendTweetsToInboxTaskMock.VerifyAll(); + sendTweetsToSharedInboxTaskMock.VerifyAll(); + followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -704,15 +922,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -790,15 +1013,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } }