sam/BirdsiteLIVE
sam
/
BirdsiteLIVE
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

added auto clean-up on failing follower

This commit is contained in:
Nicolas Constant 2022-02-02 21:33:45 -05:00
parent 26cca6a306
commit 3a998b60ac
No known key found for this signature in database
GPG Key ID: 1E9F677FB01A5688
4 changed files with 260 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/BirdsiteLive.Common/Settings/InstanceSettings.cs
View File

@ -13,5 +13,6 @@
public string SensitiveTwitterAccounts { get; set; }
public int FailingTwitterUserCleanUpThreshold { get; set; }
public int FailingFollowerCleanUpThreshold { get; set; } = -1;
}
}

20
src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs
View File

@ -5,9 +5,11 @@ using System.Net;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Domain;
using BirdsiteLive.Moderation.Actions;
using BirdsiteLive.Pipeline.Contracts;
using BirdsiteLive.Pipeline.Models;
using BirdsiteLive.Pipeline.Processors.SubTasks;
@ -23,14 +25,18 @@ namespace BirdsiteLive.Pipeline.Processors
private readonly ISendTweetsToInboxTask _sendTweetsToInboxTask;
private readonly ISendTweetsToSharedInboxTask _sendTweetsToSharedInbox;
private readonly IFollowersDal _followersDal;
private readonly InstanceSettings _instanceSettings;
private readonly ILogger<SendTweetsToFollowersProcessor> _logger;
private readonly IRemoveFollowerAction _removeFollowerAction;
#region Ctor
public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox, IFollowersDal followersDal, ILogger<SendTweetsToFollowersProcessor> logger)
public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox, IFollowersDal followersDal, ILogger<SendTweetsToFollowersProcessor> logger, InstanceSettings instanceSettings, IRemoveFollowerAction removeFollowerAction)
{
_sendTweetsToInboxTask = sendTweetsToInboxTask;
_sendTweetsToSharedInbox = sendTweetsToSharedInbox;
_logger = logger;
_instanceSettings = instanceSettings;
_removeFollowerAction = removeFollowerAction;
_followersDal = followersDal;
}
#endregion
@ -107,7 +113,17 @@ namespace BirdsiteLive.Pipeline.Processors
private async Task ProcessFailingUserAsync(Follower follower)
{
follower.PostingErrorCount++;
await _followersDal.UpdateFollowerAsync(follower);
if (follower.PostingErrorCount > _instanceSettings.FailingFollowerCleanUpThreshold
&& _instanceSettings.FailingFollowerCleanUpThreshold > 0
|| follower.PostingErrorCount > 2147483600)
{
await _removeFollowerAction.ProcessAsync(follower);
}
else
{
await _followersDal.UpdateFollowerAsync(follower);
}
}
}
}

3
src/BirdsiteLive/appsettings.json
View File

@ -23,7 +23,8 @@
"MaxUsersCapacity": 1000,
"UnlistedTwitterAccounts": null,
"SensitiveTwitterAccounts": null,
"FailingTwitterUserCleanUpThreshold": 700
"FailingTwitterUserCleanUpThreshold": 700,
"FailingFollowerCleanUpThreshold": 30000
},
"Db": {
"Type": "postgres",

250
src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs
View File

@ -2,8 +2,10 @@
using System.Threading;
using System.Threading.Tasks;
using System.Xml;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Moderation.Actions;
using BirdsiteLive.Pipeline.Models;
using BirdsiteLive.Pipeline.Processors;
using BirdsiteLive.Pipeline.Processors.SubTasks;
@ -72,17 +74,22 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
.Returns(Task.CompletedTask);
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
var settings = new InstanceSettings();
var removeFollowerMock = new Mock<IRemoveFollowerAction>(MockBehavior.Strict);
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object);
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
sendTweetsToSharedInboxTaskMock.VerifyAll();
followersDalMock.VerifyAll();
removeFollowerMock.VerifyAll();
#endregion
}
@ -147,15 +154,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
var settings = new InstanceSettings();
var removeFollowerMock = new Mock<IRemoveFollowerAction>(MockBehavior.Strict);
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object);
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
sendTweetsToSharedInboxTaskMock.VerifyAll();
followersDalMock.VerifyAll();
removeFollowerMock.VerifyAll();
#endregion
}
@ -229,15 +241,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
.Returns(Task.CompletedTask);
var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
var settings = new InstanceSettings();
var removeFollowerMock = new Mock<IRemoveFollowerAction>(MockBehavior.Strict);
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object);
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
sendTweetsToSharedInboxTaskMock.VerifyAll();
followersDalMock.VerifyAll();
removeFollowerMock.VerifyAll();
#endregion
}
@ -312,15 +329,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
.Returns(Task.CompletedTask);
var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
var settings = new InstanceSettings();
var removeFollowerMock = new Mock<IRemoveFollowerAction>(MockBehavior.Strict);
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object);
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
sendTweetsToSharedInboxTaskMock.VerifyAll();
followersDalMock.VerifyAll();
removeFollowerMock.VerifyAll();
#endregion
}
@ -400,15 +422,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
.Returns(Task.CompletedTask);
var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
var settings = new InstanceSettings();
var removeFollowerMock = new Mock<IRemoveFollowerAction>(MockBehavior.Strict);
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object);
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
sendTweetsToSharedInboxTaskMock.VerifyAll();
followersDalMock.VerifyAll();
removeFollowerMock.VerifyAll();
#endregion
}
@ -471,15 +498,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
var settings = new InstanceSettings();
var removeFollowerMock = new Mock<IRemoveFollowerAction>(MockBehavior.Strict);
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object);
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
sendTweetsToSharedInboxTaskMock.VerifyAll();
followersDalMock.VerifyAll();
removeFollowerMock.VerifyAll();
#endregion
}
@ -543,15 +575,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
var settings = new InstanceSettings();
var removeFollowerMock = new Mock<IRemoveFollowerAction>(MockBehavior.Strict);
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object);
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
sendTweetsToSharedInboxTaskMock.VerifyAll();
followersDalMock.VerifyAll();
removeFollowerMock.VerifyAll();
#endregion
}
@ -623,15 +660,196 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
.Returns(Task.CompletedTask);
var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
var settings = new InstanceSettings();
var removeFollowerMock = new Mock<IRemoveFollowerAction>(MockBehavior.Strict);
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object);
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
sendTweetsToSharedInboxTaskMock.VerifyAll();
followersDalMock.VerifyAll();
removeFollowerMock.VerifyAll();
#endregion
}
[TestMethod]
public async Task ProcessAsync_MultiInstances_Inbox_OneTweet_Error_SettingsThreshold_Test()
{
#region Stubs
var tweetId = 1;
var host1 = "domain1.ext";
var host2 = "domain2.ext";
var inbox = "/user/inbox";
var userId1 = 2;
var userId2 = 3;
var userAcct = "user";
var userWithTweets = new UserWithDataToSync()
{
Tweets = new[]
{
new ExtractedTweet
{
Id = tweetId
}
},
User = new SyncTwitterUser
{
Acct = userAcct
},
Followers = new[]
{
new Follower
{
Id = userId1,
Host = host1,
InboxRoute = inbox
},
new Follower
{
Id = userId2,
Host = host2,
InboxRoute = inbox,
PostingErrorCount = 42
},
}
};
#endregion
#region Mocks
var sendTweetsToInboxTaskMock = new Mock<ISendTweetsToInboxTask>(MockBehavior.Strict);
sendTweetsToInboxTaskMock
.Setup(x => x.ExecuteAsync(
It.Is<ExtractedTweet[]>(y => y.Length == 1),
It.Is<Follower>(y => y.Id == userId1),
It.Is<SyncTwitterUser>(y => y.Acct == userAcct)))
.Returns(Task.CompletedTask);
sendTweetsToInboxTaskMock
.Setup(x => x.ExecuteAsync(
It.Is<ExtractedTweet[]>(y => y.Length == 1),
It.Is<Follower>(y => y.Id == userId2),
It.Is<SyncTwitterUser>(y => y.Acct == userAcct)))
.Throws(new Exception());
var sendTweetsToSharedInboxTaskMock = new Mock<ISendTweetsToSharedInboxTask>(MockBehavior.Strict);
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
var settings = new InstanceSettings
{
FailingFollowerCleanUpThreshold = 10
};
var removeFollowerMock = new Mock<IRemoveFollowerAction>(MockBehavior.Strict);
removeFollowerMock
.Setup(x => x.ProcessAsync(It.Is<Follower>(y => y.Id == userId2)))
.Returns(Task.CompletedTask);
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
sendTweetsToSharedInboxTaskMock.VerifyAll();
followersDalMock.VerifyAll();
removeFollowerMock.VerifyAll();
#endregion
}
[TestMethod]
public async Task ProcessAsync_MultiInstances_Inbox_OneTweet_Error_MaxThreshold_Test()
{
#region Stubs
var tweetId = 1;
var host1 = "domain1.ext";
var host2 = "domain2.ext";
var inbox = "/user/inbox";
var userId1 = 2;
var userId2 = 3;
var userAcct = "user";
var userWithTweets = new UserWithDataToSync()
{
Tweets = new[]
{
new ExtractedTweet
{
Id = tweetId
}
},
User = new SyncTwitterUser
{
Acct = userAcct
},
Followers = new[]
{
new Follower
{
Id = userId1,
Host = host1,
InboxRoute = inbox
},
new Follower
{
Id = userId2,
Host = host2,
InboxRoute = inbox,
PostingErrorCount = 2147483600
},
}
};
#endregion
#region Mocks
var sendTweetsToInboxTaskMock = new Mock<ISendTweetsToInboxTask>(MockBehavior.Strict);
sendTweetsToInboxTaskMock
.Setup(x => x.ExecuteAsync(
It.Is<ExtractedTweet[]>(y => y.Length == 1),
It.Is<Follower>(y => y.Id == userId1),
It.Is<SyncTwitterUser>(y => y.Acct == userAcct)))
.Returns(Task.CompletedTask);
sendTweetsToInboxTaskMock
.Setup(x => x.ExecuteAsync(
It.Is<ExtractedTweet[]>(y => y.Length == 1),
It.Is<Follower>(y => y.Id == userId2),
It.Is<SyncTwitterUser>(y => y.Acct == userAcct)))
.Throws(new Exception());
var sendTweetsToSharedInboxTaskMock = new Mock<ISendTweetsToSharedInboxTask>(MockBehavior.Strict);
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
var settings = new InstanceSettings
{
FailingFollowerCleanUpThreshold = 0
};
var removeFollowerMock = new Mock<IRemoveFollowerAction>(MockBehavior.Strict);
removeFollowerMock
.Setup(x => x.ProcessAsync(It.Is<Follower>(y => y.Id == userId2)))
.Returns(Task.CompletedTask);
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
sendTweetsToSharedInboxTaskMock.VerifyAll();
followersDalMock.VerifyAll();
removeFollowerMock.VerifyAll();
#endregion
}
@ -704,15 +922,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
.Returns(Task.CompletedTask);
var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
var settings = new InstanceSettings();
var removeFollowerMock = new Mock<IRemoveFollowerAction>(MockBehavior.Strict);
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object);
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
sendTweetsToSharedInboxTaskMock.VerifyAll();
followersDalMock.VerifyAll();
removeFollowerMock.VerifyAll();
#endregion
}
@ -790,15 +1013,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
.Returns(Task.CompletedTask);
var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
var settings = new InstanceSettings();
var removeFollowerMock = new Mock<IRemoveFollowerAction>(MockBehavior.Strict);
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object);
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
sendTweetsToSharedInboxTaskMock.VerifyAll();
followersDalMock.VerifyAll();
removeFollowerMock.VerifyAll();
#endregion
}
}