sam/BirdsiteLIVE
sam/BirdsiteLIVE
Archived
1
0
Fork 0
Code Issues 1 Pull requests 1 Projects Releases Packages 1 Wiki Activity

added test coverage and warmup strategy

Browse source
This commit is contained in:
Nicolas Constant 2021-01-22 23:07:25 -05:00
parent 89c041f332
commit 3e7a136902
No known key found for this signature in database
GPG key ID: 1E9F677FB01A5688
3 changed files with 71 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
View file

@ -19,7 +19,6 @@ namespace BirdsiteLive.Pipeline.Processors
private readonly InstanceSettings _instanceSettings;
public int WaitFactor = 1000 * 60; //1 min
private int StartUpWarming = 4;
#region Ctor
public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, InstanceSettings instanceSettings, ILogger<RetrieveTwitterUsersProcessor> logger)
@ -32,16 +31,20 @@ namespace BirdsiteLive.Pipeline.Processors
public async Task GetTwitterUsersAsync(BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock, CancellationToken ct)
{
var totalUsers = await _twitterUserDal.GetTwitterUsersCountAsync();
var warmUpMaxCapacity = _instanceSettings.MaxUsersCapacity / 4;
var warmUpIterations = warmUpMaxCapacity == 0 ? 0 : (int) (totalUsers / (float) warmUpMaxCapacity);
for (; ; )
{
ct.ThrowIfCancellationRequested();
try
{
var maxUsers = StartUpWarming > 0
var maxUsers = warmUpIterations > 0
? _instanceSettings.MaxUsersCapacity / 4
: _instanceSettings.MaxUsersCapacity;
StartUpWarming--;
warmUpIterations--;
var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsers);
var userCount = users.Any() ? users.Length : 1;

35
src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs
View file

@ -119,6 +119,41 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
}
[TestMethod]
public async Task GetAllTwitterUsers_Limited()
{
var now = DateTime.Now;
var oldest = now.AddDays(-3);
var newest = now.AddDays(-2);
var dal = new TwitterUserPostgresDal(_settings);
for (var i = 0; i < 20; i++)
{
var acct = $"myid{i}";
var lastTweetId = 1548L;
await dal.CreateTwitterUserAsync(acct, lastTweetId);
}
var allUsers = await dal.GetAllTwitterUsersAsync(100);
for (var i = 0; i < 20; i++)
{
var user = allUsers[i];
var date = i % 2 == 0 ? oldest : newest;
await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, date);
}
var result = await dal.GetAllTwitterUsersAsync(10);
Assert.AreEqual(10, result.Length);
Assert.IsFalse(result[0].Id == default);
Assert.IsFalse(result[0].Acct == default);
Assert.IsFalse(result[0].LastTweetPostedId == default);
Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
foreach (var acc in result)
Assert.IsTrue(Math.Abs((acc.LastSync - oldest.ToUniversalTime()).TotalMilliseconds) < 1000);
}
[TestMethod]
public async Task CountTwitterUsers()
{

35
src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs
View file

@ -29,7 +29,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
};
var settings = new InstanceSettings
{
MaxUsersCapacity = 10
MaxUsersCapacity = 40
};
#endregion
@ -37,9 +37,13 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == settings.MaxUsersCapacity)))
It.Is<int>(y => y == settings.MaxUsersCapacity/4)))
.ReturnsAsync(users);
twitterUserDalMock
.Setup(x => x.GetTwitterUsersCountAsync())
.ReturnsAsync(10);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
@ -69,7 +73,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var settings = new InstanceSettings
{
MaxUsersCapacity = 100
MaxUsersCapacity = 400
};
#endregion
@ -81,6 +85,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
.ReturnsAsync(users.ToArray())
.ReturnsAsync(new SyncTwitterUser[0]);
twitterUserDalMock
.Setup(x => x.GetTwitterUsersCountAsync())
.ReturnsAsync(30);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
@ -110,7 +118,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var settings = new InstanceSettings
{
MaxUsersCapacity = 10
MaxUsersCapacity = 400
};
#endregion
@ -118,10 +126,14 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.SetupSequence(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == settings.MaxUsersCapacity)))
It.Is<int>(y => y == settings.MaxUsersCapacity/4)))
.ReturnsAsync(users.ToArray())
.ReturnsAsync(new SyncTwitterUser[0]);
twitterUserDalMock
.Setup(x => x.GetTwitterUsersCountAsync())
.ReturnsAsync(31);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
@ -158,6 +170,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
It.Is<int>(y => y == settings.MaxUsersCapacity)))
.ReturnsAsync(new SyncTwitterUser[0]);
twitterUserDalMock
.Setup(x => x.GetTwitterUsersCountAsync())
.ReturnsAsync(1000);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
@ -192,6 +208,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
It.Is<int>(y => y == settings.MaxUsersCapacity)))
.Returns(async () => await DelayFaultedTask<SyncTwitterUser[]>(new Exception()));
twitterUserDalMock
.Setup(x => x.GetTwitterUsersCountAsync())
.ReturnsAsync(1000);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
@ -224,6 +244,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
#region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetTwitterUsersCountAsync())
.ReturnsAsync(1000);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion