sam/BirdsiteLIVE
sam/BirdsiteLIVE
Archived
1
0
Fork 0
Code Issues 1 Pull requests 1 Projects Releases Packages 1 Wiki Activity

refactoring warm up logic and tests

Browse source
This commit is contained in:
Nicolas Constant 2021-01-22 23:38:35 -05:00
parent 3e7a136902
commit 34cb33160a
No known key found for this signature in database
GPG key ID: 1E9F677FB01A5688
6 changed files with 204 additions and 74 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj
View file

@ -17,4 +17,8 @@
<ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
</ItemGroup>
<ItemGroup>
<Folder Include="Tools\" />
</ItemGroup>
</Project>

18
src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
View file

@ -8,6 +8,7 @@ using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Pipeline.Contracts;
using BirdsiteLive.Pipeline.Tools;
using Microsoft.Extensions.Logging;
namespace BirdsiteLive.Pipeline.Processors
@ -15,37 +16,30 @@ namespace BirdsiteLive.Pipeline.Processors
public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor
{
private readonly ITwitterUserDal _twitterUserDal;
private readonly IMaxUsersNumberProvider _maxUsersNumberProvider;
private readonly ILogger<RetrieveTwitterUsersProcessor> _logger;
private readonly InstanceSettings _instanceSettings;
public int WaitFactor = 1000 * 60; //1 min
#region Ctor
public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, InstanceSettings instanceSettings, ILogger<RetrieveTwitterUsersProcessor> logger)
public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IMaxUsersNumberProvider maxUsersNumberProvider, ILogger<RetrieveTwitterUsersProcessor> logger)
{
_twitterUserDal = twitterUserDal;
_instanceSettings = instanceSettings;
_maxUsersNumberProvider = maxUsersNumberProvider;
_logger = logger;
}
#endregion
public async Task GetTwitterUsersAsync(BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock, CancellationToken ct)
{
var totalUsers = await _twitterUserDal.GetTwitterUsersCountAsync();
var warmUpMaxCapacity = _instanceSettings.MaxUsersCapacity / 4;
var warmUpIterations = warmUpMaxCapacity == 0 ? 0 : (int) (totalUsers / (float) warmUpMaxCapacity);
for (; ; )
{
ct.ThrowIfCancellationRequested();
try
{
var maxUsers = warmUpIterations > 0
? _instanceSettings.MaxUsersCapacity / 4
: _instanceSettings.MaxUsersCapacity;
warmUpIterations--;
var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsers);
var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync();
var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber);
var userCount = users.Any() ? users.Length : 1;
var splitNumber = (int) Math.Ceiling(userCount / 15d);

49
src/BirdsiteLive.Pipeline/Tools/MaxUsersNumberProvider.cs Normal file
View file

@ -0,0 +1,49 @@
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
namespace BirdsiteLive.Pipeline.Tools
{
public interface IMaxUsersNumberProvider
{
Task<int> GetMaxUsersNumberAsync();
}
public class MaxUsersNumberProvider : IMaxUsersNumberProvider
{
private readonly InstanceSettings _instanceSettings;
private readonly ITwitterUserDal _twitterUserDal;
private int _totalUsersCount = -1;
private int _warmUpIterations;
#region Ctor
public MaxUsersNumberProvider(InstanceSettings instanceSettings, ITwitterUserDal twitterUserDal)
{
_instanceSettings = instanceSettings;
_twitterUserDal = twitterUserDal;
}
#endregion
public async Task<int> GetMaxUsersNumberAsync()
{
// Init data
if (_totalUsersCount == -1)
{
_totalUsersCount = await _twitterUserDal.GetTwitterUsersCountAsync();
var warmUpMaxCapacity = _instanceSettings.MaxUsersCapacity / 4;
_warmUpIterations = warmUpMaxCapacity == 0 ? 0 : (int)(_totalUsersCount / (float)warmUpMaxCapacity);
}
// Return if warm up ended
if (_warmUpIterations <= 0) return _instanceSettings.MaxUsersCapacity;
// Calculate warm up value
var maxUsers = _warmUpIterations > 0
? _instanceSettings.MaxUsersCapacity / 4
: _instanceSettings.MaxUsersCapacity;
_warmUpIterations--;
return maxUsers;
}
}
}

4
src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj
View file

@ -18,4 +18,8 @@
<ProjectReference Include="..\..\BirdsiteLive.Pipeline\BirdsiteLive.Pipeline.csproj" />
</ItemGroup>
<ItemGroup>
<Folder Include="Tools\" />
</ItemGroup>
</Project>

124
src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs
View file

@ -7,6 +7,7 @@ using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Pipeline.Processors;
using BirdsiteLive.Pipeline.Tools;
using Microsoft.Extensions.Logging;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Moq;
@ -27,33 +28,32 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
new SyncTwitterUser(),
new SyncTwitterUser(),
};
var settings = new InstanceSettings
{
MaxUsersCapacity = 40
};
var maxUsers = 1000;
#endregion
#region Mocks
var maxUsersNumberProviderMock = new Mock<IMaxUsersNumberProvider>(MockBehavior.Strict);
maxUsersNumberProviderMock
.Setup(x => x.GetMaxUsersNumberAsync())
.ReturnsAsync(maxUsers);
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == settings.MaxUsersCapacity/4)))
It.Is<int>(y => y == maxUsers)))
.ReturnsAsync(users);
twitterUserDalMock
.Setup(x => x.GetTwitterUsersCountAsync())
.ReturnsAsync(10);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object);
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object);
processor.WaitFactor = 10;
processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
await Task.Delay(50);
#region Validations
maxUsersNumberProviderMock.VerifyAll();
twitterUserDalMock.VerifyAll();
Assert.AreEqual(3, buffer.Count);
buffer.TryReceive(out var result);
@ -71,34 +71,36 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
for (var i = 0; i < 30; i++)
users.Add(new SyncTwitterUser());
var settings = new InstanceSettings
{
MaxUsersCapacity = 400
};
var maxUsers = 1000;
#endregion
#region Mocks
var maxUsersNumberProviderMock = new Mock<IMaxUsersNumberProvider>(MockBehavior.Strict);
maxUsersNumberProviderMock
.Setup(x => x.GetMaxUsersNumberAsync())
.ReturnsAsync(maxUsers);
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.SetupSequence(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == settings.MaxUsersCapacity)))
It.Is<int>(y => y == maxUsers)))
.ReturnsAsync(users.ToArray())
.ReturnsAsync(new SyncTwitterUser[0])
.ReturnsAsync(new SyncTwitterUser[0])
.ReturnsAsync(new SyncTwitterUser[0])
.ReturnsAsync(new SyncTwitterUser[0]);
twitterUserDalMock
.Setup(x => x.GetTwitterUsersCountAsync())
.ReturnsAsync(30);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object);
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object);
processor.WaitFactor = 2;
processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
await Task.Delay(300);
#region Validations
maxUsersNumberProviderMock.VerifyAll();
twitterUserDalMock.VerifyAll();
Assert.AreEqual(15, buffer.Count);
buffer.TryReceive(out var result);
@ -116,34 +118,36 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
for (var i = 0; i < 31; i++)
users.Add(new SyncTwitterUser());
var settings = new InstanceSettings
{
MaxUsersCapacity = 400
};
var maxUsers = 1000;
#endregion
#region Mocks
var maxUsersNumberProviderMock = new Mock<IMaxUsersNumberProvider>(MockBehavior.Strict);
maxUsersNumberProviderMock
.Setup(x => x.GetMaxUsersNumberAsync())
.ReturnsAsync(maxUsers);
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.SetupSequence(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == settings.MaxUsersCapacity/4)))
It.Is<int>(y => y == maxUsers)))
.ReturnsAsync(users.ToArray())
.ReturnsAsync(new SyncTwitterUser[0])
.ReturnsAsync(new SyncTwitterUser[0])
.ReturnsAsync(new SyncTwitterUser[0])
.ReturnsAsync(new SyncTwitterUser[0]);
twitterUserDalMock
.Setup(x => x.GetTwitterUsersCountAsync())
.ReturnsAsync(31);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object);
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object);
processor.WaitFactor = 2;
processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
await Task.Delay(200);
#region Validations
maxUsersNumberProviderMock.VerifyAll();
twitterUserDalMock.VerifyAll();
Assert.AreEqual(11, buffer.Count);
buffer.TryReceive(out var result);
@ -157,33 +161,32 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
#region Stubs
var buffer = new BufferBlock<SyncTwitterUser[]>();
var settings = new InstanceSettings
{
MaxUsersCapacity = 10
};
var maxUsers = 1000;
#endregion
#region Mocks
var maxUsersNumberProviderMock = new Mock<IMaxUsersNumberProvider>(MockBehavior.Strict);
maxUsersNumberProviderMock
.Setup(x => x.GetMaxUsersNumberAsync())
.ReturnsAsync(maxUsers);
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == settings.MaxUsersCapacity)))
It.Is<int>(y => y == maxUsers)))
.ReturnsAsync(new SyncTwitterUser[0]);
twitterUserDalMock
.Setup(x => x.GetTwitterUsersCountAsync())
.ReturnsAsync(1000);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object);
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object);
processor.WaitFactor = 1;
processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
await Task.Delay(50);
#region Validations
maxUsersNumberProviderMock.VerifyAll();
twitterUserDalMock.VerifyAll();
Assert.AreEqual(0, buffer.Count);
#endregion
@ -195,33 +198,32 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
#region Stubs
var buffer = new BufferBlock<SyncTwitterUser[]>();
var settings = new InstanceSettings
{
MaxUsersCapacity = 10
};
var maxUsers = 1000;
#endregion
#region Mocks
var maxUsersNumberProviderMock = new Mock<IMaxUsersNumberProvider>(MockBehavior.Strict);
maxUsersNumberProviderMock
.Setup(x => x.GetMaxUsersNumberAsync())
.ReturnsAsync(maxUsers);
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == settings.MaxUsersCapacity)))
It.Is<int>(y => y == maxUsers)))
.Returns(async () => await DelayFaultedTask<SyncTwitterUser[]>(new Exception()));
twitterUserDalMock
.Setup(x => x.GetTwitterUsersCountAsync())
.ReturnsAsync(1000);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object);
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object);
processor.WaitFactor = 10;
var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
await Task.WhenAny(t, Task.Delay(50));
#region Validations
maxUsersNumberProviderMock.VerifyAll();
twitterUserDalMock.VerifyAll();
Assert.AreEqual(0, buffer.Count);
#endregion
@ -236,23 +238,21 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var canTokenS = new CancellationTokenSource();
canTokenS.Cancel();
var settings = new InstanceSettings
{
MaxUsersCapacity = 10
};
var maxUsers = 1000;
#endregion
#region Mocks
var maxUsersNumberProviderMock = new Mock<IMaxUsersNumberProvider>(MockBehavior.Strict);
maxUsersNumberProviderMock
.Setup(x => x.GetMaxUsersNumberAsync())
.ReturnsAsync(maxUsers);
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetTwitterUsersCountAsync())
.ReturnsAsync(1000);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object);
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object);
processor.WaitFactor = 1;
await processor.GetTwitterUsersAsync(buffer, canTokenS.Token);
}

79
src/Tests/BirdsiteLive.Pipeline.Tests/Tools/MaxUsersNumberProviderTests.cs Normal file
View file

@ -0,0 +1,79 @@
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.Pipeline.Tools;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Moq;
namespace BirdsiteLive.Pipeline.Tests.Tools
{
[TestClass]
public class MaxUsersNumberProviderTests
{
[TestMethod]
public async Task GetMaxUsersNumberAsync_WarmUp_Test()
{
#region Stubs
var settings = new InstanceSettings
{
MaxUsersCapacity = 1000
};
#endregion
#region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetTwitterUsersCountAsync())
.ReturnsAsync(1000);
#endregion
var provider = new MaxUsersNumberProvider(settings, twitterUserDalMock.Object);
var result = await provider.GetMaxUsersNumberAsync();
Assert.AreEqual(250, result);
result = await provider.GetMaxUsersNumberAsync();
Assert.AreEqual(250, result);
result = await provider.GetMaxUsersNumberAsync();
Assert.AreEqual(250, result);
result = await provider.GetMaxUsersNumberAsync();
Assert.AreEqual(250, result);
result = await provider.GetMaxUsersNumberAsync();
Assert.AreEqual(1000, result);
#region Validations
twitterUserDalMock.VerifyAll();
#endregion
}
[TestMethod]
public async Task GetMaxUsersNumberAsync_NoWarmUp_Test()
{
#region Stubs
var settings = new InstanceSettings
{
MaxUsersCapacity = 1000
};
#endregion
#region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetTwitterUsersCountAsync())
.ReturnsAsync(249);
#endregion
var provider = new MaxUsersNumberProvider(settings, twitterUserDalMock.Object);
var result = await provider.GetMaxUsersNumberAsync();
Assert.AreEqual(1000, result);
#region Validations
twitterUserDalMock.VerifyAll();
#endregion
}
}
}