diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs index 847e23c..eba1868 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs @@ -19,7 +19,6 @@ namespace BirdsiteLive.Pipeline.Processors private readonly InstanceSettings _instanceSettings; public int WaitFactor = 1000 * 60; //1 min - private int StartUpWarming = 4; #region Ctor public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, InstanceSettings instanceSettings, ILogger logger) @@ -32,16 +31,20 @@ namespace BirdsiteLive.Pipeline.Processors public async Task GetTwitterUsersAsync(BufferBlock twitterUsersBufferBlock, CancellationToken ct) { + var totalUsers = await _twitterUserDal.GetTwitterUsersCountAsync(); + var warmUpMaxCapacity = _instanceSettings.MaxUsersCapacity / 4; + var warmUpIterations = warmUpMaxCapacity == 0 ? 0 : (int) (totalUsers / (float) warmUpMaxCapacity); + for (; ; ) { ct.ThrowIfCancellationRequested(); try { - var maxUsers = StartUpWarming > 0 + var maxUsers = warmUpIterations > 0 ? _instanceSettings.MaxUsersCapacity / 4 : _instanceSettings.MaxUsersCapacity; - StartUpWarming--; + warmUpIterations--; var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsers); var userCount = users.Any() ? users.Length : 1; diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs index 6a1e96b..12e6f7f 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs @@ -119,6 +119,41 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default); } + [TestMethod] + public async Task GetAllTwitterUsers_Limited() + { + var now = DateTime.Now; + var oldest = now.AddDays(-3); + var newest = now.AddDays(-2); + + var dal = new TwitterUserPostgresDal(_settings); + for (var i = 0; i < 20; i++) + { + var acct = $"myid{i}"; + var lastTweetId = 1548L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + } + + var allUsers = await dal.GetAllTwitterUsersAsync(100); + for (var i = 0; i < 20; i++) + { + var user = allUsers[i]; + var date = i % 2 == 0 ? oldest : newest; + await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, date); + } + + var result = await dal.GetAllTwitterUsersAsync(10); + Assert.AreEqual(10, result.Length); + Assert.IsFalse(result[0].Id == default); + Assert.IsFalse(result[0].Acct == default); + Assert.IsFalse(result[0].LastTweetPostedId == default); + Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default); + + foreach (var acc in result) + Assert.IsTrue(Math.Abs((acc.LastSync - oldest.ToUniversalTime()).TotalMilliseconds) < 1000); + } + [TestMethod] public async Task CountTwitterUsers() { diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs index 6323889..ef33e78 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs @@ -29,7 +29,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors }; var settings = new InstanceSettings { - MaxUsersCapacity = 10 + MaxUsersCapacity = 40 }; #endregion @@ -37,9 +37,13 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == settings.MaxUsersCapacity))) + It.Is(y => y == settings.MaxUsersCapacity/4))) .ReturnsAsync(users); + twitterUserDalMock + .Setup(x => x.GetTwitterUsersCountAsync()) + .ReturnsAsync(10); + var loggerMock = new Mock>(); #endregion @@ -69,7 +73,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var settings = new InstanceSettings { - MaxUsersCapacity = 100 + MaxUsersCapacity = 400 }; #endregion @@ -81,6 +85,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .ReturnsAsync(users.ToArray()) .ReturnsAsync(new SyncTwitterUser[0]); + twitterUserDalMock + .Setup(x => x.GetTwitterUsersCountAsync()) + .ReturnsAsync(30); + var loggerMock = new Mock>(); #endregion @@ -110,7 +118,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var settings = new InstanceSettings { - MaxUsersCapacity = 10 + MaxUsersCapacity = 400 }; #endregion @@ -118,10 +126,14 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .SetupSequence(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == settings.MaxUsersCapacity))) + It.Is(y => y == settings.MaxUsersCapacity/4))) .ReturnsAsync(users.ToArray()) .ReturnsAsync(new SyncTwitterUser[0]); + twitterUserDalMock + .Setup(x => x.GetTwitterUsersCountAsync()) + .ReturnsAsync(31); + var loggerMock = new Mock>(); #endregion @@ -158,6 +170,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == settings.MaxUsersCapacity))) .ReturnsAsync(new SyncTwitterUser[0]); + twitterUserDalMock + .Setup(x => x.GetTwitterUsersCountAsync()) + .ReturnsAsync(1000); + var loggerMock = new Mock>(); #endregion @@ -192,6 +208,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == settings.MaxUsersCapacity))) .Returns(async () => await DelayFaultedTask(new Exception())); + twitterUserDalMock + .Setup(x => x.GetTwitterUsersCountAsync()) + .ReturnsAsync(1000); + var loggerMock = new Mock>(); #endregion @@ -224,6 +244,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); + + twitterUserDalMock + .Setup(x => x.GetTwitterUsersCountAsync()) + .ReturnsAsync(1000); + var loggerMock = new Mock>(); #endregion