added test coverage and warmup strategy

src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs

@@ -19,7 +19,6 @@ namespace BirdsiteLive.Pipeline.Processors
         private readonly InstanceSettings _instanceSettings;
         
         public int WaitFactor = 1000 * 60; //1 min
-        private int StartUpWarming = 4;
 
         #region Ctor
         public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, InstanceSettings instanceSettings, ILogger<RetrieveTwitterUsersProcessor> logger)
@@ -32,16 +31,20 @@ namespace BirdsiteLive.Pipeline.Processors
 
         public async Task GetTwitterUsersAsync(BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock, CancellationToken ct)
         {
+            var totalUsers = await _twitterUserDal.GetTwitterUsersCountAsync();
+            var warmUpMaxCapacity = _instanceSettings.MaxUsersCapacity / 4;
+            var warmUpIterations = warmUpMaxCapacity == 0 ? 0 : (int) (totalUsers / (float) warmUpMaxCapacity);
+
             for (; ; )
             {
                 ct.ThrowIfCancellationRequested();
 
                 try
                 {
-                    var maxUsers = StartUpWarming > 0
+                    var maxUsers = warmUpIterations > 0
                         ? _instanceSettings.MaxUsersCapacity / 4
                         : _instanceSettings.MaxUsersCapacity;
-                    StartUpWarming--;
+                    warmUpIterations--;
                     var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsers);
 
                     var userCount = users.Any() ? users.Length : 1;

src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs

@@ -119,6 +119,41 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
         }
 
+        [TestMethod]
+        public async Task GetAllTwitterUsers_Limited()
+        {
+            var now = DateTime.Now;
+            var oldest = now.AddDays(-3);
+            var newest = now.AddDays(-2);
+
+            var dal = new TwitterUserPostgresDal(_settings);
+            for (var i = 0; i < 20; i++)
+            {
+                var acct = $"myid{i}";
+                var lastTweetId = 1548L;
+
+                await dal.CreateTwitterUserAsync(acct, lastTweetId);
+            }
+
+            var allUsers = await dal.GetAllTwitterUsersAsync(100);
+            for (var i = 0; i < 20; i++)
+            {
+                var user = allUsers[i];
+                var date = i % 2 == 0 ? oldest : newest;
+                await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, date);
+            }
+
+            var result = await dal.GetAllTwitterUsersAsync(10);
+            Assert.AreEqual(10, result.Length);
+            Assert.IsFalse(result[0].Id == default);
+            Assert.IsFalse(result[0].Acct == default);
+            Assert.IsFalse(result[0].LastTweetPostedId == default);
+            Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
+
+            foreach (var acc in result)
+                Assert.IsTrue(Math.Abs((acc.LastSync - oldest.ToUniversalTime()).TotalMilliseconds) < 1000);
+        }
+
         [TestMethod]
         public async Task CountTwitterUsers()
         {

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs

@@ -29,7 +29,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             };
             var settings = new InstanceSettings
             {
-                MaxUsersCapacity = 10
+                MaxUsersCapacity = 40
             };
             #endregion
 
@@ -37,9 +37,13 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .Setup(x => x.GetAllTwitterUsersAsync(
-                    It.Is<int>(y => y == settings.MaxUsersCapacity)))
+                    It.Is<int>(y => y == settings.MaxUsersCapacity/4)))
                 .ReturnsAsync(users);
 
+            twitterUserDalMock
+                .Setup(x => x.GetTwitterUsersCountAsync())
+                .ReturnsAsync(10);
+
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
@@ -69,7 +73,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
 
             var settings = new InstanceSettings
             {
-                MaxUsersCapacity = 100
+                MaxUsersCapacity = 400
             };
             #endregion
 
@@ -81,6 +85,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 .ReturnsAsync(users.ToArray())
                 .ReturnsAsync(new SyncTwitterUser[0]);
 
+            twitterUserDalMock
+                .Setup(x => x.GetTwitterUsersCountAsync())
+                .ReturnsAsync(30);
+
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
@@ -110,7 +118,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
 
             var settings = new InstanceSettings
             {
-                MaxUsersCapacity = 10
+                MaxUsersCapacity = 400
             };
             #endregion
 
@@ -118,10 +126,14 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .SetupSequence(x => x.GetAllTwitterUsersAsync(
-                    It.Is<int>(y => y == settings.MaxUsersCapacity)))
+                    It.Is<int>(y => y == settings.MaxUsersCapacity/4)))
                 .ReturnsAsync(users.ToArray())
                 .ReturnsAsync(new SyncTwitterUser[0]);
 
+            twitterUserDalMock
+                .Setup(x => x.GetTwitterUsersCountAsync())
+                .ReturnsAsync(31);
+
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
@@ -158,6 +170,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                     It.Is<int>(y => y == settings.MaxUsersCapacity)))
                 .ReturnsAsync(new SyncTwitterUser[0]);
 
+            twitterUserDalMock
+                .Setup(x => x.GetTwitterUsersCountAsync())
+                .ReturnsAsync(1000);
+
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
@@ -192,6 +208,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                     It.Is<int>(y => y == settings.MaxUsersCapacity)))
                 .Returns(async () => await DelayFaultedTask<SyncTwitterUser[]>(new Exception()));
 
+            twitterUserDalMock
+                .Setup(x => x.GetTwitterUsersCountAsync())
+                .ReturnsAsync(1000);
+
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
@@ -224,6 +244,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
 
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
+
+            twitterUserDalMock
+                .Setup(x => x.GetTwitterUsersCountAsync())
+                .ReturnsAsync(1000);
+
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion