added pipeline processor to analyse user state

src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj

@@ -13,6 +13,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\BirdsiteLive.Domain\BirdsiteLive.Domain.csproj" />
+    <ProjectReference Include="..\BirdsiteLive.Moderation\BirdsiteLive.Moderation.csproj" />
     <ProjectReference Include="..\BirdsiteLive.Twitter\BirdsiteLive.Twitter.csproj" />
     <ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
   </ItemGroup>

src/BirdsiteLive.Pipeline/Contracts/IRefreshTwitterUserStatusProcessor.cs

@@ -0,0 +1,12 @@
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Pipeline.Models;
+
+namespace BirdsiteLive.Pipeline.Contracts
+{
+    public interface IRefreshTwitterUserStatusProcessor
+    {
+        Task<UserWithDataToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct);
+    }
+}

src/BirdsiteLive.Pipeline/Contracts/IRetrieveFollowersProcessor.cs

@@ -7,7 +7,7 @@ namespace BirdsiteLive.Pipeline.Contracts
 {
     public interface IRetrieveFollowersProcessor
     {
-        Task<IEnumerable<UserWithTweetsToSync>> ProcessAsync(UserWithTweetsToSync[] userWithTweetsToSyncs, CancellationToken ct);
+        Task<IEnumerable<UserWithDataToSync>> ProcessAsync(UserWithDataToSync[] userWithTweetsToSyncs, CancellationToken ct);
         //IAsyncEnumerable<UserWithTweetsToSync> ProcessAsync(UserWithTweetsToSync[] userWithTweetsToSyncs, CancellationToken ct);
     }
 }

src/BirdsiteLive.Pipeline/Contracts/IRetrieveTweetsProcessor.cs

@@ -7,6 +7,6 @@ namespace BirdsiteLive.Pipeline.Contracts
 {
     public interface IRetrieveTweetsProcessor
     {
-        Task<UserWithTweetsToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct);
+        Task<UserWithDataToSync[]> ProcessAsync(UserWithDataToSync[] syncTwitterUsers, CancellationToken ct);
     }
 }

src/BirdsiteLive.Pipeline/Contracts/ISaveProgressionProcessor.cs

@@ -6,6 +6,6 @@ namespace BirdsiteLive.Pipeline.Contracts
 {
     public interface ISaveProgressionProcessor
     {
-        Task ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct);
+        Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct);
     }
 }

src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs

@@ -6,6 +6,6 @@ namespace BirdsiteLive.Pipeline.Contracts
 {
     public interface ISendTweetsToFollowersProcessor
     {
-        Task<UserWithTweetsToSync> ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct);
+        Task<UserWithDataToSync> ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct);
     }
 }

src/BirdsiteLive.Pipeline/Models/UserWithTweetsToSync.cs

@@ -4,10 +4,13 @@ using Tweetinvi.Models;
 
 namespace BirdsiteLive.Pipeline.Models
 {
-    public class UserWithTweetsToSync
+    public class UserWithDataToSync
     {
         public SyncTwitterUser User { get; set; }
         public ExtractedTweet[] Tweets { get; set; }
         public Follower[] Followers { get; set; }
+
+        public bool IsUserProtected { get; set; }
+        public bool IsUserNotRetrieved { get; set; }
     }
 }

src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs

@@ -0,0 +1,69 @@
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Moderation.Actions;
+using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Models;
+using BirdsiteLive.Twitter;
+
+namespace BirdsiteLive.Pipeline.Processors
+{
+    public class RefreshTwitterUserStatusProcessor : IRefreshTwitterUserStatusProcessor
+    {
+        private const int FetchingErrorCountThreshold = 10;
+        private readonly ICachedTwitterUserService _twitterUserService;
+        private readonly ITwitterUserDal _twitterUserDal;
+        private readonly IRemoveTwitterAccountAction _removeTwitterAccountAction;
+
+        #region Ctor
+        public RefreshTwitterUserStatusProcessor(ICachedTwitterUserService twitterUserService)
+        {
+            _twitterUserService = twitterUserService;
+        }
+        #endregion
+
+        public async Task<UserWithDataToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct)
+        {
+            var usersWtData = new List<UserWithDataToSync>();
+
+            foreach (var user in syncTwitterUsers)
+            {
+                var userView = _twitterUserService.GetUser(user.Acct);
+                if (userView == null)
+                {
+                    await AnalyseFailingUserAsync(user);
+                }
+                else if (!userView.Protected)
+                {
+                    var userWtData = new UserWithDataToSync
+                    {
+                        User = user
+                    };
+                    usersWtData.Add(userWtData);
+                }
+            }
+
+            return usersWtData.ToArray();
+        }
+
+        private async Task AnalyseFailingUserAsync(SyncTwitterUser user)
+        {
+            var dbUser = await _twitterUserDal.GetTwitterUserAsync(user.Acct);
+            dbUser.FetchingErrorCount++;
+
+            if (dbUser.FetchingErrorCount > FetchingErrorCountThreshold)
+            {
+                await _removeTwitterAccountAction.ProcessAsync(user);
+            }
+            else
+            {
+                await _twitterUserDal.UpdateTwitterUserAsync(dbUser);
+            }
+
+            // Purge
+            _twitterUserService.PurgeUser(user.Acct);
+        }
+    }
+}

src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs

@@ -18,7 +18,7 @@ namespace BirdsiteLive.Pipeline.Processors
         }
         #endregion
 
-        public async Task<IEnumerable<UserWithTweetsToSync>> ProcessAsync(UserWithTweetsToSync[] userWithTweetsToSyncs, CancellationToken ct)
+        public async Task<IEnumerable<UserWithDataToSync>> ProcessAsync(UserWithDataToSync[] userWithTweetsToSyncs, CancellationToken ct)
         {
             //TODO multithread this
             foreach (var user in userWithTweetsToSyncs)

src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs

@@ -31,33 +31,30 @@ namespace BirdsiteLive.Pipeline.Processors
         }
         #endregion
 
-        public async Task<UserWithTweetsToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct)
+        public async Task<UserWithDataToSync[]> ProcessAsync(UserWithDataToSync[] syncTwitterUsers, CancellationToken ct)
         {
-            var usersWtTweets = new List<UserWithTweetsToSync>();
+            var usersWtTweets = new List<UserWithDataToSync>();
 
             //TODO multithread this
-            foreach (var user in syncTwitterUsers)
+            foreach (var userWtData in syncTwitterUsers)
             {
+                var user = userWtData.User;
                 var tweets = RetrieveNewTweets(user);
                 if (tweets.Length > 0 && user.LastTweetPostedId != -1)
                 {
-                    var userWtTweets = new UserWithTweetsToSync
-                    {
-                        User = user,
-                        Tweets = tweets
-                    };
-                    usersWtTweets.Add(userWtTweets);
+                    userWtData.Tweets = tweets;
+                    usersWtTweets.Add(userWtData);
                 }
                 else if (tweets.Length > 0 && user.LastTweetPostedId == -1)
                 {
                     var tweetId = tweets.Last().Id;
                     var now = DateTime.UtcNow;
-                    await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, now);
+                    await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now);
                 }
                 else
                 {
                     var now = DateTime.UtcNow;
-                    await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, now);
+                    await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now);
                 }
             }
 

src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs

@@ -22,7 +22,7 @@ namespace BirdsiteLive.Pipeline.Processors
         }
         #endregion
 
-        public async Task ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct)
+        public async Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
         {
             try
             {
@@ -49,7 +49,7 @@ namespace BirdsiteLive.Pipeline.Processors
                 var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();
                 var minimumSync = followingSyncStatuses.Min();
                 var now = DateTime.UtcNow;
-                await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, now);
+                await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now);
             }
             catch (Exception e)
             {

src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs

@@ -33,7 +33,7 @@ namespace BirdsiteLive.Pipeline.Processors
         }
         #endregion
 
-        public async Task<UserWithTweetsToSync> ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct)
+        public async Task<UserWithDataToSync> ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
         {
             var user = userWithTweetsToSync.User;
 

src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
 using System.Threading.Tasks.Dataflow;
@@ -17,6 +18,7 @@ namespace BirdsiteLive.Pipeline
     public class StatusPublicationPipeline : IStatusPublicationPipeline
     {
         private readonly IRetrieveTwitterUsersProcessor _retrieveTwitterAccountsProcessor;
+        private readonly IRefreshTwitterUserStatusProcessor _refreshTwitterUserStatusProcessor;
         private readonly IRetrieveTweetsProcessor _retrieveTweetsProcessor;
         private readonly IRetrieveFollowersProcessor _retrieveFollowersProcessor;
         private readonly ISendTweetsToFollowersProcessor _sendTweetsToFollowersProcessor;
@@ -24,13 +26,14 @@ namespace BirdsiteLive.Pipeline
         private readonly ILogger<StatusPublicationPipeline> _logger;
 
         #region Ctor
-        public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ISaveProgressionProcessor saveProgressionProcessor, ILogger<StatusPublicationPipeline> logger)
+        public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ISaveProgressionProcessor saveProgressionProcessor, IRefreshTwitterUserStatusProcessor refreshTwitterUserStatusProcessor, ILogger<StatusPublicationPipeline> logger)
         {
             _retrieveTweetsProcessor = retrieveTweetsProcessor;
             _retrieveTwitterAccountsProcessor = retrieveTwitterAccountsProcessor;
             _retrieveFollowersProcessor = retrieveFollowersProcessor;
             _sendTweetsToFollowersProcessor = sendTweetsToFollowersProcessor;
             _saveProgressionProcessor = saveProgressionProcessor;
+            _refreshTwitterUserStatusProcessor = refreshTwitterUserStatusProcessor;
 
             _logger = logger;
         }
@@ -39,16 +42,21 @@ namespace BirdsiteLive.Pipeline
         public async Task ExecuteAsync(CancellationToken ct)
         {
             // Create blocks 
-            var twitterUsersBufferBlock = new BufferBlock<SyncTwitterUser[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
-            var retrieveTweetsBlock = new TransformBlock<SyncTwitterUser[], UserWithTweetsToSync[]>(async x => await _retrieveTweetsProcessor.ProcessAsync(x, ct));
-            var retrieveTweetsBufferBlock = new BufferBlock<UserWithTweetsToSync[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
-            var retrieveFollowersBlock = new TransformManyBlock<UserWithTweetsToSync[], UserWithTweetsToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct));
-            var retrieveFollowersBufferBlock = new BufferBlock<UserWithTweetsToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
-            var sendTweetsToFollowersBlock = new TransformBlock<UserWithTweetsToSync, UserWithTweetsToSync>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
-            var sendTweetsToFollowersBufferBlock = new BufferBlock<UserWithTweetsToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
-            var saveProgressionBlock = new ActionBlock<UserWithTweetsToSync>(async x => await _saveProgressionProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
+            var twitterUserToRefreshBufferBlock = new BufferBlock<SyncTwitterUser[]>(new DataflowBlockOptions
+                { BoundedCapacity = 1, CancellationToken = ct });
+            var twitterUserToRefreshBlock = new TransformBlock<SyncTwitterUser[], UserWithDataToSync[]>(async x => await _refreshTwitterUserStatusProcessor.ProcessAsync(x, ct));
+            var twitterUsersBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
+            var retrieveTweetsBlock = new TransformBlock<UserWithDataToSync[], UserWithDataToSync[]>(async x => await _retrieveTweetsProcessor.ProcessAsync(x, ct));
+            var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
+            var retrieveFollowersBlock = new TransformManyBlock<UserWithDataToSync[], UserWithDataToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct));
+            var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
+            var sendTweetsToFollowersBlock = new TransformBlock<UserWithDataToSync, UserWithDataToSync>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
+            var sendTweetsToFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
+            var saveProgressionBlock = new ActionBlock<UserWithDataToSync>(async x => await _saveProgressionProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
 
             // Link pipeline
+            twitterUserToRefreshBufferBlock.LinkTo(twitterUserToRefreshBlock, new DataflowLinkOptions { PropagateCompletion = true });
+            twitterUserToRefreshBlock.LinkTo(twitterUsersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
             twitterUsersBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions { PropagateCompletion = true });
             retrieveTweetsBlock.LinkTo(retrieveTweetsBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
             retrieveTweetsBufferBlock.LinkTo(retrieveFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
@@ -58,7 +66,7 @@ namespace BirdsiteLive.Pipeline
             sendTweetsToFollowersBufferBlock.LinkTo(saveProgressionBlock, new DataflowLinkOptions { PropagateCompletion = true });
 
             // Launch twitter user retriever
-            var retrieveTwitterAccountsTask = _retrieveTwitterAccountsProcessor.GetTwitterUsersAsync(twitterUsersBufferBlock, ct);
+            var retrieveTwitterAccountsTask = _retrieveTwitterAccountsProcessor.GetTwitterUsersAsync(twitterUserToRefreshBufferBlock, ct);
 
             // Wait
             await Task.WhenAny(new[] { retrieveTwitterAccountsTask, saveProgressionBlock.Completion });

src/BirdsiteLive.Twitter/TwitterUserService.cs

@@ -49,6 +49,9 @@ namespace BirdsiteLive.Twitter
             catch (Exception e)
             {
                 _logger.LogError(e, "Error retrieving user {Username}", username);
+
+                // TODO keep track of error, see where to remove user if too much errors
+
                 return null;
             }
 

src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs

@@ -23,7 +23,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
     public class DbInitializerPostgresDal : PostgresBase, IDbInitializerDal
     {
         private readonly PostgresTools _tools;
-        private readonly Version _currentVersion = new Version(2, 1);
+        private readonly Version _currentVersion = new Version(2, 2);
         private const string DbVersionType = "db-version";
 
         #region Ctor
@@ -132,7 +132,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
             return new[]
             {
                 new Tuple<Version, Version>(new Version(1,0), new Version(2,0)),
-                new Tuple<Version, Version>(new Version(2,0), new Version(2,1))
+                new Tuple<Version, Version>(new Version(2,0), new Version(2,1)),
+                new Tuple<Version, Version>(new Version(2,1), new Version(2,2))
             };
         }
 
@@ -151,6 +152,11 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
                 var addActorId = $@"ALTER TABLE {_settings.FollowersTableName} ADD actorId VARCHAR(2048)";
                 await _tools.ExecuteRequestAsync(addActorId);
             }
+            else if (from == new Version(2, 1) && to == new Version(2, 2))
+            {
+                var addLastSync = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD fetchingErrorCount SMALLINT";
+                await _tools.ExecuteRequestAsync(addLastSync);
+            }
             else
             {
                 throw new NotImplementedException();

src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs

@@ -99,23 +99,28 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
             }
         }
 
-        public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, DateTime lastSync)
+        public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync)
         {
             if(id == default) throw new ArgumentException("id");
             if(lastTweetPostedId == default) throw new ArgumentException("lastTweetPostedId");
             if(lastTweetSynchronizedForAllFollowersId == default) throw new ArgumentException("lastTweetSynchronizedForAllFollowersId");
             if(lastSync == default) throw new ArgumentException("lastSync");
 
-            var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId, lastSync = @lastSync WHERE id = @id";
+            var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId, fetchingErrorCount = @fetchingErrorCount, lastSync = @lastSync WHERE id = @id";
 
             using (var dbConnection = Connection)
             {
                 dbConnection.Open();
 
-                await dbConnection.QueryAsync(query, new { id, lastTweetPostedId,  lastTweetSynchronizedForAllFollowersId, lastSync = lastSync.ToUniversalTime() });
+                await dbConnection.QueryAsync(query, new { id, lastTweetPostedId,  lastTweetSynchronizedForAllFollowersId, fetchingErrorCount, lastSync = lastSync.ToUniversalTime() });
             }
         }
 
+        public async Task UpdateTwitterUserAsync(SyncTwitterUser user)
+        {
+            await UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, user.LastSync);
+        }
+
         public async Task DeleteTwitterUserAsync(string acct)
         {
             if (string.IsNullOrWhiteSpace(acct)) throw new ArgumentException("acct");

src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs

@@ -11,7 +11,8 @@ namespace BirdsiteLive.DAL.Contracts
         Task<SyncTwitterUser> GetTwitterUserAsync(int id);
         Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber);
         Task<SyncTwitterUser[]> GetAllTwitterUsersAsync();
-        Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, DateTime lastSync);
+        Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync);
+        Task UpdateTwitterUserAsync(SyncTwitterUser user);
         Task DeleteTwitterUserAsync(string acct);
         Task DeleteTwitterUserAsync(int id);
         Task<int> GetTwitterUsersCountAsync();

src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs

@@ -11,5 +11,7 @@ namespace BirdsiteLive.DAL.Models
         public long LastTweetSynchronizedForAllFollowersId { get; set; }
 
         public DateTime LastSync { get; set; }
+
+        public int FetchingErrorCount { get; set; } //TODO: update DAL
     }
 }

src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Diagnostics;
 using System.Threading.Tasks;
 using System.Xml;
 using BirdsiteLive.DAL.Postgres.DataAccessLayers;
@@ -47,6 +48,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             Assert.AreEqual(acct, result.Acct);
             Assert.AreEqual(lastTweetId, result.LastTweetPostedId);
             Assert.AreEqual(lastTweetId, result.LastTweetSynchronizedForAllFollowersId);
+            Assert.AreEqual(0, result.FetchingErrorCount);
             Assert.IsTrue(result.Id > 0);
         }
 
@@ -83,13 +85,47 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var updatedLastTweetId = 1600L;
             var updatedLastSyncId = 1550L;
             var now = DateTime.Now;
-            await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, now);
+            var errors = 15;
+            await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now);
 
             result = await dal.GetTwitterUserAsync(acct);
 
             Assert.AreEqual(acct, result.Acct);
             Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId);
             Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId);
+            Assert.AreEqual(errors, result.FetchingErrorCount);
+            Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100);
+        }
+
+        [TestMethod]
+        public async Task CreateUpdate2AndGetUser()
+        {
+            var acct = "myid";
+            var lastTweetId = 1548L;
+
+            var dal = new TwitterUserPostgresDal(_settings);
+
+            await dal.CreateTwitterUserAsync(acct, lastTweetId);
+            var result = await dal.GetTwitterUserAsync(acct);
+
+
+            var updatedLastTweetId = 1600L;
+            var updatedLastSyncId = 1550L;
+            var now = DateTime.Now;
+            var errors = 15;
+
+            result.LastTweetPostedId = updatedLastTweetId;
+            result.LastTweetSynchronizedForAllFollowersId = updatedLastSyncId;
+            result.FetchingErrorCount = errors;
+            result.LastSync = now;
+            await dal.UpdateTwitterUserAsync(result);
+
+            result = await dal.GetTwitterUserAsync(acct);
+
+            Assert.AreEqual(acct, result.Acct);
+            Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId);
+            Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId);
+            Assert.AreEqual(errors, result.FetchingErrorCount);
             Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100);
         }
 
@@ -98,7 +134,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
         public async Task Update_NoId()
         {
             var dal = new TwitterUserPostgresDal(_settings);
-            await dal.UpdateTwitterUserAsync(default, default, default, DateTime.UtcNow);
+            await dal.UpdateTwitterUserAsync(default, default, default, default, DateTime.UtcNow);
         }
 
         [TestMethod]
@@ -106,7 +142,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
         public async Task Update_NoLastTweetPostedId()
         {
             var dal = new TwitterUserPostgresDal(_settings);
-            await dal.UpdateTwitterUserAsync(12, default, default, DateTime.UtcNow);
+            await dal.UpdateTwitterUserAsync(12, default, default, default, DateTime.UtcNow);
         }
 
         [TestMethod]
@@ -114,7 +150,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
         public async Task Update_NoLastTweetSynchronizedForAllFollowersId()
         {
             var dal = new TwitterUserPostgresDal(_settings);
-            await dal.UpdateTwitterUserAsync(12, 9556, default, DateTime.UtcNow);
+            await dal.UpdateTwitterUserAsync(12, 9556, default, default, DateTime.UtcNow);
         }
 
         [TestMethod]
@@ -122,7 +158,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
         public async Task Update_NoLastSync()
         {
             var dal = new TwitterUserPostgresDal(_settings);
-            await dal.UpdateTwitterUserAsync(12, 9556, 65, default);
+            await dal.UpdateTwitterUserAsync(12, 9556, 65, default, default);
         }
 
         [TestMethod]
@@ -216,7 +252,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             {
                 var user = allUsers[i];
                 var date = i % 2 == 0 ? oldest : newest;
-                await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, date);
+                await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date);
             }
 
             var result = await dal.GetAllTwitterUsersAsync(10);

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveFollowersProcessorTests.cs

@@ -21,16 +21,16 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var userId1 = 1;
             var userId2 = 2;
 
-            var users = new List<UserWithTweetsToSync>
+            var users = new List<UserWithDataToSync>
             {
-                new UserWithTweetsToSync
+                new UserWithDataToSync
                 {
                     User = new SyncTwitterUser
                     {
                         Id = userId1
                     }
                 },
-                new UserWithTweetsToSync
+                new UserWithDataToSync
                 {
                     User = new SyncTwitterUser
                     {

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs

@@ -41,7 +41,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 }
             };
 
-            var usersWithTweets = new UserWithTweetsToSync
+            var usersWithTweets = new UserWithDataToSync
             {
                 Tweets = new []
                 {
@@ -65,6 +65,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                     It.Is<int>(y => y == user.Id),
                     It.Is<long>(y => y == tweet2.Id),
                     It.Is<long>(y => y == tweet2.Id),
+                    It.Is<int>(y => y == 0),
                     It.IsAny<DateTime>()
                 ))
                 .Returns(Task.CompletedTask);
@@ -107,7 +108,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 }
             };
 
-            var usersWithTweets = new UserWithTweetsToSync
+            var usersWithTweets = new UserWithDataToSync
             {
                 Tweets = new[]
                 {
@@ -130,6 +131,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                     It.Is<int>(y => y == user.Id),
                     It.Is<long>(y => y == tweet3.Id),
                     It.Is<long>(y => y == tweet2.Id),
+                    It.Is<int>(y => y == 0),
                     It.IsAny<DateTime>()
                 ))
                 .Returns(Task.CompletedTask);
@@ -181,7 +183,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 }
             };
 
-            var usersWithTweets = new UserWithTweetsToSync
+            var usersWithTweets = new UserWithDataToSync
             {
                 Tweets = new[]
                 {
@@ -205,6 +207,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                     It.Is<int>(y => y == user.Id),
                     It.Is<long>(y => y == tweet3.Id),
                     It.Is<long>(y => y == tweet2.Id),
+                    It.Is<int>(y => y == 0),
                     It.IsAny<DateTime>()
                 ))
                 .Returns(Task.CompletedTask);

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs

@@ -26,7 +26,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var userId2 = 3;
             var userAcct = "user";
 
-            var userWithTweets = new UserWithTweetsToSync()
+            var userWithTweets = new UserWithDataToSync()
             {
                 Tweets = new []
                 {
@@ -93,7 +93,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var userId2 = 3;
             var userAcct = "user";
 
-            var userWithTweets = new UserWithTweetsToSync()
+            var userWithTweets = new UserWithDataToSync()
             {
                 Tweets = new[]
                 {
@@ -163,7 +163,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var userId2 = 3;
             var userAcct = "user";
 
-            var userWithTweets = new UserWithTweetsToSync()
+            var userWithTweets = new UserWithDataToSync()
             {
                 Tweets = new[]
                 {
@@ -237,7 +237,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var userId2 = 3;
             var userAcct = "user";
 
-            var userWithTweets = new UserWithTweetsToSync()
+            var userWithTweets = new UserWithDataToSync()
             {
                 Tweets = new[]
                 {
@@ -306,7 +306,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var userId2 = 3;
             var userAcct = "user";
 
-            var userWithTweets = new UserWithTweetsToSync()
+            var userWithTweets = new UserWithDataToSync()
             {
                 Tweets = new[]
                 {
@@ -375,7 +375,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var userId2 = 3;
             var userAcct = "user";
 
-            var userWithTweets = new UserWithTweetsToSync()
+            var userWithTweets = new UserWithDataToSync()
             {
                 Tweets = new[]
                 {