diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs index 25835c6..228a252 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs @@ -60,7 +60,7 @@ namespace BirdsiteLive.Pipeline.Processors await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now); } - await Task.Delay(2 * 1000); + await Task.Delay(1000); } return usersWtTweets.ToArray(); diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs index d08d9cb..5ad7ba6 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs @@ -42,7 +42,7 @@ namespace BirdsiteLive.Pipeline.Processors //var splitUsers = users.Split(25).ToList(); var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync(); - var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber); + var users = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(maxUsersNumber); var userCount = users.Any() ? Math.Min(users.Length, 25) : 1; //var splitNumber = (int) Math.Ceiling(userCount / 15d); diff --git a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs index 2df326a..6192016 100644 --- a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs +++ b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs @@ -39,7 +39,7 @@ namespace BirdsiteLive.Twitter.Tools _logger = logger; aTimer = new System.Timers.Timer(); - aTimer.Interval = 900000; // 15 minutes + aTimer.Interval = 5 * 60 * 1000; aTimer.Elapsed += async (sender, e) => await RefreshCred(); aTimer.Start(); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs index 11214d4..32af8a0 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs @@ -86,6 +86,19 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } } + public async Task GetAllTwitterUsersWithFollowersAsync(int maxNumber) + { + var query = "SELECT * FROM (SELECT unnest(followings) as follow FROM followers GROUP BY follow) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id LIMIT @maxNumber"; + + using (var dbConnection = Connection) + { + dbConnection.Open(); + + var result = await dbConnection.QueryAsync(query, new { maxNumber }); + return result.ToArray(); + } + } + public async Task GetAllTwitterUsersAsync(int maxNumber) { var query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber"; diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs index ef2cc36..f6b592f 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs @@ -9,6 +9,7 @@ namespace BirdsiteLive.DAL.Contracts Task CreateTwitterUserAsync(string acct, long lastTweetPostedId); Task GetTwitterUserAsync(string acct); Task GetTwitterUserAsync(int id); + Task GetAllTwitterUsersWithFollowersAsync(int maxNumber); Task GetAllTwitterUsersAsync(int maxNumber); Task GetAllTwitterUsersAsync(); Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync); diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs index 71951f5..9459cb4 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs @@ -39,7 +39,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync( + .Setup(x => x.GetAllTwitterUsersWithFollowersAsync( It.Is(y => y == maxUsers))) .ReturnsAsync(users); @@ -82,7 +82,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .SetupSequence(x => x.GetAllTwitterUsersAsync( + .SetupSequence(x => x.GetAllTwitterUsersWithFollowersAsync( It.Is(y => y == maxUsers))) .ReturnsAsync(users.ToArray()) .ReturnsAsync(new SyncTwitterUser[0]) @@ -129,7 +129,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .SetupSequence(x => x.GetAllTwitterUsersAsync( + .SetupSequence(x => x.GetAllTwitterUsersWithFollowersAsync( It.Is(y => y == maxUsers))) .ReturnsAsync(users.ToArray()) .ReturnsAsync(new SyncTwitterUser[0]) @@ -143,13 +143,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 2; var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None); - var t2 = Task.Run(async () => - { - while (buffer.Count < 11) - await Task.Delay(50); - }); - await Task.WhenAny(t, t2, Task.Delay(5000)); + await Task.WhenAny(t, Task.Delay(5000)); #region Validations maxUsersNumberProviderMock.VerifyAll(); @@ -177,7 +172,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync( + .Setup(x => x.GetAllTwitterUsersWithFollowersAsync( It.Is(y => y == maxUsers))) .ReturnsAsync(new SyncTwitterUser[0]); @@ -214,7 +209,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync( + .Setup(x => x.GetAllTwitterUsersWithFollowersAsync( It.Is(y => y == maxUsers))) .Returns(async () => await DelayFaultedTask(new Exception()));