diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs index 428b5eb..1214002 100644 --- a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs +++ b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs @@ -7,5 +7,6 @@ public string AdminEmail { get; set; } public bool ResolveMentionsInProfiles { get; set; } public bool PublishReplies { get; set; } + public int MaxUsersCapacity { get; set; } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj index 5d93cb1..884af18 100644 --- a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj +++ b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj @@ -17,4 +17,8 @@ + + + + diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs index ef20cad..ffcf9a9 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -45,7 +46,8 @@ namespace BirdsiteLive.Pipeline.Processors else if (tweets.Length > 0 && user.LastTweetPostedId == -1) { var tweetId = tweets.Last().Id; - await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId); + var now = DateTime.UtcNow; + await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, now); } } diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs index f556831..ebb87fc 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs @@ -4,9 +4,11 @@ using System.Threading; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; using BirdsiteLive.Common.Extensions; +using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Tools; using Microsoft.Extensions.Logging; namespace BirdsiteLive.Pipeline.Processors @@ -14,13 +16,16 @@ namespace BirdsiteLive.Pipeline.Processors public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor { private readonly ITwitterUserDal _twitterUserDal; + private readonly IMaxUsersNumberProvider _maxUsersNumberProvider; private readonly ILogger _logger; + public int WaitFactor = 1000 * 60; //1 min #region Ctor - public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, ILogger logger) + public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IMaxUsersNumberProvider maxUsersNumberProvider, ILogger logger) { _twitterUserDal = twitterUserDal; + _maxUsersNumberProvider = maxUsersNumberProvider; _logger = logger; } #endregion @@ -33,7 +38,8 @@ namespace BirdsiteLive.Pipeline.Processors try { - var users = await _twitterUserDal.GetAllTwitterUsersAsync(); + var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync(); + var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber); var userCount = users.Any() ? users.Length : 1; var splitNumber = (int) Math.Ceiling(userCount / 15d); diff --git a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs index 5b305e7..c7cbc36 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs @@ -1,4 +1,5 @@ -using System.Linq; +using System; +using System.Linq; using System.Threading; using System.Threading.Tasks; using BirdsiteLive.DAL.Contracts; @@ -23,7 +24,8 @@ namespace BirdsiteLive.Pipeline.Processors var userId = userWithTweetsToSync.User.Id; var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max(); var minimumSync = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).Min(); - await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync); + var now = DateTime.UtcNow; + await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, now); } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Tools/MaxUsersNumberProvider.cs b/src/BirdsiteLive.Pipeline/Tools/MaxUsersNumberProvider.cs new file mode 100644 index 0000000..c84b7b1 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Tools/MaxUsersNumberProvider.cs @@ -0,0 +1,49 @@ +using System.Threading.Tasks; +using BirdsiteLive.Common.Settings; +using BirdsiteLive.DAL.Contracts; + +namespace BirdsiteLive.Pipeline.Tools +{ + public interface IMaxUsersNumberProvider + { + Task GetMaxUsersNumberAsync(); + } + + public class MaxUsersNumberProvider : IMaxUsersNumberProvider + { + private readonly InstanceSettings _instanceSettings; + private readonly ITwitterUserDal _twitterUserDal; + + private int _totalUsersCount = -1; + private int _warmUpIterations; + + #region Ctor + public MaxUsersNumberProvider(InstanceSettings instanceSettings, ITwitterUserDal twitterUserDal) + { + _instanceSettings = instanceSettings; + _twitterUserDal = twitterUserDal; + } + #endregion + + public async Task GetMaxUsersNumberAsync() + { + // Init data + if (_totalUsersCount == -1) + { + _totalUsersCount = await _twitterUserDal.GetTwitterUsersCountAsync(); + var warmUpMaxCapacity = _instanceSettings.MaxUsersCapacity / 4; + _warmUpIterations = warmUpMaxCapacity == 0 ? 0 : (int)(_totalUsersCount / (float)warmUpMaxCapacity); + } + + // Return if warm up ended + if (_warmUpIterations <= 0) return _instanceSettings.MaxUsersCapacity; + + // Calculate warm up value + var maxUsers = _warmUpIterations > 0 + ? _instanceSettings.MaxUsersCapacity / 4 + : _instanceSettings.MaxUsersCapacity; + _warmUpIterations--; + return maxUsers; + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.sln b/src/BirdsiteLive.sln index bf78d55..0a35bf6 100644 --- a/src/BirdsiteLive.sln +++ b/src/BirdsiteLive.sln @@ -39,6 +39,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Domain.Tests", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Pipeline.Tests", "Tests\BirdsiteLive.Pipeline.Tests\BirdsiteLive.Pipeline.Tests.csproj", "{BF51CA81-5A7A-46F8-B4FB-861C6BE59298}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdsiteLive.DAL.Tests", "Tests\BirdsiteLive.DAL.Tests\BirdsiteLive.DAL.Tests.csproj", "{5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -101,6 +103,10 @@ Global {BF51CA81-5A7A-46F8-B4FB-861C6BE59298}.Debug|Any CPU.Build.0 = Debug|Any CPU {BF51CA81-5A7A-46F8-B4FB-861C6BE59298}.Release|Any CPU.ActiveCfg = Release|Any CPU {BF51CA81-5A7A-46F8-B4FB-861C6BE59298}.Release|Any CPU.Build.0 = Release|Any CPU + {5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -119,6 +125,7 @@ Global {2A8CC30D-D775-47D1-9388-F72A5C32DE2A} = {DA3C160C-4811-4E26-A5AD-42B81FAF2D7C} {F544D745-89A8-4DEA-B61C-A7E6C53C1D63} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94} {BF51CA81-5A7A-46F8-B4FB-861C6BE59298} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94} + {5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {69E8DCAD-4C37-4010-858F-5F94E6FBABCE} diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index 4729487..df9c11b 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -4,7 +4,7 @@ netcoreapp3.1 d21486de-a812-47eb-a419-05682bb68856 Linux - 0.10.1 + 0.11.0 diff --git a/src/BirdsiteLive/Services/FederationService.cs b/src/BirdsiteLive/Services/FederationService.cs index f2c2e94..9acab41 100644 --- a/src/BirdsiteLive/Services/FederationService.cs +++ b/src/BirdsiteLive/Services/FederationService.cs @@ -1,6 +1,8 @@ using System; +using System.Linq; using System.Threading; using System.Threading.Tasks; +using BirdsiteLive.DAL; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.Pipeline; using Microsoft.Extensions.Hosting; @@ -9,36 +11,21 @@ namespace BirdsiteLive.Services { public class FederationService : BackgroundService { - private readonly IDbInitializerDal _dbInitializerDal; + private readonly IDatabaseInitializer _databaseInitializer; private readonly IStatusPublicationPipeline _statusPublicationPipeline; #region Ctor - public FederationService(IDbInitializerDal dbInitializerDal, IStatusPublicationPipeline statusPublicationPipeline) + public FederationService(IDatabaseInitializer databaseInitializer, IStatusPublicationPipeline statusPublicationPipeline) { - _dbInitializerDal = dbInitializerDal; + _databaseInitializer = databaseInitializer; _statusPublicationPipeline = statusPublicationPipeline; } #endregion protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - await DbInitAsync(); + await _databaseInitializer.InitAndMigrateDbAsync(); await _statusPublicationPipeline.ExecuteAsync(stoppingToken); } - - private async Task DbInitAsync() - { - var currentVersion = await _dbInitializerDal.GetCurrentDbVersionAsync(); - var mandatoryVersion = _dbInitializerDal.GetMandatoryDbVersion(); - - if (currentVersion == null) - { - await _dbInitializerDal.InitDbAsync(); - } - else if (currentVersion != mandatoryVersion) - { - throw new NotImplementedException(); - } - } } } \ No newline at end of file diff --git a/src/BirdsiteLive/appsettings.json b/src/BirdsiteLive/appsettings.json index 2e566ff..3dc47ff 100644 --- a/src/BirdsiteLive/appsettings.json +++ b/src/BirdsiteLive/appsettings.json @@ -14,7 +14,8 @@ "Domain": "domain.name", "AdminEmail": "me@domain.name", "ResolveMentionsInProfiles": true, - "PublishReplies": false + "PublishReplies": false, + "MaxUsersCapacity": 1400 }, "Db": { "Type": "postgres", diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs index c6d5c81..ca883ff 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs @@ -23,7 +23,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public class DbInitializerPostgresDal : PostgresBase, IDbInitializerDal { private readonly PostgresTools _tools; - private readonly Version _currentVersion = new Version(1,0); + private readonly Version _currentVersion = new Version(2, 0); private const string DbVersionType = "db-version"; #region Ctor @@ -32,7 +32,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers _tools = tools; } #endregion - + public async Task GetCurrentDbVersionAsync() { var query = $"SELECT * FROM {_settings.DbVersionTableName} WHERE type = @type"; @@ -65,17 +65,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers return _currentVersion; } - public Tuple[] GetMigrationPatterns() - { - return new Tuple[0]; - } - - public Task MigrateDbAsync(Version from, Version to) - { - throw new NotImplementedException(); - } - - public async Task InitDbAsync() + public async Task InitDbAsync() { // Create version table var createVersion = $@"CREATE TABLE {_settings.DbVersionTableName} @@ -124,13 +114,53 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers await _tools.ExecuteRequestAsync(createCachedTweets); // Insert version to db + var firstVersion = new Version(1, 0); using (var dbConnection = Connection) { dbConnection.Open(); await dbConnection.ExecuteAsync( $"INSERT INTO {_settings.DbVersionTableName} (type,major,minor) VALUES(@type,@major,@minor)", - new { type = DbVersionType, major = _currentVersion.Major, minor = _currentVersion.Minor }); + new { type = DbVersionType, major = firstVersion.Major, minor = firstVersion.Minor }); + } + + return firstVersion; + } + + public Tuple[] GetMigrationPatterns() + { + return new[] + { + new Tuple(new Version(1,0), new Version(2,0)) + }; + } + + public async Task MigrateDbAsync(Version from, Version to) + { + if (from == new Version(1, 0) && to == new Version(2, 0)) + { + var addLastSync = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD lastSync TIMESTAMP (2) WITHOUT TIME ZONE"; + await _tools.ExecuteRequestAsync(addLastSync); + + var addIndex = $@"CREATE INDEX IF NOT EXISTS lastsync_twitteruser ON {_settings.TwitterUserTableName}(lastSync)"; + await _tools.ExecuteRequestAsync(addIndex); + + await UpdateDbVersionAsync(to); + return to; + } + + throw new NotImplementedException(); + } + + private async Task UpdateDbVersionAsync(Version newVersion) + { + using (var dbConnection = Connection) + { + dbConnection.Open(); + + await dbConnection.ExecuteAsync( + $"UPDATE {_settings.DbVersionTableName} SET major = @major, minor = @minor WHERE type = @type", + new { type = DbVersionType, major = newVersion.Major, minor = newVersion.Minor }); } } diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs index 082229a..afbf7d1 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs @@ -62,32 +62,33 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } } - public async Task GetAllTwitterUsersAsync() + public async Task GetAllTwitterUsersAsync(int maxNumber) { - var query = $"SELECT * FROM {_settings.TwitterUserTableName}"; + var query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC LIMIT @maxNumber"; using (var dbConnection = Connection) { dbConnection.Open(); - var result = await dbConnection.QueryAsync(query); + var result = await dbConnection.QueryAsync(query, new { maxNumber }); return result.ToArray(); } } - public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId) + public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, DateTime lastSync) { if(id == default) throw new ArgumentException("id"); if(lastTweetPostedId == default) throw new ArgumentException("lastTweetPostedId"); if(lastTweetSynchronizedForAllFollowersId == default) throw new ArgumentException("lastTweetSynchronizedForAllFollowersId"); - - var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId WHERE id = @id"; + if(lastSync == default) throw new ArgumentException("lastSync"); + + var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId, lastSync = @lastSync WHERE id = @id"; using (var dbConnection = Connection) { dbConnection.Open(); - await dbConnection.QueryAsync(query, new { id, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId }); + await dbConnection.QueryAsync(query, new { id, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId, lastSync = lastSync.ToUniversalTime() }); } } diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Tools/PostgresTools.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Tools/PostgresTools.cs index 223b1ea..32a379f 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Tools/PostgresTools.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Tools/PostgresTools.cs @@ -18,18 +18,11 @@ namespace BirdsiteLive.DAL.Postgres.Tools public async Task ExecuteRequestAsync(string request) { - try + using (var conn = new NpgsqlConnection(_settings.ConnString)) + using (var cmd = new NpgsqlCommand(request, conn)) { - using (var conn = new NpgsqlConnection(_settings.ConnString)) - using (var cmd = new NpgsqlCommand(request, conn)) - { - await conn.OpenAsync(); - await cmd.ExecuteNonQueryAsync(); - } - } - catch (Exception e) - { - Console.WriteLine(e); + await conn.OpenAsync(); + await cmd.ExecuteNonQueryAsync(); } } } diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IDbInitializerDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IDbInitializerDal.cs index b786386..9d7db56 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IDbInitializerDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IDbInitializerDal.cs @@ -9,7 +9,7 @@ namespace BirdsiteLive.DAL.Contracts Task GetCurrentDbVersionAsync(); Version GetMandatoryDbVersion(); Tuple[] GetMigrationPatterns(); - Task MigrateDbAsync(Version from, Version to); - Task InitDbAsync(); + Task MigrateDbAsync(Version from, Version to); + Task InitDbAsync(); } } \ No newline at end of file diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs index 48d5661..1fa8127 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs @@ -1,4 +1,5 @@ -using System.Threading.Tasks; +using System; +using System.Threading.Tasks; using BirdsiteLive.DAL.Models; namespace BirdsiteLive.DAL.Contracts @@ -7,8 +8,8 @@ namespace BirdsiteLive.DAL.Contracts { Task CreateTwitterUserAsync(string acct, long lastTweetPostedId); Task GetTwitterUserAsync(string acct); - Task GetAllTwitterUsersAsync(); - Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId); + Task GetAllTwitterUsersAsync(int maxNumber); + Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, DateTime lastSync); Task DeleteTwitterUserAsync(string acct); Task GetTwitterUsersCountAsync(); } diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/DatabaseInitializer.cs b/src/DataAccessLayers/BirdsiteLive.DAL/DatabaseInitializer.cs new file mode 100644 index 0000000..39e1e84 --- /dev/null +++ b/src/DataAccessLayers/BirdsiteLive.DAL/DatabaseInitializer.cs @@ -0,0 +1,46 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; + +namespace BirdsiteLive.DAL +{ + public interface IDatabaseInitializer + { + Task InitAndMigrateDbAsync(); + } + + public class DatabaseInitializer : IDatabaseInitializer + { + private readonly IDbInitializerDal _dbInitializerDal; + + #region Ctor + public DatabaseInitializer(IDbInitializerDal dbInitializerDal) + { + _dbInitializerDal = dbInitializerDal; + } + #endregion + + public async Task InitAndMigrateDbAsync() + { + var currentVersion = await _dbInitializerDal.GetCurrentDbVersionAsync(); + var mandatoryVersion = _dbInitializerDal.GetMandatoryDbVersion(); + + if (currentVersion == mandatoryVersion) return; + + // Init Db + if (currentVersion == null) + currentVersion = await _dbInitializerDal.InitDbAsync(); + + // Migrate Db + var migrationPatterns = _dbInitializerDal.GetMigrationPatterns(); + while (migrationPatterns.Any(x => x.Item1 == currentVersion)) + { + var migration = migrationPatterns.First(x => x.Item1 == currentVersion); + currentVersion = await _dbInitializerDal.MigrateDbAsync(migration.Item1, migration.Item2); + } + + if (currentVersion != mandatoryVersion) throw new Exception("Migrating DB failed"); + } + } +} \ No newline at end of file diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs index 8061fc8..59be0a5 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs @@ -1,4 +1,6 @@ -namespace BirdsiteLive.DAL.Models +using System; + +namespace BirdsiteLive.DAL.Models { public class SyncTwitterUser { @@ -7,5 +9,7 @@ public long LastTweetPostedId { get; set; } public long LastTweetSynchronizedForAllFollowersId { get; set; } + + public DateTime LastSync { get; set; } } } \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/CachedTweetsPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/CachedTweetsPostgresDalTests.cs index 021784e..24672c3 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/CachedTweetsPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/CachedTweetsPostgresDalTests.cs @@ -14,21 +14,15 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task TestInit() { var dal = new DbInitializerPostgresDal(_settings, _tools); - await dal.InitDbAsync(); + var init = new DatabaseInitializer(dal); + await init.InitAndMigrateDbAsync(); } [TestCleanup] public async Task CleanUp() { var dal = new DbInitializerPostgresDal(_settings, _tools); - try - { - await dal.DeleteAllAsync(); - } - catch (Exception e) - { - Console.WriteLine(e); - } + await dal.DeleteAllAsync(); } [TestMethod] diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/DbInitializerPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/DbInitializerPostgresDalTests.cs index 7fc5383..a186dc3 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/DbInitializerPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/DbInitializerPostgresDalTests.cs @@ -17,17 +17,14 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers { await dal.DeleteAllAsync(); } - catch (Exception e) - { - Console.WriteLine(e); - } + catch (Exception ) { } } [TestMethod] public async Task GetCurrentDbVersionAsync_UninitializedDb() { var dal = new DbInitializerPostgresDal(_settings, _tools); - + var current = await dal.GetCurrentDbVersionAsync(); Assert.IsNull(current); } @@ -35,11 +32,11 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers [TestMethod] public async Task InitDbAsync() { + var mandatory = new Version(1, 0); var dal = new DbInitializerPostgresDal(_settings, _tools); await dal.InitDbAsync(); var current = await dal.GetCurrentDbVersionAsync(); - var mandatory = dal.GetMandatoryDbVersion(); Assert.IsNotNull(current); Assert.AreEqual(mandatory.Minor, current.Minor); Assert.AreEqual(mandatory.Major, current.Major); diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs index cd6162d..cf08856 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs @@ -16,21 +16,15 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task TestInit() { var dal = new DbInitializerPostgresDal(_settings, _tools); - await dal.InitDbAsync(); + var init = new DatabaseInitializer(dal); + await init.InitAndMigrateDbAsync(); } [TestCleanup] public async Task CleanUp() { var dal = new DbInitializerPostgresDal(_settings, _tools); - try - { - await dal.DeleteAllAsync(); - } - catch (Exception e) - { - Console.WriteLine(e); - } + await dal.DeleteAllAsync(); } [TestMethod] @@ -38,7 +32,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers { var acct = "myhandle"; var host = "domain.ext"; - var following = new[] {12, 19, 23}; + var following = new[] { 12, 19, 23 }; var followingSync = new Dictionary() { {12, 165L}, @@ -47,7 +41,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers }; var inboxRoute = "/myhandle/inbox"; var sharedInboxRoute = "/inbox"; - + var dal = new FollowersPostgresDal(_settings); await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, following, followingSync); @@ -105,7 +99,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers //User 1 var acct = "myhandle1"; var host = "domain.ext"; - var following = new[] { 1,2,3 }; + var following = new[] { 1, 2, 3 }; var followingSync = new Dictionary(); var inboxRoute = "/myhandle1/inbox"; var sharedInboxRoute = "/inbox"; @@ -202,7 +196,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers }; result.Followings = updatedFollowing.ToList(); result.FollowingsSyncStatus = updatedFollowingSync; - + await dal.UpdateFollowerAsync(result); result = await dal.GetFollowerAsync(acct, host); diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs index f900c8f..d71842f 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs @@ -14,21 +14,15 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task TestInit() { var dal = new DbInitializerPostgresDal(_settings, _tools); - await dal.InitDbAsync(); + var init = new DatabaseInitializer(dal); + await init.InitAndMigrateDbAsync(); } [TestCleanup] public async Task CleanUp() { var dal = new DbInitializerPostgresDal(_settings, _tools); - try - { - await dal.DeleteAllAsync(); - } - catch (Exception e) - { - Console.WriteLine(e); - } + await dal.DeleteAllAsync(); } [TestMethod] @@ -70,13 +64,15 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers var updatedLastTweetId = 1600L; var updatedLastSyncId = 1550L; - await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId); + var now = DateTime.Now; + await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, now); result = await dal.GetTwitterUserAsync(acct); Assert.AreEqual(acct, result.Acct); Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId); Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId); + Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100); } [TestMethod] @@ -108,7 +104,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers await dal.CreateTwitterUserAsync(acct, lastTweetId); } - var result = await dal.GetAllTwitterUsersAsync(); + var result = await dal.GetAllTwitterUsersAsync(1000); Assert.AreEqual(1000, result.Length); Assert.IsFalse(result[0].Id == default); Assert.IsFalse(result[0].Acct == default); @@ -116,6 +112,41 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default); } + [TestMethod] + public async Task GetAllTwitterUsers_Limited() + { + var now = DateTime.Now; + var oldest = now.AddDays(-3); + var newest = now.AddDays(-2); + + var dal = new TwitterUserPostgresDal(_settings); + for (var i = 0; i < 20; i++) + { + var acct = $"myid{i}"; + var lastTweetId = 1548L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + } + + var allUsers = await dal.GetAllTwitterUsersAsync(100); + for (var i = 0; i < 20; i++) + { + var user = allUsers[i]; + var date = i % 2 == 0 ? oldest : newest; + await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, date); + } + + var result = await dal.GetAllTwitterUsersAsync(10); + Assert.AreEqual(10, result.Length); + Assert.IsFalse(result[0].Id == default); + Assert.IsFalse(result[0].Acct == default); + Assert.IsFalse(result[0].LastTweetPostedId == default); + Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default); + + foreach (var acc in result) + Assert.IsTrue(Math.Abs((acc.LastSync - oldest.ToUniversalTime()).TotalMilliseconds) < 1000); + } + [TestMethod] public async Task CountTwitterUsers() { diff --git a/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj b/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj new file mode 100644 index 0000000..0992b02 --- /dev/null +++ b/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj @@ -0,0 +1,21 @@ + + + + netcoreapp3.1 + + false + + + + + + + + + + + + + + + diff --git a/src/Tests/BirdsiteLive.DAL.Tests/DatabaseInitializerTests.cs b/src/Tests/BirdsiteLive.DAL.Tests/DatabaseInitializerTests.cs new file mode 100644 index 0000000..ba11321 --- /dev/null +++ b/src/Tests/BirdsiteLive.DAL.Tests/DatabaseInitializerTests.cs @@ -0,0 +1,240 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace BirdsiteLive.DAL.Tests +{ + [TestClass] + public class DatabaseInitializerTests + { + [TestMethod] + public async Task DbInitAsync_UpToDate_Test() + { + #region Stubs + var current = new Version(2, 3); + var mandatory = new Version(2, 3); + #endregion + + #region Mocks + var dbInitializerDal = new Mock(MockBehavior.Strict); + + dbInitializerDal + .Setup(x => x.GetCurrentDbVersionAsync()) + .ReturnsAsync(current); + + dbInitializerDal + .Setup(x => x.GetMandatoryDbVersion()) + .Returns(mandatory); + #endregion + + var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object); + await dbInitializer.InitAndMigrateDbAsync(); + + #region Validations + dbInitializerDal.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task DbInitAsync_NoDb_Test() + { + #region Stubs + var current = (Version)null; + var mandatory = new Version(1, 0); + + var migrationPatterns = new Tuple[0]; + #endregion + + #region Mocks + var dbInitializerDal = new Mock(MockBehavior.Strict); + + dbInitializerDal + .Setup(x => x.GetCurrentDbVersionAsync()) + .ReturnsAsync(current); + + dbInitializerDal + .Setup(x => x.GetMandatoryDbVersion()) + .Returns(mandatory); + + dbInitializerDal + .Setup(x => x.InitDbAsync()) + .ReturnsAsync(new Version(1, 0)); + + dbInitializerDal + .Setup(x => x.GetMigrationPatterns()) + .Returns(migrationPatterns); + #endregion + + var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object); + await dbInitializer.InitAndMigrateDbAsync(); + + #region Validations + dbInitializerDal.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task DbInitAsync_NoDb_Migration_Test() + { + #region Stubs + var current = (Version)null; + var mandatory = new Version(2, 3); + + var migrationPatterns = new Tuple[] + { + new Tuple(new Version(1,0), new Version(1,7)), + new Tuple(new Version(1,7), new Version(2,0)), + new Tuple(new Version(2,0), new Version(2,3)) + }; + #endregion + + #region Mocks + var dbInitializerDal = new Mock(MockBehavior.Strict); + + dbInitializerDal + .Setup(x => x.GetCurrentDbVersionAsync()) + .ReturnsAsync(current); + + dbInitializerDal + .Setup(x => x.GetMandatoryDbVersion()) + .Returns(mandatory); + + dbInitializerDal + .Setup(x => x.InitDbAsync()) + .ReturnsAsync(new Version(1, 0)); + + dbInitializerDal + .Setup(x => x.GetMigrationPatterns()) + .Returns(migrationPatterns); + + foreach (var m in migrationPatterns) + { + dbInitializerDal + .Setup(x => x.MigrateDbAsync( + It.Is(y => y == m.Item1), + It.Is(y => y == m.Item2) + )) + .ReturnsAsync(m.Item2); + } + #endregion + + var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object); + await dbInitializer.InitAndMigrateDbAsync(); + + #region Validations + dbInitializerDal.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task DbInitAsync_HasDb_Migration_Test() + { + #region Stubs + var current = new Version(1, 7); + var mandatory = new Version(2, 3); + + var migrationPatterns = new Tuple[] + { + new Tuple(new Version(1,0), new Version(1,7)), + new Tuple(new Version(1,7), new Version(2,0)), + new Tuple(new Version(2,0), new Version(2,3)) + }; + #endregion + + #region Mocks + var dbInitializerDal = new Mock(MockBehavior.Strict); + + dbInitializerDal + .Setup(x => x.GetCurrentDbVersionAsync()) + .ReturnsAsync(current); + + dbInitializerDal + .Setup(x => x.GetMandatoryDbVersion()) + .Returns(mandatory); + + dbInitializerDal + .Setup(x => x.GetMigrationPatterns()) + .Returns(migrationPatterns); + + foreach (var m in migrationPatterns.Skip(1)) + { + dbInitializerDal + .Setup(x => x.MigrateDbAsync( + It.Is(y => y == m.Item1), + It.Is(y => y == m.Item2) + )) + .ReturnsAsync(m.Item2); + } + #endregion + + var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object); + await dbInitializer.InitAndMigrateDbAsync(); + + #region Validations + dbInitializerDal.VerifyAll(); + #endregion + } + + [TestMethod] + [ExpectedException(typeof(Exception))] + public async Task DbInitAsync_NoDb_Migration_Error_Test() + { + #region Stubs + var current = (Version)null; + var mandatory = new Version(2, 3); + + var migrationPatterns = new Tuple[] + { + new Tuple(new Version(1,0), new Version(1,7)), + new Tuple(new Version(1,7), new Version(2,0)), + new Tuple(new Version(2,0), new Version(2,2)) + }; + #endregion + + #region Mocks + var dbInitializerDal = new Mock(MockBehavior.Strict); + + dbInitializerDal + .Setup(x => x.GetCurrentDbVersionAsync()) + .ReturnsAsync(current); + + dbInitializerDal + .Setup(x => x.GetMandatoryDbVersion()) + .Returns(mandatory); + + dbInitializerDal + .Setup(x => x.InitDbAsync()) + .ReturnsAsync(new Version(1, 0)); + + dbInitializerDal + .Setup(x => x.GetMigrationPatterns()) + .Returns(migrationPatterns); + + foreach (var m in migrationPatterns) + { + dbInitializerDal + .Setup(x => x.MigrateDbAsync( + It.Is(y => y == m.Item1), + It.Is(y => y == m.Item2) + )) + .ReturnsAsync(m.Item2); + } + #endregion + + var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object); + try + { + await dbInitializer.InitAndMigrateDbAsync(); + } + finally + { + #region Validations + dbInitializerDal.VerifyAll(); + #endregion + } + } + } +} diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj b/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj index 3dd6984..aa7750b 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj @@ -18,4 +18,8 @@ + + + + diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs index 2bf5d74..d66c2f7 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs @@ -1,3 +1,4 @@ +using System; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -54,7 +55,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Setup(x => x.UpdateTwitterUserAsync( It.Is(y => y == user1.Id), It.Is(y => y == tweets.Last().Id), - It.Is(y => y == tweets.Last().Id) + It.Is(y => y == tweets.Last().Id), + It.IsAny() )) .Returns(Task.CompletedTask); #endregion diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs index 12c5682..5600e1c 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs @@ -3,9 +3,11 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; +using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Processors; +using BirdsiteLive.Pipeline.Tools; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -26,24 +28,32 @@ namespace BirdsiteLive.Pipeline.Tests.Processors new SyncTwitterUser(), new SyncTwitterUser(), }; + var maxUsers = 1000; #endregion #region Mocks + var maxUsersNumberProviderMock = new Mock(MockBehavior.Strict); + maxUsersNumberProviderMock + .Setup(x => x.GetMaxUsersNumberAsync()) + .ReturnsAsync(maxUsers); + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync( + It.Is(y => y == maxUsers))) .ReturnsAsync(users); - + var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 10; processor.GetTwitterUsersAsync(buffer, CancellationToken.None); await Task.Delay(50); #region Validations + maxUsersNumberProviderMock.VerifyAll(); twitterUserDalMock.VerifyAll(); Assert.AreEqual(3, buffer.Count); buffer.TryReceive(out var result); @@ -60,25 +70,37 @@ namespace BirdsiteLive.Pipeline.Tests.Processors for (var i = 0; i < 30; i++) users.Add(new SyncTwitterUser()); + + var maxUsers = 1000; #endregion #region Mocks + var maxUsersNumberProviderMock = new Mock(MockBehavior.Strict); + maxUsersNumberProviderMock + .Setup(x => x.GetMaxUsersNumberAsync()) + .ReturnsAsync(maxUsers); + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .SetupSequence(x => x.GetAllTwitterUsersAsync()) + .SetupSequence(x => x.GetAllTwitterUsersAsync( + It.Is(y => y == maxUsers))) .ReturnsAsync(users.ToArray()) + .ReturnsAsync(new SyncTwitterUser[0]) + .ReturnsAsync(new SyncTwitterUser[0]) + .ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0]); var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 2; processor.GetTwitterUsersAsync(buffer, CancellationToken.None); await Task.Delay(300); #region Validations + maxUsersNumberProviderMock.VerifyAll(); twitterUserDalMock.VerifyAll(); Assert.AreEqual(15, buffer.Count); buffer.TryReceive(out var result); @@ -95,25 +117,37 @@ namespace BirdsiteLive.Pipeline.Tests.Processors for (var i = 0; i < 31; i++) users.Add(new SyncTwitterUser()); + + var maxUsers = 1000; #endregion #region Mocks + var maxUsersNumberProviderMock = new Mock(MockBehavior.Strict); + maxUsersNumberProviderMock + .Setup(x => x.GetMaxUsersNumberAsync()) + .ReturnsAsync(maxUsers); + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .SetupSequence(x => x.GetAllTwitterUsersAsync()) + .SetupSequence(x => x.GetAllTwitterUsersAsync( + It.Is(y => y == maxUsers))) .ReturnsAsync(users.ToArray()) + .ReturnsAsync(new SyncTwitterUser[0]) + .ReturnsAsync(new SyncTwitterUser[0]) + .ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0]); - + var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 2; processor.GetTwitterUsersAsync(buffer, CancellationToken.None); await Task.Delay(200); #region Validations + maxUsersNumberProviderMock.VerifyAll(); twitterUserDalMock.VerifyAll(); Assert.AreEqual(11, buffer.Count); buffer.TryReceive(out var result); @@ -126,24 +160,33 @@ namespace BirdsiteLive.Pipeline.Tests.Processors { #region Stubs var buffer = new BufferBlock(); + + var maxUsers = 1000; #endregion #region Mocks + var maxUsersNumberProviderMock = new Mock(MockBehavior.Strict); + maxUsersNumberProviderMock + .Setup(x => x.GetMaxUsersNumberAsync()) + .ReturnsAsync(maxUsers); + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync( + It.Is(y => y == maxUsers))) .ReturnsAsync(new SyncTwitterUser[0]); var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 1; processor.GetTwitterUsersAsync(buffer, CancellationToken.None); await Task.Delay(50); #region Validations + maxUsersNumberProviderMock.VerifyAll(); twitterUserDalMock.VerifyAll(); Assert.AreEqual(0, buffer.Count); #endregion @@ -154,24 +197,33 @@ namespace BirdsiteLive.Pipeline.Tests.Processors { #region Stubs var buffer = new BufferBlock(); + + var maxUsers = 1000; #endregion #region Mocks + var maxUsersNumberProviderMock = new Mock(MockBehavior.Strict); + maxUsersNumberProviderMock + .Setup(x => x.GetMaxUsersNumberAsync()) + .ReturnsAsync(maxUsers); + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync( + It.Is(y => y == maxUsers))) .Returns(async () => await DelayFaultedTask(new Exception())); var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 10; var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None); await Task.WhenAny(t, Task.Delay(50)); #region Validations + maxUsersNumberProviderMock.VerifyAll(); twitterUserDalMock.VerifyAll(); Assert.AreEqual(0, buffer.Count); #endregion @@ -185,14 +237,22 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var buffer = new BufferBlock(); var canTokenS = new CancellationTokenSource(); canTokenS.Cancel(); + + var maxUsers = 1000; #endregion #region Mocks + var maxUsersNumberProviderMock = new Mock(MockBehavior.Strict); + maxUsersNumberProviderMock + .Setup(x => x.GetMaxUsersNumberAsync()) + .ReturnsAsync(maxUsers); + var twitterUserDalMock = new Mock(MockBehavior.Strict); + var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 1; await processor.GetTwitterUsersAsync(buffer, canTokenS.Token); } diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs index d3880e6..b2a99b9 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using BirdsiteLive.DAL.Contracts; @@ -60,7 +61,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Setup(x => x.UpdateTwitterUserAsync( It.Is(y => y == user.Id), It.Is(y => y == tweet2.Id), - It.Is(y => y == tweet2.Id) + It.Is(y => y == tweet2.Id), + It.IsAny() )) .Returns(Task.CompletedTask); #endregion @@ -123,7 +125,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Setup(x => x.UpdateTwitterUserAsync( It.Is(y => y == user.Id), It.Is(y => y == tweet3.Id), - It.Is(y => y == tweet2.Id) + It.Is(y => y == tweet2.Id), + It.IsAny() )) .Returns(Task.CompletedTask); #endregion @@ -194,7 +197,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Setup(x => x.UpdateTwitterUserAsync( It.Is(y => y == user.Id), It.Is(y => y == tweet3.Id), - It.Is(y => y == tweet2.Id) + It.Is(y => y == tweet2.Id), + It.IsAny() )) .Returns(Task.CompletedTask); #endregion diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Tools/MaxUsersNumberProviderTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Tools/MaxUsersNumberProviderTests.cs new file mode 100644 index 0000000..d48beb8 --- /dev/null +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Tools/MaxUsersNumberProviderTests.cs @@ -0,0 +1,79 @@ +using System.Threading.Tasks; +using BirdsiteLive.Common.Settings; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.Pipeline.Tools; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace BirdsiteLive.Pipeline.Tests.Tools +{ + [TestClass] + public class MaxUsersNumberProviderTests + { + [TestMethod] + public async Task GetMaxUsersNumberAsync_WarmUp_Test() + { + #region Stubs + var settings = new InstanceSettings + { + MaxUsersCapacity = 1000 + }; + #endregion + + #region Mocks + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.GetTwitterUsersCountAsync()) + .ReturnsAsync(1000); + #endregion + + var provider = new MaxUsersNumberProvider(settings, twitterUserDalMock.Object); + + var result = await provider.GetMaxUsersNumberAsync(); + Assert.AreEqual(250, result); + + result = await provider.GetMaxUsersNumberAsync(); + Assert.AreEqual(250, result); + + result = await provider.GetMaxUsersNumberAsync(); + Assert.AreEqual(250, result); + + result = await provider.GetMaxUsersNumberAsync(); + Assert.AreEqual(250, result); + + result = await provider.GetMaxUsersNumberAsync(); + Assert.AreEqual(1000, result); + + #region Validations + twitterUserDalMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task GetMaxUsersNumberAsync_NoWarmUp_Test() + { + #region Stubs + var settings = new InstanceSettings + { + MaxUsersCapacity = 1000 + }; + #endregion + + #region Mocks + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.GetTwitterUsersCountAsync()) + .ReturnsAsync(249); + #endregion + + var provider = new MaxUsersNumberProvider(settings, twitterUserDalMock.Object); + + var result = await provider.GetMaxUsersNumberAsync(); + Assert.AreEqual(1000, result); + + #region Validations + twitterUserDalMock.VerifyAll(); + #endregion + } + } +} \ No newline at end of file