From e54eebc9b51977a82064c1807512f206cca1b2a3 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 22 Jan 2021 19:44:47 -0500 Subject: [PATCH 01/12] added MaxUserCapacity setting --- src/BirdsiteLive.Common/Settings/InstanceSettings.cs | 1 + src/BirdsiteLive/appsettings.json | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs index 428b5eb..1214002 100644 --- a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs +++ b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs @@ -7,5 +7,6 @@ public string AdminEmail { get; set; } public bool ResolveMentionsInProfiles { get; set; } public bool PublishReplies { get; set; } + public int MaxUsersCapacity { get; set; } } } \ No newline at end of file diff --git a/src/BirdsiteLive/appsettings.json b/src/BirdsiteLive/appsettings.json index 2e566ff..3dc47ff 100644 --- a/src/BirdsiteLive/appsettings.json +++ b/src/BirdsiteLive/appsettings.json @@ -14,7 +14,8 @@ "Domain": "domain.name", "AdminEmail": "me@domain.name", "ResolveMentionsInProfiles": true, - "PublishReplies": false + "PublishReplies": false, + "MaxUsersCapacity": 1400 }, "Db": { "Type": "postgres", From 8708a529d68f9222411c48874bad79ee09684f2b Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 22 Jan 2021 20:17:22 -0500 Subject: [PATCH 02/12] added DatabaseInitializer + Tests --- src/BirdsiteLive.sln | 7 + .../Services/FederationService.cs | 25 +- .../Contracts/IDbInitializerDal.cs | 4 +- .../BirdsiteLive.DAL/DatabaseInitializer.cs | 46 ++++ .../BirdsiteLive.DAL.Tests.csproj | 21 ++ .../DatabaseInitializerTests.cs | 240 ++++++++++++++++++ 6 files changed, 322 insertions(+), 21 deletions(-) create mode 100644 src/DataAccessLayers/BirdsiteLive.DAL/DatabaseInitializer.cs create mode 100644 src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj create mode 100644 src/Tests/BirdsiteLive.DAL.Tests/DatabaseInitializerTests.cs diff --git a/src/BirdsiteLive.sln b/src/BirdsiteLive.sln index bf78d55..0a35bf6 100644 --- a/src/BirdsiteLive.sln +++ b/src/BirdsiteLive.sln @@ -39,6 +39,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Domain.Tests", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Pipeline.Tests", "Tests\BirdsiteLive.Pipeline.Tests\BirdsiteLive.Pipeline.Tests.csproj", "{BF51CA81-5A7A-46F8-B4FB-861C6BE59298}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdsiteLive.DAL.Tests", "Tests\BirdsiteLive.DAL.Tests\BirdsiteLive.DAL.Tests.csproj", "{5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -101,6 +103,10 @@ Global {BF51CA81-5A7A-46F8-B4FB-861C6BE59298}.Debug|Any CPU.Build.0 = Debug|Any CPU {BF51CA81-5A7A-46F8-B4FB-861C6BE59298}.Release|Any CPU.ActiveCfg = Release|Any CPU {BF51CA81-5A7A-46F8-B4FB-861C6BE59298}.Release|Any CPU.Build.0 = Release|Any CPU + {5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -119,6 +125,7 @@ Global {2A8CC30D-D775-47D1-9388-F72A5C32DE2A} = {DA3C160C-4811-4E26-A5AD-42B81FAF2D7C} {F544D745-89A8-4DEA-B61C-A7E6C53C1D63} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94} {BF51CA81-5A7A-46F8-B4FB-861C6BE59298} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94} + {5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {69E8DCAD-4C37-4010-858F-5F94E6FBABCE} diff --git a/src/BirdsiteLive/Services/FederationService.cs b/src/BirdsiteLive/Services/FederationService.cs index f2c2e94..6835fd4 100644 --- a/src/BirdsiteLive/Services/FederationService.cs +++ b/src/BirdsiteLive/Services/FederationService.cs @@ -1,6 +1,8 @@ using System; +using System.Linq; using System.Threading; using System.Threading.Tasks; +using BirdsiteLive.DAL; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.Pipeline; using Microsoft.Extensions.Hosting; @@ -9,36 +11,21 @@ namespace BirdsiteLive.Services { public class FederationService : BackgroundService { - private readonly IDbInitializerDal _dbInitializerDal; + private readonly IDatabaseInitializer _databaseInitializer; private readonly IStatusPublicationPipeline _statusPublicationPipeline; #region Ctor - public FederationService(IDbInitializerDal dbInitializerDal, IStatusPublicationPipeline statusPublicationPipeline) + public FederationService(IDatabaseInitializer databaseInitializer, IStatusPublicationPipeline statusPublicationPipeline) { - _dbInitializerDal = dbInitializerDal; + _databaseInitializer = databaseInitializer; _statusPublicationPipeline = statusPublicationPipeline; } #endregion protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - await DbInitAsync(); + await _databaseInitializer.DbInitAsync(); await _statusPublicationPipeline.ExecuteAsync(stoppingToken); } - - private async Task DbInitAsync() - { - var currentVersion = await _dbInitializerDal.GetCurrentDbVersionAsync(); - var mandatoryVersion = _dbInitializerDal.GetMandatoryDbVersion(); - - if (currentVersion == null) - { - await _dbInitializerDal.InitDbAsync(); - } - else if (currentVersion != mandatoryVersion) - { - throw new NotImplementedException(); - } - } } } \ No newline at end of file diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IDbInitializerDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IDbInitializerDal.cs index b786386..9d7db56 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IDbInitializerDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IDbInitializerDal.cs @@ -9,7 +9,7 @@ namespace BirdsiteLive.DAL.Contracts Task GetCurrentDbVersionAsync(); Version GetMandatoryDbVersion(); Tuple[] GetMigrationPatterns(); - Task MigrateDbAsync(Version from, Version to); - Task InitDbAsync(); + Task MigrateDbAsync(Version from, Version to); + Task InitDbAsync(); } } \ No newline at end of file diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/DatabaseInitializer.cs b/src/DataAccessLayers/BirdsiteLive.DAL/DatabaseInitializer.cs new file mode 100644 index 0000000..4fe24df --- /dev/null +++ b/src/DataAccessLayers/BirdsiteLive.DAL/DatabaseInitializer.cs @@ -0,0 +1,46 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; + +namespace BirdsiteLive.DAL +{ + public interface IDatabaseInitializer + { + Task DbInitAsync(); + } + + public class DatabaseInitializer : IDatabaseInitializer + { + private readonly IDbInitializerDal _dbInitializerDal; + + #region Ctor + public DatabaseInitializer(IDbInitializerDal dbInitializerDal) + { + _dbInitializerDal = dbInitializerDal; + } + #endregion + + public async Task DbInitAsync() + { + var currentVersion = await _dbInitializerDal.GetCurrentDbVersionAsync(); + var mandatoryVersion = _dbInitializerDal.GetMandatoryDbVersion(); + + if (currentVersion == mandatoryVersion) return; + + // Init Db + var migrationPatterns = _dbInitializerDal.GetMigrationPatterns(); + if (currentVersion == null) + currentVersion = await _dbInitializerDal.InitDbAsync(); + + // Migrate Db + while (migrationPatterns.Any(x => x.Item1 == currentVersion)) + { + var migration = migrationPatterns.First(x => x.Item1 == currentVersion); + currentVersion = await _dbInitializerDal.MigrateDbAsync(migration.Item1, migration.Item2); + } + + if (currentVersion != mandatoryVersion) throw new Exception("Migrating DB failed"); + } + } +} \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj b/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj new file mode 100644 index 0000000..0992b02 --- /dev/null +++ b/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj @@ -0,0 +1,21 @@ + + + + netcoreapp3.1 + + false + + + + + + + + + + + + + + + diff --git a/src/Tests/BirdsiteLive.DAL.Tests/DatabaseInitializerTests.cs b/src/Tests/BirdsiteLive.DAL.Tests/DatabaseInitializerTests.cs new file mode 100644 index 0000000..bd847d2 --- /dev/null +++ b/src/Tests/BirdsiteLive.DAL.Tests/DatabaseInitializerTests.cs @@ -0,0 +1,240 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace BirdsiteLive.DAL.Tests +{ + [TestClass] + public class DatabaseInitializerTests + { + [TestMethod] + public async Task DbInitAsync_UpToDate_Test() + { + #region Stubs + var current = new Version(2, 3); + var mandatory = new Version(2, 3); + #endregion + + #region Mocks + var dbInitializerDal = new Mock(MockBehavior.Strict); + + dbInitializerDal + .Setup(x => x.GetCurrentDbVersionAsync()) + .ReturnsAsync(current); + + dbInitializerDal + .Setup(x => x.GetMandatoryDbVersion()) + .Returns(mandatory); + #endregion + + var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object); + await dbInitializer.DbInitAsync(); + + #region Validations + dbInitializerDal.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task DbInitAsync_NoDb_Test() + { + #region Stubs + var current = (Version)null; + var mandatory = new Version(1, 0); + + var migrationPatterns = new Tuple[0]; + #endregion + + #region Mocks + var dbInitializerDal = new Mock(MockBehavior.Strict); + + dbInitializerDal + .Setup(x => x.GetCurrentDbVersionAsync()) + .ReturnsAsync(current); + + dbInitializerDal + .Setup(x => x.GetMandatoryDbVersion()) + .Returns(mandatory); + + dbInitializerDal + .Setup(x => x.InitDbAsync()) + .ReturnsAsync(new Version(1, 0)); + + dbInitializerDal + .Setup(x => x.GetMigrationPatterns()) + .Returns(migrationPatterns); + #endregion + + var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object); + await dbInitializer.DbInitAsync(); + + #region Validations + dbInitializerDal.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task DbInitAsync_NoDb_Migration_Test() + { + #region Stubs + var current = (Version)null; + var mandatory = new Version(2, 3); + + var migrationPatterns = new Tuple[] + { + new Tuple(new Version(1,0), new Version(1,7)), + new Tuple(new Version(1,7), new Version(2,0)), + new Tuple(new Version(2,0), new Version(2,3)) + }; + #endregion + + #region Mocks + var dbInitializerDal = new Mock(MockBehavior.Strict); + + dbInitializerDal + .Setup(x => x.GetCurrentDbVersionAsync()) + .ReturnsAsync(current); + + dbInitializerDal + .Setup(x => x.GetMandatoryDbVersion()) + .Returns(mandatory); + + dbInitializerDal + .Setup(x => x.InitDbAsync()) + .ReturnsAsync(new Version(1, 0)); + + dbInitializerDal + .Setup(x => x.GetMigrationPatterns()) + .Returns(migrationPatterns); + + foreach (var m in migrationPatterns) + { + dbInitializerDal + .Setup(x => x.MigrateDbAsync( + It.Is(y => y == m.Item1), + It.Is(y => y == m.Item2) + )) + .ReturnsAsync(m.Item2); + } + #endregion + + var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object); + await dbInitializer.DbInitAsync(); + + #region Validations + dbInitializerDal.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task DbInitAsync_HasDb_Migration_Test() + { + #region Stubs + var current = new Version(1, 7); + var mandatory = new Version(2, 3); + + var migrationPatterns = new Tuple[] + { + new Tuple(new Version(1,0), new Version(1,7)), + new Tuple(new Version(1,7), new Version(2,0)), + new Tuple(new Version(2,0), new Version(2,3)) + }; + #endregion + + #region Mocks + var dbInitializerDal = new Mock(MockBehavior.Strict); + + dbInitializerDal + .Setup(x => x.GetCurrentDbVersionAsync()) + .ReturnsAsync(current); + + dbInitializerDal + .Setup(x => x.GetMandatoryDbVersion()) + .Returns(mandatory); + + dbInitializerDal + .Setup(x => x.GetMigrationPatterns()) + .Returns(migrationPatterns); + + foreach (var m in migrationPatterns.Skip(1)) + { + dbInitializerDal + .Setup(x => x.MigrateDbAsync( + It.Is(y => y == m.Item1), + It.Is(y => y == m.Item2) + )) + .ReturnsAsync(m.Item2); + } + #endregion + + var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object); + await dbInitializer.DbInitAsync(); + + #region Validations + dbInitializerDal.VerifyAll(); + #endregion + } + + [TestMethod] + [ExpectedException(typeof(Exception))] + public async Task DbInitAsync_NoDb_Migration_Error_Test() + { + #region Stubs + var current = (Version)null; + var mandatory = new Version(2, 3); + + var migrationPatterns = new Tuple[] + { + new Tuple(new Version(1,0), new Version(1,7)), + new Tuple(new Version(1,7), new Version(2,0)), + new Tuple(new Version(2,0), new Version(2,2)) + }; + #endregion + + #region Mocks + var dbInitializerDal = new Mock(MockBehavior.Strict); + + dbInitializerDal + .Setup(x => x.GetCurrentDbVersionAsync()) + .ReturnsAsync(current); + + dbInitializerDal + .Setup(x => x.GetMandatoryDbVersion()) + .Returns(mandatory); + + dbInitializerDal + .Setup(x => x.InitDbAsync()) + .ReturnsAsync(new Version(1, 0)); + + dbInitializerDal + .Setup(x => x.GetMigrationPatterns()) + .Returns(migrationPatterns); + + foreach (var m in migrationPatterns) + { + dbInitializerDal + .Setup(x => x.MigrateDbAsync( + It.Is(y => y == m.Item1), + It.Is(y => y == m.Item2) + )) + .ReturnsAsync(m.Item2); + } + #endregion + + var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object); + try + { + await dbInitializer.DbInitAsync(); + } + finally + { + #region Validations + dbInitializerDal.VerifyAll(); + #endregion + } + } + } +} From 8146b6ea5a0d12874fdba93c7c4dc9354f186cd9 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 22 Jan 2021 20:18:42 -0500 Subject: [PATCH 03/12] added LastSync to TwitterUser (wip) --- .../DbInitializerPostgresDal.cs | 41 ++++++++++++------- .../Models/SyncTwitterUser.cs | 6 ++- .../DbInitializerPostgresDalTests.cs | 2 +- 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs index c6d5c81..a50e05b 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs @@ -23,7 +23,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public class DbInitializerPostgresDal : PostgresBase, IDbInitializerDal { private readonly PostgresTools _tools; - private readonly Version _currentVersion = new Version(1,0); + private readonly Version _currentVersion = new Version(2, 0); private const string DbVersionType = "db-version"; #region Ctor @@ -32,7 +32,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers _tools = tools; } #endregion - + public async Task GetCurrentDbVersionAsync() { var query = $"SELECT * FROM {_settings.DbVersionTableName} WHERE type = @type"; @@ -65,17 +65,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers return _currentVersion; } - public Tuple[] GetMigrationPatterns() - { - return new Tuple[0]; - } - - public Task MigrateDbAsync(Version from, Version to) - { - throw new NotImplementedException(); - } - - public async Task InitDbAsync() + public async Task InitDbAsync() { // Create version table var createVersion = $@"CREATE TABLE {_settings.DbVersionTableName} @@ -124,14 +114,37 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers await _tools.ExecuteRequestAsync(createCachedTweets); // Insert version to db + var firstVersion = new Version(1, 0); using (var dbConnection = Connection) { dbConnection.Open(); await dbConnection.ExecuteAsync( $"INSERT INTO {_settings.DbVersionTableName} (type,major,minor) VALUES(@type,@major,@minor)", - new { type = DbVersionType, major = _currentVersion.Major, minor = _currentVersion.Minor }); + new { type = DbVersionType, major = firstVersion.Major, minor = firstVersion.Minor }); } + + return firstVersion; + } + + public Tuple[] GetMigrationPatterns() + { + return new[] + { + new Tuple(new Version(1,0), new Version(2,0)) + }; + } + + public async Task MigrateDbAsync(Version from, Version to) + { + if (from == new Version(1, 0) && to == new Version(2, 0)) + { + var addLastSync = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD lastSync TIMESTAMP (2) WITHOUT TIME ZONE"; + await _tools.ExecuteRequestAsync(addLastSync); + return new Version(2, 0); + } + + throw new NotImplementedException(); } public async Task DeleteAllAsync() diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs index 8061fc8..59be0a5 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs @@ -1,4 +1,6 @@ -namespace BirdsiteLive.DAL.Models +using System; + +namespace BirdsiteLive.DAL.Models { public class SyncTwitterUser { @@ -7,5 +9,7 @@ public long LastTweetPostedId { get; set; } public long LastTweetSynchronizedForAllFollowersId { get; set; } + + public DateTime LastSync { get; set; } } } \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/DbInitializerPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/DbInitializerPostgresDalTests.cs index 7fc5383..9ede041 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/DbInitializerPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/DbInitializerPostgresDalTests.cs @@ -35,11 +35,11 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers [TestMethod] public async Task InitDbAsync() { + var mandatory = new Version(1, 0); var dal = new DbInitializerPostgresDal(_settings, _tools); await dal.InitDbAsync(); var current = await dal.GetCurrentDbVersionAsync(); - var mandatory = dal.GetMandatoryDbVersion(); Assert.IsNotNull(current); Assert.AreEqual(mandatory.Minor, current.Minor); Assert.AreEqual(mandatory.Major, current.Major); From b4c01ad326cd4ecedd5f35324b086e0b31d770c7 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 22 Jan 2021 21:05:01 -0500 Subject: [PATCH 04/12] save last sync --- src/BirdsiteLive/Services/FederationService.cs | 2 +- .../DataAccessLayers/TwitterUserPostgresDal.cs | 9 +++++---- .../BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs | 5 +++-- .../BirdsiteLive.DAL/DatabaseInitializer.cs | 4 ++-- .../DataAccessLayers/CachedTweetsPostgresDalTests.cs | 3 ++- .../DataAccessLayers/FollowersPostgresDalTests.cs | 3 ++- .../DataAccessLayers/TwitterUserPostgresDalTests.cs | 7 +++++-- .../BirdsiteLive.DAL.Tests/DatabaseInitializerTests.cs | 10 +++++----- 8 files changed, 25 insertions(+), 18 deletions(-) diff --git a/src/BirdsiteLive/Services/FederationService.cs b/src/BirdsiteLive/Services/FederationService.cs index 6835fd4..9acab41 100644 --- a/src/BirdsiteLive/Services/FederationService.cs +++ b/src/BirdsiteLive/Services/FederationService.cs @@ -24,7 +24,7 @@ namespace BirdsiteLive.Services protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - await _databaseInitializer.DbInitAsync(); + await _databaseInitializer.InitAndMigrateDbAsync(); await _statusPublicationPipeline.ExecuteAsync(stoppingToken); } } diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs index 082229a..0cb811f 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs @@ -75,19 +75,20 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } } - public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId) + public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, DateTime lastSync) { if(id == default) throw new ArgumentException("id"); if(lastTweetPostedId == default) throw new ArgumentException("lastTweetPostedId"); if(lastTweetSynchronizedForAllFollowersId == default) throw new ArgumentException("lastTweetSynchronizedForAllFollowersId"); - - var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId WHERE id = @id"; + if(lastSync == default) throw new ArgumentException("lastSync"); + + var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId, lastSync = @lastSync WHERE id = @id"; using (var dbConnection = Connection) { dbConnection.Open(); - await dbConnection.QueryAsync(query, new { id, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId }); + await dbConnection.QueryAsync(query, new { id, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId, lastSync = lastSync.ToUniversalTime() }); } } diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs index 48d5661..65d9697 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs @@ -1,4 +1,5 @@ -using System.Threading.Tasks; +using System; +using System.Threading.Tasks; using BirdsiteLive.DAL.Models; namespace BirdsiteLive.DAL.Contracts @@ -8,7 +9,7 @@ namespace BirdsiteLive.DAL.Contracts Task CreateTwitterUserAsync(string acct, long lastTweetPostedId); Task GetTwitterUserAsync(string acct); Task GetAllTwitterUsersAsync(); - Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId); + Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, DateTime lastSync); Task DeleteTwitterUserAsync(string acct); Task GetTwitterUsersCountAsync(); } diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/DatabaseInitializer.cs b/src/DataAccessLayers/BirdsiteLive.DAL/DatabaseInitializer.cs index 4fe24df..2a5f60d 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/DatabaseInitializer.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/DatabaseInitializer.cs @@ -7,7 +7,7 @@ namespace BirdsiteLive.DAL { public interface IDatabaseInitializer { - Task DbInitAsync(); + Task InitAndMigrateDbAsync(); } public class DatabaseInitializer : IDatabaseInitializer @@ -21,7 +21,7 @@ namespace BirdsiteLive.DAL } #endregion - public async Task DbInitAsync() + public async Task InitAndMigrateDbAsync() { var currentVersion = await _dbInitializerDal.GetCurrentDbVersionAsync(); var mandatoryVersion = _dbInitializerDal.GetMandatoryDbVersion(); diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/CachedTweetsPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/CachedTweetsPostgresDalTests.cs index 021784e..f173685 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/CachedTweetsPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/CachedTweetsPostgresDalTests.cs @@ -14,7 +14,8 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task TestInit() { var dal = new DbInitializerPostgresDal(_settings, _tools); - await dal.InitDbAsync(); + var init = new DatabaseInitializer(dal); + await init.InitAndMigrateDbAsync(); } [TestCleanup] diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs index cd6162d..b2289c4 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs @@ -16,7 +16,8 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task TestInit() { var dal = new DbInitializerPostgresDal(_settings, _tools); - await dal.InitDbAsync(); + var init = new DatabaseInitializer(dal); + await init.InitAndMigrateDbAsync(); } [TestCleanup] diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs index f900c8f..5fa600c 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs @@ -14,7 +14,8 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task TestInit() { var dal = new DbInitializerPostgresDal(_settings, _tools); - await dal.InitDbAsync(); + var init = new DatabaseInitializer(dal); + await init.InitAndMigrateDbAsync(); } [TestCleanup] @@ -70,13 +71,15 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers var updatedLastTweetId = 1600L; var updatedLastSyncId = 1550L; - await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId); + var now = DateTime.Now; + await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, now); result = await dal.GetTwitterUserAsync(acct); Assert.AreEqual(acct, result.Acct); Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId); Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId); + Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100); } [TestMethod] diff --git a/src/Tests/BirdsiteLive.DAL.Tests/DatabaseInitializerTests.cs b/src/Tests/BirdsiteLive.DAL.Tests/DatabaseInitializerTests.cs index bd847d2..ba11321 100644 --- a/src/Tests/BirdsiteLive.DAL.Tests/DatabaseInitializerTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Tests/DatabaseInitializerTests.cs @@ -31,7 +31,7 @@ namespace BirdsiteLive.DAL.Tests #endregion var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object); - await dbInitializer.DbInitAsync(); + await dbInitializer.InitAndMigrateDbAsync(); #region Validations dbInitializerDal.VerifyAll(); @@ -69,7 +69,7 @@ namespace BirdsiteLive.DAL.Tests #endregion var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object); - await dbInitializer.DbInitAsync(); + await dbInitializer.InitAndMigrateDbAsync(); #region Validations dbInitializerDal.VerifyAll(); @@ -122,7 +122,7 @@ namespace BirdsiteLive.DAL.Tests #endregion var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object); - await dbInitializer.DbInitAsync(); + await dbInitializer.InitAndMigrateDbAsync(); #region Validations dbInitializerDal.VerifyAll(); @@ -171,7 +171,7 @@ namespace BirdsiteLive.DAL.Tests #endregion var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object); - await dbInitializer.DbInitAsync(); + await dbInitializer.InitAndMigrateDbAsync(); #region Validations dbInitializerDal.VerifyAll(); @@ -227,7 +227,7 @@ namespace BirdsiteLive.DAL.Tests var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object); try { - await dbInitializer.DbInitAsync(); + await dbInitializer.InitAndMigrateDbAsync(); } finally { From 89c041f332c4492980296847cdf861150fe47514 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 22 Jan 2021 21:23:27 -0500 Subject: [PATCH 05/12] added limit to user retrieval --- .../Processors/RetrieveTweetsProcessor.cs | 6 +- .../RetrieveTwitterUsersProcessor.cs | 13 ++++- .../Processors/SaveProgressionProcessor.cs | 6 +- .../TwitterUserPostgresDal.cs | 6 +- .../Contracts/ITwitterUserDal.cs | 2 +- .../TwitterUserPostgresDalTests.cs | 2 +- .../RetrieveTweetsProcessorTests.cs | 4 +- .../RetrieveTwitterUsersProcessorTests.cs | 57 +++++++++++++++---- .../SaveProgressionProcessorTests.cs | 12 ++-- 9 files changed, 81 insertions(+), 27 deletions(-) diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs index ef20cad..ffcf9a9 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -45,7 +46,8 @@ namespace BirdsiteLive.Pipeline.Processors else if (tweets.Length > 0 && user.LastTweetPostedId == -1) { var tweetId = tweets.Last().Id; - await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId); + var now = DateTime.UtcNow; + await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, now); } } diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs index f556831..847e23c 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs @@ -4,6 +4,7 @@ using System.Threading; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; using BirdsiteLive.Common.Extensions; +using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Contracts; @@ -15,12 +16,16 @@ namespace BirdsiteLive.Pipeline.Processors { private readonly ITwitterUserDal _twitterUserDal; private readonly ILogger _logger; + private readonly InstanceSettings _instanceSettings; + public int WaitFactor = 1000 * 60; //1 min + private int StartUpWarming = 4; #region Ctor - public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, ILogger logger) + public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, InstanceSettings instanceSettings, ILogger logger) { _twitterUserDal = twitterUserDal; + _instanceSettings = instanceSettings; _logger = logger; } #endregion @@ -33,7 +38,11 @@ namespace BirdsiteLive.Pipeline.Processors try { - var users = await _twitterUserDal.GetAllTwitterUsersAsync(); + var maxUsers = StartUpWarming > 0 + ? _instanceSettings.MaxUsersCapacity / 4 + : _instanceSettings.MaxUsersCapacity; + StartUpWarming--; + var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsers); var userCount = users.Any() ? users.Length : 1; var splitNumber = (int) Math.Ceiling(userCount / 15d); diff --git a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs index 5b305e7..c7cbc36 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs @@ -1,4 +1,5 @@ -using System.Linq; +using System; +using System.Linq; using System.Threading; using System.Threading.Tasks; using BirdsiteLive.DAL.Contracts; @@ -23,7 +24,8 @@ namespace BirdsiteLive.Pipeline.Processors var userId = userWithTweetsToSync.User.Id; var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max(); var minimumSync = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).Min(); - await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync); + var now = DateTime.UtcNow; + await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, now); } } } \ No newline at end of file diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs index 0cb811f..afbf7d1 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs @@ -62,15 +62,15 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } } - public async Task GetAllTwitterUsersAsync() + public async Task GetAllTwitterUsersAsync(int maxNumber) { - var query = $"SELECT * FROM {_settings.TwitterUserTableName}"; + var query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC LIMIT @maxNumber"; using (var dbConnection = Connection) { dbConnection.Open(); - var result = await dbConnection.QueryAsync(query); + var result = await dbConnection.QueryAsync(query, new { maxNumber }); return result.ToArray(); } } diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs index 65d9697..1fa8127 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs @@ -8,7 +8,7 @@ namespace BirdsiteLive.DAL.Contracts { Task CreateTwitterUserAsync(string acct, long lastTweetPostedId); Task GetTwitterUserAsync(string acct); - Task GetAllTwitterUsersAsync(); + Task GetAllTwitterUsersAsync(int maxNumber); Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, DateTime lastSync); Task DeleteTwitterUserAsync(string acct); Task GetTwitterUsersCountAsync(); diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs index 5fa600c..6a1e96b 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs @@ -111,7 +111,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers await dal.CreateTwitterUserAsync(acct, lastTweetId); } - var result = await dal.GetAllTwitterUsersAsync(); + var result = await dal.GetAllTwitterUsersAsync(1000); Assert.AreEqual(1000, result.Length); Assert.IsFalse(result[0].Id == default); Assert.IsFalse(result[0].Acct == default); diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs index 2bf5d74..d66c2f7 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs @@ -1,3 +1,4 @@ +using System; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -54,7 +55,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Setup(x => x.UpdateTwitterUserAsync( It.Is(y => y == user1.Id), It.Is(y => y == tweets.Last().Id), - It.Is(y => y == tweets.Last().Id) + It.Is(y => y == tweets.Last().Id), + It.IsAny() )) .Returns(Task.CompletedTask); #endregion diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs index 12c5682..6323889 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; +using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Processors; @@ -26,18 +27,23 @@ namespace BirdsiteLive.Pipeline.Tests.Processors new SyncTwitterUser(), new SyncTwitterUser(), }; + var settings = new InstanceSettings + { + MaxUsersCapacity = 10 + }; #endregion #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync( + It.Is(y => y == settings.MaxUsersCapacity))) .ReturnsAsync(users); var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object); processor.WaitFactor = 10; processor.GetTwitterUsersAsync(buffer, CancellationToken.None); @@ -60,19 +66,25 @@ namespace BirdsiteLive.Pipeline.Tests.Processors for (var i = 0; i < 30; i++) users.Add(new SyncTwitterUser()); + + var settings = new InstanceSettings + { + MaxUsersCapacity = 100 + }; #endregion #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .SetupSequence(x => x.GetAllTwitterUsersAsync()) + .SetupSequence(x => x.GetAllTwitterUsersAsync( + It.Is(y => y == settings.MaxUsersCapacity))) .ReturnsAsync(users.ToArray()) .ReturnsAsync(new SyncTwitterUser[0]); var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object); processor.WaitFactor = 2; processor.GetTwitterUsersAsync(buffer, CancellationToken.None); @@ -95,19 +107,25 @@ namespace BirdsiteLive.Pipeline.Tests.Processors for (var i = 0; i < 31; i++) users.Add(new SyncTwitterUser()); + + var settings = new InstanceSettings + { + MaxUsersCapacity = 10 + }; #endregion #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .SetupSequence(x => x.GetAllTwitterUsersAsync()) + .SetupSequence(x => x.GetAllTwitterUsersAsync( + It.Is(y => y == settings.MaxUsersCapacity))) .ReturnsAsync(users.ToArray()) .ReturnsAsync(new SyncTwitterUser[0]); var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object); processor.WaitFactor = 2; processor.GetTwitterUsersAsync(buffer, CancellationToken.None); @@ -126,18 +144,24 @@ namespace BirdsiteLive.Pipeline.Tests.Processors { #region Stubs var buffer = new BufferBlock(); + + var settings = new InstanceSettings + { + MaxUsersCapacity = 10 + }; #endregion #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync( + It.Is(y => y == settings.MaxUsersCapacity))) .ReturnsAsync(new SyncTwitterUser[0]); var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object); processor.WaitFactor = 1; processor.GetTwitterUsersAsync(buffer, CancellationToken.None); @@ -154,18 +178,24 @@ namespace BirdsiteLive.Pipeline.Tests.Processors { #region Stubs var buffer = new BufferBlock(); + + var settings = new InstanceSettings + { + MaxUsersCapacity = 10 + }; #endregion #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync( + It.Is(y => y == settings.MaxUsersCapacity))) .Returns(async () => await DelayFaultedTask(new Exception())); var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object); processor.WaitFactor = 10; var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None); @@ -185,6 +215,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var buffer = new BufferBlock(); var canTokenS = new CancellationTokenSource(); canTokenS.Cancel(); + + var settings = new InstanceSettings + { + MaxUsersCapacity = 10 + }; #endregion #region Mocks @@ -192,7 +227,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object); processor.WaitFactor = 1; await processor.GetTwitterUsersAsync(buffer, canTokenS.Token); } diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs index d3880e6..b2a99b9 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using BirdsiteLive.DAL.Contracts; @@ -60,7 +61,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Setup(x => x.UpdateTwitterUserAsync( It.Is(y => y == user.Id), It.Is(y => y == tweet2.Id), - It.Is(y => y == tweet2.Id) + It.Is(y => y == tweet2.Id), + It.IsAny() )) .Returns(Task.CompletedTask); #endregion @@ -123,7 +125,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Setup(x => x.UpdateTwitterUserAsync( It.Is(y => y == user.Id), It.Is(y => y == tweet3.Id), - It.Is(y => y == tweet2.Id) + It.Is(y => y == tweet2.Id), + It.IsAny() )) .Returns(Task.CompletedTask); #endregion @@ -194,7 +197,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Setup(x => x.UpdateTwitterUserAsync( It.Is(y => y == user.Id), It.Is(y => y == tweet3.Id), - It.Is(y => y == tweet2.Id) + It.Is(y => y == tweet2.Id), + It.IsAny() )) .Returns(Task.CompletedTask); #endregion From 3e7a136902417b1f5981ff39f4fdd144b5c42721 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 22 Jan 2021 23:07:25 -0500 Subject: [PATCH 06/12] added test coverage and warmup strategy --- .../RetrieveTwitterUsersProcessor.cs | 9 +++-- .../TwitterUserPostgresDalTests.cs | 35 +++++++++++++++++++ .../RetrieveTwitterUsersProcessorTests.cs | 35 ++++++++++++++++--- 3 files changed, 71 insertions(+), 8 deletions(-) diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs index 847e23c..eba1868 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs @@ -19,7 +19,6 @@ namespace BirdsiteLive.Pipeline.Processors private readonly InstanceSettings _instanceSettings; public int WaitFactor = 1000 * 60; //1 min - private int StartUpWarming = 4; #region Ctor public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, InstanceSettings instanceSettings, ILogger logger) @@ -32,16 +31,20 @@ namespace BirdsiteLive.Pipeline.Processors public async Task GetTwitterUsersAsync(BufferBlock twitterUsersBufferBlock, CancellationToken ct) { + var totalUsers = await _twitterUserDal.GetTwitterUsersCountAsync(); + var warmUpMaxCapacity = _instanceSettings.MaxUsersCapacity / 4; + var warmUpIterations = warmUpMaxCapacity == 0 ? 0 : (int) (totalUsers / (float) warmUpMaxCapacity); + for (; ; ) { ct.ThrowIfCancellationRequested(); try { - var maxUsers = StartUpWarming > 0 + var maxUsers = warmUpIterations > 0 ? _instanceSettings.MaxUsersCapacity / 4 : _instanceSettings.MaxUsersCapacity; - StartUpWarming--; + warmUpIterations--; var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsers); var userCount = users.Any() ? users.Length : 1; diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs index 6a1e96b..12e6f7f 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs @@ -119,6 +119,41 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default); } + [TestMethod] + public async Task GetAllTwitterUsers_Limited() + { + var now = DateTime.Now; + var oldest = now.AddDays(-3); + var newest = now.AddDays(-2); + + var dal = new TwitterUserPostgresDal(_settings); + for (var i = 0; i < 20; i++) + { + var acct = $"myid{i}"; + var lastTweetId = 1548L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + } + + var allUsers = await dal.GetAllTwitterUsersAsync(100); + for (var i = 0; i < 20; i++) + { + var user = allUsers[i]; + var date = i % 2 == 0 ? oldest : newest; + await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, date); + } + + var result = await dal.GetAllTwitterUsersAsync(10); + Assert.AreEqual(10, result.Length); + Assert.IsFalse(result[0].Id == default); + Assert.IsFalse(result[0].Acct == default); + Assert.IsFalse(result[0].LastTweetPostedId == default); + Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default); + + foreach (var acc in result) + Assert.IsTrue(Math.Abs((acc.LastSync - oldest.ToUniversalTime()).TotalMilliseconds) < 1000); + } + [TestMethod] public async Task CountTwitterUsers() { diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs index 6323889..ef33e78 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs @@ -29,7 +29,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors }; var settings = new InstanceSettings { - MaxUsersCapacity = 10 + MaxUsersCapacity = 40 }; #endregion @@ -37,9 +37,13 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == settings.MaxUsersCapacity))) + It.Is(y => y == settings.MaxUsersCapacity/4))) .ReturnsAsync(users); + twitterUserDalMock + .Setup(x => x.GetTwitterUsersCountAsync()) + .ReturnsAsync(10); + var loggerMock = new Mock>(); #endregion @@ -69,7 +73,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var settings = new InstanceSettings { - MaxUsersCapacity = 100 + MaxUsersCapacity = 400 }; #endregion @@ -81,6 +85,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .ReturnsAsync(users.ToArray()) .ReturnsAsync(new SyncTwitterUser[0]); + twitterUserDalMock + .Setup(x => x.GetTwitterUsersCountAsync()) + .ReturnsAsync(30); + var loggerMock = new Mock>(); #endregion @@ -110,7 +118,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var settings = new InstanceSettings { - MaxUsersCapacity = 10 + MaxUsersCapacity = 400 }; #endregion @@ -118,10 +126,14 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .SetupSequence(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == settings.MaxUsersCapacity))) + It.Is(y => y == settings.MaxUsersCapacity/4))) .ReturnsAsync(users.ToArray()) .ReturnsAsync(new SyncTwitterUser[0]); + twitterUserDalMock + .Setup(x => x.GetTwitterUsersCountAsync()) + .ReturnsAsync(31); + var loggerMock = new Mock>(); #endregion @@ -158,6 +170,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == settings.MaxUsersCapacity))) .ReturnsAsync(new SyncTwitterUser[0]); + twitterUserDalMock + .Setup(x => x.GetTwitterUsersCountAsync()) + .ReturnsAsync(1000); + var loggerMock = new Mock>(); #endregion @@ -192,6 +208,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == settings.MaxUsersCapacity))) .Returns(async () => await DelayFaultedTask(new Exception())); + twitterUserDalMock + .Setup(x => x.GetTwitterUsersCountAsync()) + .ReturnsAsync(1000); + var loggerMock = new Mock>(); #endregion @@ -224,6 +244,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); + + twitterUserDalMock + .Setup(x => x.GetTwitterUsersCountAsync()) + .ReturnsAsync(1000); + var loggerMock = new Mock>(); #endregion From 34cb33160a306893b63f68055135c721be41cd6d Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 22 Jan 2021 23:38:35 -0500 Subject: [PATCH 07/12] refactoring warm up logic and tests --- .../BirdsiteLive.Pipeline.csproj | 4 + .../RetrieveTwitterUsersProcessor.cs | 18 +-- .../Tools/MaxUsersNumberProvider.cs | 49 +++++++ .../BirdsiteLive.Pipeline.Tests.csproj | 4 + .../RetrieveTwitterUsersProcessorTests.cs | 124 +++++++++--------- .../Tools/MaxUsersNumberProviderTests.cs | 79 +++++++++++ 6 files changed, 204 insertions(+), 74 deletions(-) create mode 100644 src/BirdsiteLive.Pipeline/Tools/MaxUsersNumberProvider.cs create mode 100644 src/Tests/BirdsiteLive.Pipeline.Tests/Tools/MaxUsersNumberProviderTests.cs diff --git a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj index 5d93cb1..884af18 100644 --- a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj +++ b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj @@ -17,4 +17,8 @@ + + + + diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs index eba1868..ebb87fc 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs @@ -8,6 +8,7 @@ using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Tools; using Microsoft.Extensions.Logging; namespace BirdsiteLive.Pipeline.Processors @@ -15,37 +16,30 @@ namespace BirdsiteLive.Pipeline.Processors public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor { private readonly ITwitterUserDal _twitterUserDal; + private readonly IMaxUsersNumberProvider _maxUsersNumberProvider; private readonly ILogger _logger; - private readonly InstanceSettings _instanceSettings; public int WaitFactor = 1000 * 60; //1 min #region Ctor - public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, InstanceSettings instanceSettings, ILogger logger) + public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IMaxUsersNumberProvider maxUsersNumberProvider, ILogger logger) { _twitterUserDal = twitterUserDal; - _instanceSettings = instanceSettings; + _maxUsersNumberProvider = maxUsersNumberProvider; _logger = logger; } #endregion public async Task GetTwitterUsersAsync(BufferBlock twitterUsersBufferBlock, CancellationToken ct) { - var totalUsers = await _twitterUserDal.GetTwitterUsersCountAsync(); - var warmUpMaxCapacity = _instanceSettings.MaxUsersCapacity / 4; - var warmUpIterations = warmUpMaxCapacity == 0 ? 0 : (int) (totalUsers / (float) warmUpMaxCapacity); - for (; ; ) { ct.ThrowIfCancellationRequested(); try { - var maxUsers = warmUpIterations > 0 - ? _instanceSettings.MaxUsersCapacity / 4 - : _instanceSettings.MaxUsersCapacity; - warmUpIterations--; - var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsers); + var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync(); + var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber); var userCount = users.Any() ? users.Length : 1; var splitNumber = (int) Math.Ceiling(userCount / 15d); diff --git a/src/BirdsiteLive.Pipeline/Tools/MaxUsersNumberProvider.cs b/src/BirdsiteLive.Pipeline/Tools/MaxUsersNumberProvider.cs new file mode 100644 index 0000000..c84b7b1 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Tools/MaxUsersNumberProvider.cs @@ -0,0 +1,49 @@ +using System.Threading.Tasks; +using BirdsiteLive.Common.Settings; +using BirdsiteLive.DAL.Contracts; + +namespace BirdsiteLive.Pipeline.Tools +{ + public interface IMaxUsersNumberProvider + { + Task GetMaxUsersNumberAsync(); + } + + public class MaxUsersNumberProvider : IMaxUsersNumberProvider + { + private readonly InstanceSettings _instanceSettings; + private readonly ITwitterUserDal _twitterUserDal; + + private int _totalUsersCount = -1; + private int _warmUpIterations; + + #region Ctor + public MaxUsersNumberProvider(InstanceSettings instanceSettings, ITwitterUserDal twitterUserDal) + { + _instanceSettings = instanceSettings; + _twitterUserDal = twitterUserDal; + } + #endregion + + public async Task GetMaxUsersNumberAsync() + { + // Init data + if (_totalUsersCount == -1) + { + _totalUsersCount = await _twitterUserDal.GetTwitterUsersCountAsync(); + var warmUpMaxCapacity = _instanceSettings.MaxUsersCapacity / 4; + _warmUpIterations = warmUpMaxCapacity == 0 ? 0 : (int)(_totalUsersCount / (float)warmUpMaxCapacity); + } + + // Return if warm up ended + if (_warmUpIterations <= 0) return _instanceSettings.MaxUsersCapacity; + + // Calculate warm up value + var maxUsers = _warmUpIterations > 0 + ? _instanceSettings.MaxUsersCapacity / 4 + : _instanceSettings.MaxUsersCapacity; + _warmUpIterations--; + return maxUsers; + } + } +} \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj b/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj index 3dd6984..aa7750b 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj @@ -18,4 +18,8 @@ + + + + diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs index ef33e78..5600e1c 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs @@ -7,6 +7,7 @@ using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Processors; +using BirdsiteLive.Pipeline.Tools; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -27,33 +28,32 @@ namespace BirdsiteLive.Pipeline.Tests.Processors new SyncTwitterUser(), new SyncTwitterUser(), }; - var settings = new InstanceSettings - { - MaxUsersCapacity = 40 - }; + var maxUsers = 1000; #endregion #region Mocks + var maxUsersNumberProviderMock = new Mock(MockBehavior.Strict); + maxUsersNumberProviderMock + .Setup(x => x.GetMaxUsersNumberAsync()) + .ReturnsAsync(maxUsers); + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == settings.MaxUsersCapacity/4))) + It.Is(y => y == maxUsers))) .ReturnsAsync(users); - - twitterUserDalMock - .Setup(x => x.GetTwitterUsersCountAsync()) - .ReturnsAsync(10); - + var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 10; processor.GetTwitterUsersAsync(buffer, CancellationToken.None); await Task.Delay(50); #region Validations + maxUsersNumberProviderMock.VerifyAll(); twitterUserDalMock.VerifyAll(); Assert.AreEqual(3, buffer.Count); buffer.TryReceive(out var result); @@ -71,34 +71,36 @@ namespace BirdsiteLive.Pipeline.Tests.Processors for (var i = 0; i < 30; i++) users.Add(new SyncTwitterUser()); - var settings = new InstanceSettings - { - MaxUsersCapacity = 400 - }; + var maxUsers = 1000; #endregion #region Mocks + var maxUsersNumberProviderMock = new Mock(MockBehavior.Strict); + maxUsersNumberProviderMock + .Setup(x => x.GetMaxUsersNumberAsync()) + .ReturnsAsync(maxUsers); + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .SetupSequence(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == settings.MaxUsersCapacity))) + It.Is(y => y == maxUsers))) .ReturnsAsync(users.ToArray()) + .ReturnsAsync(new SyncTwitterUser[0]) + .ReturnsAsync(new SyncTwitterUser[0]) + .ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0]); - twitterUserDalMock - .Setup(x => x.GetTwitterUsersCountAsync()) - .ReturnsAsync(30); - var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 2; processor.GetTwitterUsersAsync(buffer, CancellationToken.None); await Task.Delay(300); #region Validations + maxUsersNumberProviderMock.VerifyAll(); twitterUserDalMock.VerifyAll(); Assert.AreEqual(15, buffer.Count); buffer.TryReceive(out var result); @@ -116,34 +118,36 @@ namespace BirdsiteLive.Pipeline.Tests.Processors for (var i = 0; i < 31; i++) users.Add(new SyncTwitterUser()); - var settings = new InstanceSettings - { - MaxUsersCapacity = 400 - }; + var maxUsers = 1000; #endregion #region Mocks + var maxUsersNumberProviderMock = new Mock(MockBehavior.Strict); + maxUsersNumberProviderMock + .Setup(x => x.GetMaxUsersNumberAsync()) + .ReturnsAsync(maxUsers); + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .SetupSequence(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == settings.MaxUsersCapacity/4))) + It.Is(y => y == maxUsers))) .ReturnsAsync(users.ToArray()) + .ReturnsAsync(new SyncTwitterUser[0]) + .ReturnsAsync(new SyncTwitterUser[0]) + .ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0]); - - twitterUserDalMock - .Setup(x => x.GetTwitterUsersCountAsync()) - .ReturnsAsync(31); - + var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 2; processor.GetTwitterUsersAsync(buffer, CancellationToken.None); await Task.Delay(200); #region Validations + maxUsersNumberProviderMock.VerifyAll(); twitterUserDalMock.VerifyAll(); Assert.AreEqual(11, buffer.Count); buffer.TryReceive(out var result); @@ -157,33 +161,32 @@ namespace BirdsiteLive.Pipeline.Tests.Processors #region Stubs var buffer = new BufferBlock(); - var settings = new InstanceSettings - { - MaxUsersCapacity = 10 - }; + var maxUsers = 1000; #endregion #region Mocks + var maxUsersNumberProviderMock = new Mock(MockBehavior.Strict); + maxUsersNumberProviderMock + .Setup(x => x.GetMaxUsersNumberAsync()) + .ReturnsAsync(maxUsers); + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == settings.MaxUsersCapacity))) + It.Is(y => y == maxUsers))) .ReturnsAsync(new SyncTwitterUser[0]); - twitterUserDalMock - .Setup(x => x.GetTwitterUsersCountAsync()) - .ReturnsAsync(1000); - var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 1; processor.GetTwitterUsersAsync(buffer, CancellationToken.None); await Task.Delay(50); #region Validations + maxUsersNumberProviderMock.VerifyAll(); twitterUserDalMock.VerifyAll(); Assert.AreEqual(0, buffer.Count); #endregion @@ -195,33 +198,32 @@ namespace BirdsiteLive.Pipeline.Tests.Processors #region Stubs var buffer = new BufferBlock(); - var settings = new InstanceSettings - { - MaxUsersCapacity = 10 - }; + var maxUsers = 1000; #endregion #region Mocks + var maxUsersNumberProviderMock = new Mock(MockBehavior.Strict); + maxUsersNumberProviderMock + .Setup(x => x.GetMaxUsersNumberAsync()) + .ReturnsAsync(maxUsers); + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == settings.MaxUsersCapacity))) + It.Is(y => y == maxUsers))) .Returns(async () => await DelayFaultedTask(new Exception())); - twitterUserDalMock - .Setup(x => x.GetTwitterUsersCountAsync()) - .ReturnsAsync(1000); - var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 10; var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None); await Task.WhenAny(t, Task.Delay(50)); #region Validations + maxUsersNumberProviderMock.VerifyAll(); twitterUserDalMock.VerifyAll(); Assert.AreEqual(0, buffer.Count); #endregion @@ -236,23 +238,21 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var canTokenS = new CancellationTokenSource(); canTokenS.Cancel(); - var settings = new InstanceSettings - { - MaxUsersCapacity = 10 - }; + var maxUsers = 1000; #endregion #region Mocks + var maxUsersNumberProviderMock = new Mock(MockBehavior.Strict); + maxUsersNumberProviderMock + .Setup(x => x.GetMaxUsersNumberAsync()) + .ReturnsAsync(maxUsers); + var twitterUserDalMock = new Mock(MockBehavior.Strict); - - twitterUserDalMock - .Setup(x => x.GetTwitterUsersCountAsync()) - .ReturnsAsync(1000); - + var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 1; await processor.GetTwitterUsersAsync(buffer, canTokenS.Token); } diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Tools/MaxUsersNumberProviderTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Tools/MaxUsersNumberProviderTests.cs new file mode 100644 index 0000000..d48beb8 --- /dev/null +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Tools/MaxUsersNumberProviderTests.cs @@ -0,0 +1,79 @@ +using System.Threading.Tasks; +using BirdsiteLive.Common.Settings; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.Pipeline.Tools; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace BirdsiteLive.Pipeline.Tests.Tools +{ + [TestClass] + public class MaxUsersNumberProviderTests + { + [TestMethod] + public async Task GetMaxUsersNumberAsync_WarmUp_Test() + { + #region Stubs + var settings = new InstanceSettings + { + MaxUsersCapacity = 1000 + }; + #endregion + + #region Mocks + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.GetTwitterUsersCountAsync()) + .ReturnsAsync(1000); + #endregion + + var provider = new MaxUsersNumberProvider(settings, twitterUserDalMock.Object); + + var result = await provider.GetMaxUsersNumberAsync(); + Assert.AreEqual(250, result); + + result = await provider.GetMaxUsersNumberAsync(); + Assert.AreEqual(250, result); + + result = await provider.GetMaxUsersNumberAsync(); + Assert.AreEqual(250, result); + + result = await provider.GetMaxUsersNumberAsync(); + Assert.AreEqual(250, result); + + result = await provider.GetMaxUsersNumberAsync(); + Assert.AreEqual(1000, result); + + #region Validations + twitterUserDalMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task GetMaxUsersNumberAsync_NoWarmUp_Test() + { + #region Stubs + var settings = new InstanceSettings + { + MaxUsersCapacity = 1000 + }; + #endregion + + #region Mocks + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.GetTwitterUsersCountAsync()) + .ReturnsAsync(249); + #endregion + + var provider = new MaxUsersNumberProvider(settings, twitterUserDalMock.Object); + + var result = await provider.GetMaxUsersNumberAsync(); + Assert.AreEqual(1000, result); + + #region Validations + twitterUserDalMock.VerifyAll(); + #endregion + } + } +} \ No newline at end of file From f6ed37b46a31f9d0d254d80ea8ea4584366ca6d2 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 22 Jan 2021 23:43:56 -0500 Subject: [PATCH 08/12] clean up --- .../CachedTweetsPostgresDalTests.cs | 9 +-------- .../DbInitializerPostgresDalTests.cs | 11 ++--------- .../FollowersPostgresDalTests.cs | 17 +++++------------ .../TwitterUserPostgresDalTests.cs | 9 +-------- 4 files changed, 9 insertions(+), 37 deletions(-) diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/CachedTweetsPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/CachedTweetsPostgresDalTests.cs index f173685..24672c3 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/CachedTweetsPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/CachedTweetsPostgresDalTests.cs @@ -22,14 +22,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task CleanUp() { var dal = new DbInitializerPostgresDal(_settings, _tools); - try - { - await dal.DeleteAllAsync(); - } - catch (Exception e) - { - Console.WriteLine(e); - } + await dal.DeleteAllAsync(); } [TestMethod] diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/DbInitializerPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/DbInitializerPostgresDalTests.cs index 9ede041..9cd59a3 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/DbInitializerPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/DbInitializerPostgresDalTests.cs @@ -13,21 +13,14 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task CleanUp() { var dal = new DbInitializerPostgresDal(_settings, _tools); - try - { - await dal.DeleteAllAsync(); - } - catch (Exception e) - { - Console.WriteLine(e); - } + await dal.DeleteAllAsync(); } [TestMethod] public async Task GetCurrentDbVersionAsync_UninitializedDb() { var dal = new DbInitializerPostgresDal(_settings, _tools); - + var current = await dal.GetCurrentDbVersionAsync(); Assert.IsNull(current); } diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs index b2289c4..cf08856 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs @@ -24,14 +24,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task CleanUp() { var dal = new DbInitializerPostgresDal(_settings, _tools); - try - { - await dal.DeleteAllAsync(); - } - catch (Exception e) - { - Console.WriteLine(e); - } + await dal.DeleteAllAsync(); } [TestMethod] @@ -39,7 +32,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers { var acct = "myhandle"; var host = "domain.ext"; - var following = new[] {12, 19, 23}; + var following = new[] { 12, 19, 23 }; var followingSync = new Dictionary() { {12, 165L}, @@ -48,7 +41,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers }; var inboxRoute = "/myhandle/inbox"; var sharedInboxRoute = "/inbox"; - + var dal = new FollowersPostgresDal(_settings); await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, following, followingSync); @@ -106,7 +99,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers //User 1 var acct = "myhandle1"; var host = "domain.ext"; - var following = new[] { 1,2,3 }; + var following = new[] { 1, 2, 3 }; var followingSync = new Dictionary(); var inboxRoute = "/myhandle1/inbox"; var sharedInboxRoute = "/inbox"; @@ -203,7 +196,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers }; result.Followings = updatedFollowing.ToList(); result.FollowingsSyncStatus = updatedFollowingSync; - + await dal.UpdateFollowerAsync(result); result = await dal.GetFollowerAsync(acct, host); diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs index 12e6f7f..d71842f 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs @@ -22,14 +22,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task CleanUp() { var dal = new DbInitializerPostgresDal(_settings, _tools); - try - { - await dal.DeleteAllAsync(); - } - catch (Exception e) - { - Console.WriteLine(e); - } + await dal.DeleteAllAsync(); } [TestMethod] From 2f933ffc9c83b670e41909c0a127b483a51aef47 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 23 Jan 2021 00:11:31 -0500 Subject: [PATCH 09/12] added index --- .../DataAccessLayers/DbInitializerPostgresDal.cs | 4 ++++ .../Tools/PostgresTools.cs | 15 ++++----------- .../DbInitializerPostgresDalTests.cs | 6 +++++- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs index a50e05b..fd4b703 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs @@ -141,6 +141,10 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers { var addLastSync = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD lastSync TIMESTAMP (2) WITHOUT TIME ZONE"; await _tools.ExecuteRequestAsync(addLastSync); + + var addIndex = $@"CREATE INDEX IF NOT EXISTS lastsync_twitteruser ON {_settings.TwitterUserTableName}(lastSync)"; + await _tools.ExecuteRequestAsync(addIndex); + return new Version(2, 0); } diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Tools/PostgresTools.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Tools/PostgresTools.cs index 223b1ea..32a379f 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Tools/PostgresTools.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Tools/PostgresTools.cs @@ -18,18 +18,11 @@ namespace BirdsiteLive.DAL.Postgres.Tools public async Task ExecuteRequestAsync(string request) { - try + using (var conn = new NpgsqlConnection(_settings.ConnString)) + using (var cmd = new NpgsqlCommand(request, conn)) { - using (var conn = new NpgsqlConnection(_settings.ConnString)) - using (var cmd = new NpgsqlCommand(request, conn)) - { - await conn.OpenAsync(); - await cmd.ExecuteNonQueryAsync(); - } - } - catch (Exception e) - { - Console.WriteLine(e); + await conn.OpenAsync(); + await cmd.ExecuteNonQueryAsync(); } } } diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/DbInitializerPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/DbInitializerPostgresDalTests.cs index 9cd59a3..a186dc3 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/DbInitializerPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/DbInitializerPostgresDalTests.cs @@ -13,7 +13,11 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task CleanUp() { var dal = new DbInitializerPostgresDal(_settings, _tools); - await dal.DeleteAllAsync(); + try + { + await dal.DeleteAllAsync(); + } + catch (Exception ) { } } [TestMethod] From cdb8aa48e16a87e966f61b15fe7aa083b3336071 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 23 Jan 2021 00:23:37 -0500 Subject: [PATCH 10/12] added Db version update --- .../DataAccessLayers/DbInitializerPostgresDal.cs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs index fd4b703..ca883ff 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs @@ -145,12 +145,25 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers var addIndex = $@"CREATE INDEX IF NOT EXISTS lastsync_twitteruser ON {_settings.TwitterUserTableName}(lastSync)"; await _tools.ExecuteRequestAsync(addIndex); - return new Version(2, 0); + await UpdateDbVersionAsync(to); + return to; } throw new NotImplementedException(); } + private async Task UpdateDbVersionAsync(Version newVersion) + { + using (var dbConnection = Connection) + { + dbConnection.Open(); + + await dbConnection.ExecuteAsync( + $"UPDATE {_settings.DbVersionTableName} SET major = @major, minor = @minor WHERE type = @type", + new { type = DbVersionType, major = newVersion.Major, minor = newVersion.Minor }); + } + } + public async Task DeleteAllAsync() { var dropsRequests = new[] From 8a147b02771a3da15abb766960cb4da14c019666 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 23 Jan 2021 00:26:23 -0500 Subject: [PATCH 11/12] clean up --- src/DataAccessLayers/BirdsiteLive.DAL/DatabaseInitializer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/DatabaseInitializer.cs b/src/DataAccessLayers/BirdsiteLive.DAL/DatabaseInitializer.cs index 2a5f60d..39e1e84 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/DatabaseInitializer.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/DatabaseInitializer.cs @@ -29,11 +29,11 @@ namespace BirdsiteLive.DAL if (currentVersion == mandatoryVersion) return; // Init Db - var migrationPatterns = _dbInitializerDal.GetMigrationPatterns(); if (currentVersion == null) currentVersion = await _dbInitializerDal.InitDbAsync(); // Migrate Db + var migrationPatterns = _dbInitializerDal.GetMigrationPatterns(); while (migrationPatterns.Any(x => x.Item1 == currentVersion)) { var migration = migrationPatterns.First(x => x.Item1 == currentVersion); From 1f7bc8936c9a747b35b7062c6e4c79e3d0f8f073 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 23 Jan 2021 00:29:42 -0500 Subject: [PATCH 12/12] road to 0.11.0 --- src/BirdsiteLive/BirdsiteLive.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index 4729487..df9c11b 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -4,7 +4,7 @@ netcoreapp3.1 d21486de-a812-47eb-a419-05682bb68856 Linux - 0.10.1 + 0.11.0