sam/BirdsiteLIVE
sam/BirdsiteLIVE
Archived
1
0
Fork 0
Code Issues 1 Pull requests 1 Projects Releases Packages 1 Wiki Activity

Merge pull request #67 from NicolasConstant/develop

Browse source 
0.11.0 PR
This commit is contained in:
Nicolas Constant 2021-01-24 02:22:11 +01:00 committed by GitHub
commit cb4883c799
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
37 changed files with 1620 additions and 143 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
VARIABLES.md
View file

@ -10,4 +10,5 @@ You can configure some of BirdsiteLIVE's settings via environment variables (tho
## Instance customization
* `Instance:Name` (default: BirdsiteLIVE) the name of the instance
* `Instance:ResolveMentionsInProfiles` (default: true) to enable or disable mentions parsing in profile's description. Resolving it will consume more User's API calls since newly discovered account can also contain references to others accounts as well. On a big instance it is recommended to disable it.
* `Instance:ResolveMentionsInProfiles` (default: true) to enable or disable mentions parsing in profile's description. Resolving it will consume more User's API calls since newly discovered account can also contain references to others accounts as well. On a big instance it is recommended to disable it.
* `Instance:PublishReplies` (default: false) to enable or disable replies publishing.

2
src/BirdsiteLive.Common/Settings/InstanceSettings.cs
View file

@ -6,5 +6,7 @@
public string Domain { get; set; }
public string AdminEmail { get; set; }
public bool ResolveMentionsInProfiles { get; set; }
public bool PublishReplies { get; set; }
public int MaxUsersCapacity { get; set; }
}
}

4
src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj
View file

@ -17,4 +17,8 @@
<ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
</ItemGroup>
<ItemGroup>
<Folder Include="Tools\" />
</ItemGroup>
</Project>

11
src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs
View file

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
@ -45,7 +46,13 @@ namespace BirdsiteLive.Pipeline.Processors
else if (tweets.Length > 0 && user.LastTweetPostedId == -1)
{
var tweetId = tweets.Last().Id;
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId);
var now = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, now);
}
else
{
var now = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, now);
}
}

10
src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
View file

@ -4,9 +4,11 @@ using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using BirdsiteLive.Common.Extensions;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Pipeline.Contracts;
using BirdsiteLive.Pipeline.Tools;
using Microsoft.Extensions.Logging;
namespace BirdsiteLive.Pipeline.Processors
@ -14,13 +16,16 @@ namespace BirdsiteLive.Pipeline.Processors
public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor
{
private readonly ITwitterUserDal _twitterUserDal;
private readonly IMaxUsersNumberProvider _maxUsersNumberProvider;
private readonly ILogger<RetrieveTwitterUsersProcessor> _logger;
public int WaitFactor = 1000 * 60; //1 min
#region Ctor
public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, ILogger<RetrieveTwitterUsersProcessor> logger)
public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IMaxUsersNumberProvider maxUsersNumberProvider, ILogger<RetrieveTwitterUsersProcessor> logger)
{
_twitterUserDal = twitterUserDal;
_maxUsersNumberProvider = maxUsersNumberProvider;
_logger = logger;
}
#endregion
@ -33,7 +38,8 @@ namespace BirdsiteLive.Pipeline.Processors
try
{
var users = await _twitterUserDal.GetAllTwitterUsersAsync();
var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync();
var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber);
var userCount = users.Any() ? users.Length : 1;
var splitNumber = (int) Math.Ceiling(userCount / 15d);

6
src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs
View file

@ -1,4 +1,5 @@
using System.Linq;
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
@ -23,7 +24,8 @@ namespace BirdsiteLive.Pipeline.Processors
var userId = userWithTweetsToSync.User.Id;
var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();
var minimumSync = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).Min();
await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync);
var now = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, now);
}
}
}

14
src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs
View file

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Domain;
@ -21,15 +22,17 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
private readonly IActivityPubService _activityPubService;
private readonly IStatusService _statusService;
private readonly IFollowersDal _followersDal;
private readonly InstanceSettings _settings;
private readonly ILogger<SendTweetsToInboxTask> _logger;
#region Ctor
public SendTweetsToInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, ILogger<SendTweetsToInboxTask> logger)
public SendTweetsToInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger<SendTweetsToInboxTask> logger)
{
_activityPubService = activityPubService;
_statusService = statusService;
_followersDal = followersDal;
_settings = settings;
_logger = logger;
}
#endregion
@ -52,8 +55,13 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
{
try
{
var note = _statusService.GetStatus(user.Acct, tweet);
await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox);
if (!tweet.IsReply ||
tweet.IsReply && tweet.IsThread ||
_settings.PublishReplies)
{
var note = _statusService.GetStatus(user.Acct, tweet);
await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox);
}
}
catch (ArgumentException e)
{

16
src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs
View file

@ -2,6 +2,7 @@
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Domain;
@ -20,14 +21,16 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
private readonly IStatusService _statusService;
private readonly IActivityPubService _activityPubService;
private readonly IFollowersDal _followersDal;
private readonly InstanceSettings _settings;
private readonly ILogger<SendTweetsToSharedInboxTask> _logger;
#region Ctor
public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, ILogger<SendTweetsToSharedInboxTask> logger)
public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger<SendTweetsToSharedInboxTask> logger)
{
_activityPubService = activityPubService;
_statusService = statusService;
_followersDal = followersDal;
_settings = settings;
_logger = logger;
}
#endregion
@ -52,8 +55,13 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
{
try
{
var note = _statusService.GetStatus(user.Acct, tweet);
await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox);
if (!tweet.IsReply ||
tweet.IsReply && tweet.IsThread ||
_settings.PublishReplies)
{
var note = _statusService.GetStatus(user.Acct, tweet);
await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox);
}
}
catch (ArgumentException e)
{
@ -66,7 +74,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
throw;
}
}
syncStatus = tweet.Id;
}
}

15
src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs
View file

@ -20,15 +20,18 @@ namespace BirdsiteLive.Pipeline
private readonly IRetrieveTweetsProcessor _retrieveTweetsProcessor;
private readonly IRetrieveFollowersProcessor _retrieveFollowersProcessor;
private readonly ISendTweetsToFollowersProcessor _sendTweetsToFollowersProcessor;
private readonly ISaveProgressionProcessor _saveProgressionProcessor;
private readonly ILogger<StatusPublicationPipeline> _logger;
#region Ctor
public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ILogger<StatusPublicationPipeline> logger)
public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ISaveProgressionProcessor saveProgressionProcessor, ILogger<StatusPublicationPipeline> logger)
{
_retrieveTweetsProcessor = retrieveTweetsProcessor;
_retrieveTwitterAccountsProcessor = retrieveTwitterAccountsProcessor;
_retrieveFollowersProcessor = retrieveFollowersProcessor;
_sendTweetsToFollowersProcessor = sendTweetsToFollowersProcessor;
_saveProgressionProcessor = saveProgressionProcessor;
_logger = logger;
}
#endregion
@ -41,7 +44,9 @@ namespace BirdsiteLive.Pipeline
var retrieveTweetsBufferBlock = new BufferBlock<UserWithTweetsToSync[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
var retrieveFollowersBlock = new TransformManyBlock<UserWithTweetsToSync[], UserWithTweetsToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct));
var retrieveFollowersBufferBlock = new BufferBlock<UserWithTweetsToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
var sendTweetsToFollowersBlock = new ActionBlock<UserWithTweetsToSync>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
var sendTweetsToFollowersBlock = new TransformBlock<UserWithTweetsToSync, UserWithTweetsToSync>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
var sendTweetsToFollowersBufferBlock = new BufferBlock<UserWithTweetsToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
var saveProgressionBlock = new ActionBlock<UserWithTweetsToSync>(async x => await _saveProgressionProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
// Link pipeline
twitterUsersBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions { PropagateCompletion = true });
@ -49,14 +54,16 @@ namespace BirdsiteLive.Pipeline
retrieveTweetsBufferBlock.LinkTo(retrieveFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
retrieveFollowersBlock.LinkTo(retrieveFollowersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
retrieveFollowersBufferBlock.LinkTo(sendTweetsToFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
sendTweetsToFollowersBlock.LinkTo(sendTweetsToFollowersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
sendTweetsToFollowersBufferBlock.LinkTo(saveProgressionBlock, new DataflowLinkOptions { PropagateCompletion = true });
// Launch twitter user retriever
var retrieveTwitterAccountsTask = _retrieveTwitterAccountsProcessor.GetTwitterUsersAsync(twitterUsersBufferBlock, ct);
// Wait
await Task.WhenAny(new[] { retrieveTwitterAccountsTask, sendTweetsToFollowersBlock.Completion });
await Task.WhenAny(new[] { retrieveTwitterAccountsTask, saveProgressionBlock.Completion });
var ex = retrieveTwitterAccountsTask.IsFaulted ? retrieveTwitterAccountsTask.Exception : sendTweetsToFollowersBlock.Completion.Exception;
var ex = retrieveTwitterAccountsTask.IsFaulted ? retrieveTwitterAccountsTask.Exception : saveProgressionBlock.Completion.Exception;
_logger.LogCritical(ex, "An error occurred, pipeline stopped");
}
}

49
src/BirdsiteLive.Pipeline/Tools/MaxUsersNumberProvider.cs Normal file
View file

@ -0,0 +1,49 @@
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
namespace BirdsiteLive.Pipeline.Tools
{
public interface IMaxUsersNumberProvider
{
Task<int> GetMaxUsersNumberAsync();
}
public class MaxUsersNumberProvider : IMaxUsersNumberProvider
{
private readonly InstanceSettings _instanceSettings;
private readonly ITwitterUserDal _twitterUserDal;
private int _totalUsersCount = -1;
private int _warmUpIterations;
#region Ctor
public MaxUsersNumberProvider(InstanceSettings instanceSettings, ITwitterUserDal twitterUserDal)
{
_instanceSettings = instanceSettings;
_twitterUserDal = twitterUserDal;
}
#endregion
public async Task<int> GetMaxUsersNumberAsync()
{
// Init data
if (_totalUsersCount == -1)
{
_totalUsersCount = await _twitterUserDal.GetTwitterUsersCountAsync();
var warmUpMaxCapacity = _instanceSettings.MaxUsersCapacity / 4;
_warmUpIterations = warmUpMaxCapacity == 0 ? 0 : (int)(_totalUsersCount / (float)warmUpMaxCapacity);
}
// Return if warm up ended
if (_warmUpIterations <= 0) return _instanceSettings.MaxUsersCapacity;
// Calculate warm up value
var maxUsers = _warmUpIterations > 0
? _instanceSettings.MaxUsersCapacity / 4
: _instanceSettings.MaxUsersCapacity;
_warmUpIterations--;
return maxUsers;
}
}
}

4
src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs
View file

@ -24,7 +24,9 @@ namespace BirdsiteLive.Twitter.Extractors
InReplyToAccount = tweet.InReplyToScreenName,
MessageContent = ExtractMessage(tweet),
Media = ExtractMedia(tweet.Media),
CreatedAt = tweet.CreatedAt.ToUniversalTime()
CreatedAt = tweet.CreatedAt.ToUniversalTime(),
IsReply = tweet.InReplyToUserId != null,
IsThread = tweet.InReplyToUserId != null && tweet.InReplyToUserId == tweet.CreatedBy.Id
};
return extractedTweet;
}

2
src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs
View file

@ -11,5 +11,7 @@ namespace BirdsiteLive.Twitter.Models
public ExtractedMedia[] Media { get; set; }
public DateTime CreatedAt { get; set; }
public string InReplyToAccount { get; set; }
public bool IsReply { get; set; }
public bool IsThread { get; set; }
}
}

2
src/BirdsiteLive.Twitter/TwitterTweetsService.cs
View file

@ -40,7 +40,7 @@ namespace BirdsiteLive.Twitter
var tweet = Tweet.GetTweet(statusId);
_statisticsHandler.CalledTweetApi();
if (tweet == null) return null; //TODO: test this
return _tweetExtractor.Extract(tweet);
return _tweetExtractor.Extract(tweet);
}
public ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1)

7
src/BirdsiteLive.sln
View file

@ -39,6 +39,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Domain.Tests",
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Pipeline.Tests", "Tests\BirdsiteLive.Pipeline.Tests\BirdsiteLive.Pipeline.Tests.csproj", "{BF51CA81-5A7A-46F8-B4FB-861C6BE59298}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdsiteLive.DAL.Tests", "Tests\BirdsiteLive.DAL.Tests\BirdsiteLive.DAL.Tests.csproj", "{5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -101,6 +103,10 @@ Global
{BF51CA81-5A7A-46F8-B4FB-861C6BE59298}.Debug|Any CPU.Build.0 = Debug|Any CPU
{BF51CA81-5A7A-46F8-B4FB-861C6BE59298}.Release|Any CPU.ActiveCfg = Release|Any CPU
{BF51CA81-5A7A-46F8-B4FB-861C6BE59298}.Release|Any CPU.Build.0 = Release|Any CPU
{5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -119,6 +125,7 @@ Global
{2A8CC30D-D775-47D1-9388-F72A5C32DE2A} = {DA3C160C-4811-4E26-A5AD-42B81FAF2D7C}
{F544D745-89A8-4DEA-B61C-A7E6C53C1D63} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
{BF51CA81-5A7A-46F8-B4FB-861C6BE59298} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
{5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {69E8DCAD-4C37-4010-858F-5F94E6FBABCE}

2
src/BirdsiteLive/BirdsiteLive.csproj
View file

@ -4,7 +4,7 @@
<TargetFramework>netcoreapp3.1</TargetFramework>
<UserSecretsId>d21486de-a812-47eb-a419-05682bb68856</UserSecretsId>
<DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
<Version>0.10.1</Version>
<Version>0.11.0</Version>
</PropertyGroup>
<ItemGroup>

25
src/BirdsiteLive/Services/FederationService.cs
View file

@ -1,6 +1,8 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.DAL;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.Pipeline;
using Microsoft.Extensions.Hosting;
@ -9,36 +11,21 @@ namespace BirdsiteLive.Services
{
public class FederationService : BackgroundService
{
private readonly IDbInitializerDal _dbInitializerDal;
private readonly IDatabaseInitializer _databaseInitializer;
private readonly IStatusPublicationPipeline _statusPublicationPipeline;
#region Ctor
public FederationService(IDbInitializerDal dbInitializerDal, IStatusPublicationPipeline statusPublicationPipeline)
public FederationService(IDatabaseInitializer databaseInitializer, IStatusPublicationPipeline statusPublicationPipeline)
{
_dbInitializerDal = dbInitializerDal;
_databaseInitializer = databaseInitializer;
_statusPublicationPipeline = statusPublicationPipeline;
}
#endregion
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
await DbInitAsync();
await _databaseInitializer.InitAndMigrateDbAsync();
await _statusPublicationPipeline.ExecuteAsync(stoppingToken);
}
private async Task DbInitAsync()
{
var currentVersion = await _dbInitializerDal.GetCurrentDbVersionAsync();
var mandatoryVersion = _dbInitializerDal.GetMandatoryDbVersion();
if (currentVersion == null)
{
await _dbInitializerDal.InitDbAsync();
}
else if (currentVersion != mandatoryVersion)
{
throw new NotImplementedException();
}
}
}
}

4
src/BirdsiteLive/appsettings.json
View file

@ -13,7 +13,9 @@
"Name": "BirdsiteLIVE",
"Domain": "domain.name",
"AdminEmail": "me@domain.name",
"ResolveMentionsInProfiles": true
"ResolveMentionsInProfiles": true,
"PublishReplies": false,
"MaxUsersCapacity": 1400
},
"Db": {
"Type": "postgres",

58
src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs
View file

@ -23,7 +23,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
public class DbInitializerPostgresDal : PostgresBase, IDbInitializerDal
{
private readonly PostgresTools _tools;
private readonly Version _currentVersion = new Version(1,0);
private readonly Version _currentVersion = new Version(2, 0);
private const string DbVersionType = "db-version";
#region Ctor
@ -32,7 +32,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
_tools = tools;
}
#endregion
public async Task<Version> GetCurrentDbVersionAsync()
{
var query = $"SELECT * FROM {_settings.DbVersionTableName} WHERE type = @type";
@ -65,17 +65,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
return _currentVersion;
}
public Tuple<Version, Version>[] GetMigrationPatterns()
{
return new Tuple<Version, Version>[0];
}
public Task MigrateDbAsync(Version from, Version to)
{
throw new NotImplementedException();
}
public async Task InitDbAsync()
public async Task<Version> InitDbAsync()
{
// Create version table
var createVersion = $@"CREATE TABLE {_settings.DbVersionTableName}
@ -124,13 +114,53 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
await _tools.ExecuteRequestAsync(createCachedTweets);
// Insert version to db
var firstVersion = new Version(1, 0);
using (var dbConnection = Connection)
{
dbConnection.Open();
await dbConnection.ExecuteAsync(
$"INSERT INTO {_settings.DbVersionTableName} (type,major,minor) VALUES(@type,@major,@minor)",
new { type = DbVersionType, major = _currentVersion.Major, minor = _currentVersion.Minor });
new { type = DbVersionType, major = firstVersion.Major, minor = firstVersion.Minor });
}
return firstVersion;
}
public Tuple<Version, Version>[] GetMigrationPatterns()
{
return new[]
{
new Tuple<Version, Version>(new Version(1,0), new Version(2,0))
};
}
public async Task<Version> MigrateDbAsync(Version from, Version to)
{
if (from == new Version(1, 0) && to == new Version(2, 0))
{
var addLastSync = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD lastSync TIMESTAMP (2) WITHOUT TIME ZONE";
await _tools.ExecuteRequestAsync(addLastSync);
var addIndex = $@"CREATE INDEX IF NOT EXISTS lastsync_twitteruser ON {_settings.TwitterUserTableName}(lastSync)";
await _tools.ExecuteRequestAsync(addIndex);
await UpdateDbVersionAsync(to);
return to;
}
throw new NotImplementedException();
}
private async Task UpdateDbVersionAsync(Version newVersion)
{
using (var dbConnection = Connection)
{
dbConnection.Open();
await dbConnection.ExecuteAsync(
$"UPDATE {_settings.DbVersionTableName} SET major = @major, minor = @minor WHERE type = @type",
new { type = DbVersionType, major = newVersion.Major, minor = newVersion.Minor });
}
}

15
src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs
View file

@ -62,32 +62,33 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
}
}
public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync()
public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber)
{
var query = $"SELECT * FROM {_settings.TwitterUserTableName}";
var query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC LIMIT @maxNumber";
using (var dbConnection = Connection)
{
dbConnection.Open();
var result = await dbConnection.QueryAsync<SyncTwitterUser>(query);
var result = await dbConnection.QueryAsync<SyncTwitterUser>(query, new { maxNumber });
return result.ToArray();
}
}
public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId)
public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, DateTime lastSync)
{
if(id == default) throw new ArgumentException("id");
if(lastTweetPostedId == default) throw new ArgumentException("lastTweetPostedId");
if(lastTweetSynchronizedForAllFollowersId == default) throw new ArgumentException("lastTweetSynchronizedForAllFollowersId");
var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId WHERE id = @id";
if(lastSync == default) throw new ArgumentException("lastSync");
var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId, lastSync = @lastSync WHERE id = @id";
using (var dbConnection = Connection)
{
dbConnection.Open();
await dbConnection.QueryAsync(query, new { id, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId });
await dbConnection.QueryAsync(query, new { id, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId, lastSync = lastSync.ToUniversalTime() });
}
}

15
src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Tools/PostgresTools.cs
View file

@ -18,18 +18,11 @@ namespace BirdsiteLive.DAL.Postgres.Tools
public async Task ExecuteRequestAsync(string request)
{
try
using (var conn = new NpgsqlConnection(_settings.ConnString))
using (var cmd = new NpgsqlCommand(request, conn))
{
using (var conn = new NpgsqlConnection(_settings.ConnString))
using (var cmd = new NpgsqlCommand(request, conn))
{
await conn.OpenAsync();
await cmd.ExecuteNonQueryAsync();
}
}
catch (Exception e)
{
Console.WriteLine(e);
await conn.OpenAsync();
await cmd.ExecuteNonQueryAsync();
}
}
}

4
src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IDbInitializerDal.cs
View file

@ -9,7 +9,7 @@ namespace BirdsiteLive.DAL.Contracts
Task<Version> GetCurrentDbVersionAsync();
Version GetMandatoryDbVersion();
Tuple<Version, Version>[] GetMigrationPatterns();
Task MigrateDbAsync(Version from, Version to);
Task InitDbAsync();
Task<Version> MigrateDbAsync(Version from, Version to);
Task<Version> InitDbAsync();
}
}

7
src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs
View file

@ -1,4 +1,5 @@
using System.Threading.Tasks;
using System;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Models;
namespace BirdsiteLive.DAL.Contracts
@ -7,8 +8,8 @@ namespace BirdsiteLive.DAL.Contracts
{
Task CreateTwitterUserAsync(string acct, long lastTweetPostedId);
Task<SyncTwitterUser> GetTwitterUserAsync(string acct);
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync();
Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId);
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber);
Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, DateTime lastSync);
Task DeleteTwitterUserAsync(string acct);
Task<int> GetTwitterUsersCountAsync();
}

46
src/DataAccessLayers/BirdsiteLive.DAL/DatabaseInitializer.cs Normal file
View file

@ -0,0 +1,46 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
namespace BirdsiteLive.DAL
{
public interface IDatabaseInitializer
{
Task InitAndMigrateDbAsync();
}
public class DatabaseInitializer : IDatabaseInitializer
{
private readonly IDbInitializerDal _dbInitializerDal;
#region Ctor
public DatabaseInitializer(IDbInitializerDal dbInitializerDal)
{
_dbInitializerDal = dbInitializerDal;
}
#endregion
public async Task InitAndMigrateDbAsync()
{
var currentVersion = await _dbInitializerDal.GetCurrentDbVersionAsync();
var mandatoryVersion = _dbInitializerDal.GetMandatoryDbVersion();
if (currentVersion == mandatoryVersion) return;
// Init Db
if (currentVersion == null)
currentVersion = await _dbInitializerDal.InitDbAsync();
// Migrate Db
var migrationPatterns = _dbInitializerDal.GetMigrationPatterns();
while (migrationPatterns.Any(x => x.Item1 == currentVersion))
{
var migration = migrationPatterns.First(x => x.Item1 == currentVersion);
currentVersion = await _dbInitializerDal.MigrateDbAsync(migration.Item1, migration.Item2);
}
if (currentVersion != mandatoryVersion) throw new Exception("Migrating DB failed");
}
}
}

6
src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs
View file

@ -1,4 +1,6 @@
namespace BirdsiteLive.DAL.Models
using System;
namespace BirdsiteLive.DAL.Models
{
public class SyncTwitterUser
{
@ -7,5 +9,7 @@
public long LastTweetPostedId { get; set; }
public long LastTweetSynchronizedForAllFollowersId { get; set; }
public DateTime LastSync { get; set; }
}
}

12
src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/CachedTweetsPostgresDalTests.cs
View file

@ -14,21 +14,15 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
public async Task TestInit()
{
var dal = new DbInitializerPostgresDal(_settings, _tools);
await dal.InitDbAsync();
var init = new DatabaseInitializer(dal);
await init.InitAndMigrateDbAsync();
}
[TestCleanup]
public async Task CleanUp()
{
var dal = new DbInitializerPostgresDal(_settings, _tools);
try
{
await dal.DeleteAllAsync();
}
catch (Exception e)
{
Console.WriteLine(e);
}
await dal.DeleteAllAsync();
}
[TestMethod]

9
src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/DbInitializerPostgresDalTests.cs
View file

@ -17,17 +17,14 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
{
await dal.DeleteAllAsync();
}
catch (Exception e)
{
Console.WriteLine(e);
}
catch (Exception ) { }
}
[TestMethod]
public async Task GetCurrentDbVersionAsync_UninitializedDb()
{
var dal = new DbInitializerPostgresDal(_settings, _tools);
var current = await dal.GetCurrentDbVersionAsync();
Assert.IsNull(current);
}
@ -35,11 +32,11 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
[TestMethod]
public async Task InitDbAsync()
{
var mandatory = new Version(1, 0);
var dal = new DbInitializerPostgresDal(_settings, _tools);
await dal.InitDbAsync();
var current = await dal.GetCurrentDbVersionAsync();
var mandatory = dal.GetMandatoryDbVersion();
Assert.IsNotNull(current);
Assert.AreEqual(mandatory.Minor, current.Minor);
Assert.AreEqual(mandatory.Major, current.Major);

20
src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs
View file

@ -16,21 +16,15 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
public async Task TestInit()
{
var dal = new DbInitializerPostgresDal(_settings, _tools);
await dal.InitDbAsync();
var init = new DatabaseInitializer(dal);
await init.InitAndMigrateDbAsync();
}
[TestCleanup]
public async Task CleanUp()
{
var dal = new DbInitializerPostgresDal(_settings, _tools);
try
{
await dal.DeleteAllAsync();
}
catch (Exception e)
{
Console.WriteLine(e);
}
await dal.DeleteAllAsync();
}
[TestMethod]
@ -38,7 +32,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
{
var acct = "myhandle";
var host = "domain.ext";
var following = new[] {12, 19, 23};
var following = new[] { 12, 19, 23 };
var followingSync = new Dictionary<int, long>()
{
{12, 165L},
@ -47,7 +41,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
};
var inboxRoute = "/myhandle/inbox";
var sharedInboxRoute = "/inbox";
var dal = new FollowersPostgresDal(_settings);
await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, following, followingSync);
@ -105,7 +99,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
//User 1
var acct = "myhandle1";
var host = "domain.ext";
var following = new[] { 1,2,3 };
var following = new[] { 1, 2, 3 };
var followingSync = new Dictionary<int, long>();
var inboxRoute = "/myhandle1/inbox";
var sharedInboxRoute = "/inbox";
@ -202,7 +196,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
};
result.Followings = updatedFollowing.ToList();
result.FollowingsSyncStatus = updatedFollowingSync;
await dal.UpdateFollowerAsync(result);
result = await dal.GetFollowerAsync(acct, host);

53
src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs
View file

@ -14,21 +14,15 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
public async Task TestInit()
{
var dal = new DbInitializerPostgresDal(_settings, _tools);
await dal.InitDbAsync();
var init = new DatabaseInitializer(dal);
await init.InitAndMigrateDbAsync();
}
[TestCleanup]
public async Task CleanUp()
{
var dal = new DbInitializerPostgresDal(_settings, _tools);
try
{
await dal.DeleteAllAsync();
}
catch (Exception e)
{
Console.WriteLine(e);
}
await dal.DeleteAllAsync();
}
[TestMethod]
@ -70,13 +64,15 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
var updatedLastTweetId = 1600L;
var updatedLastSyncId = 1550L;
await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId);
var now = DateTime.Now;
await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, now);
result = await dal.GetTwitterUserAsync(acct);
Assert.AreEqual(acct, result.Acct);
Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId);
Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId);
Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100);
}
[TestMethod]
@ -108,7 +104,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
await dal.CreateTwitterUserAsync(acct, lastTweetId);
}
var result = await dal.GetAllTwitterUsersAsync();
var result = await dal.GetAllTwitterUsersAsync(1000);
Assert.AreEqual(1000, result.Length);
Assert.IsFalse(result[0].Id == default);
Assert.IsFalse(result[0].Acct == default);
@ -116,6 +112,41 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
}
[TestMethod]
public async Task GetAllTwitterUsers_Limited()
{
var now = DateTime.Now;
var oldest = now.AddDays(-3);
var newest = now.AddDays(-2);
var dal = new TwitterUserPostgresDal(_settings);
for (var i = 0; i < 20; i++)
{
var acct = $"myid{i}";
var lastTweetId = 1548L;
await dal.CreateTwitterUserAsync(acct, lastTweetId);
}
var allUsers = await dal.GetAllTwitterUsersAsync(100);
for (var i = 0; i < 20; i++)
{
var user = allUsers[i];
var date = i % 2 == 0 ? oldest : newest;
await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, date);
}
var result = await dal.GetAllTwitterUsersAsync(10);
Assert.AreEqual(10, result.Length);
Assert.IsFalse(result[0].Id == default);
Assert.IsFalse(result[0].Acct == default);
Assert.IsFalse(result[0].LastTweetPostedId == default);
Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
foreach (var acc in result)
Assert.IsTrue(Math.Abs((acc.LastSync - oldest.ToUniversalTime()).TotalMilliseconds) < 1000);
}
[TestMethod]
public async Task CountTwitterUsers()
{

21
src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj Normal file
View file

@ -0,0 +1,21 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
<PackageReference Include="Moq" Version="4.14.5" />
<PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
<PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
<PackageReference Include="coverlet.collector" Version="1.2.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
</ItemGroup>
</Project>

240
src/Tests/BirdsiteLive.DAL.Tests/DatabaseInitializerTests.cs Normal file
View file

@ -0,0 +1,240 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Moq;
namespace BirdsiteLive.DAL.Tests
{
[TestClass]
public class DatabaseInitializerTests
{
[TestMethod]
public async Task DbInitAsync_UpToDate_Test()
{
#region Stubs
var current = new Version(2, 3);
var mandatory = new Version(2, 3);
#endregion
#region Mocks
var dbInitializerDal = new Mock<IDbInitializerDal>(MockBehavior.Strict);
dbInitializerDal
.Setup(x => x.GetCurrentDbVersionAsync())
.ReturnsAsync(current);
dbInitializerDal
.Setup(x => x.GetMandatoryDbVersion())
.Returns(mandatory);
#endregion
var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object);
await dbInitializer.InitAndMigrateDbAsync();
#region Validations
dbInitializerDal.VerifyAll();
#endregion
}
[TestMethod]
public async Task DbInitAsync_NoDb_Test()
{
#region Stubs
var current = (Version)null;
var mandatory = new Version(1, 0);
var migrationPatterns = new Tuple<Version, Version>[0];
#endregion
#region Mocks
var dbInitializerDal = new Mock<IDbInitializerDal>(MockBehavior.Strict);
dbInitializerDal
.Setup(x => x.GetCurrentDbVersionAsync())
.ReturnsAsync(current);
dbInitializerDal
.Setup(x => x.GetMandatoryDbVersion())
.Returns(mandatory);
dbInitializerDal
.Setup(x => x.InitDbAsync())
.ReturnsAsync(new Version(1, 0));
dbInitializerDal
.Setup(x => x.GetMigrationPatterns())
.Returns(migrationPatterns);
#endregion
var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object);
await dbInitializer.InitAndMigrateDbAsync();
#region Validations
dbInitializerDal.VerifyAll();
#endregion
}
[TestMethod]
public async Task DbInitAsync_NoDb_Migration_Test()
{
#region Stubs
var current = (Version)null;
var mandatory = new Version(2, 3);
var migrationPatterns = new Tuple<Version, Version>[]
{
new Tuple<Version, Version>(new Version(1,0), new Version(1,7)),
new Tuple<Version, Version>(new Version(1,7), new Version(2,0)),
new Tuple<Version, Version>(new Version(2,0), new Version(2,3))
};
#endregion
#region Mocks
var dbInitializerDal = new Mock<IDbInitializerDal>(MockBehavior.Strict);
dbInitializerDal
.Setup(x => x.GetCurrentDbVersionAsync())
.ReturnsAsync(current);
dbInitializerDal
.Setup(x => x.GetMandatoryDbVersion())
.Returns(mandatory);
dbInitializerDal
.Setup(x => x.InitDbAsync())
.ReturnsAsync(new Version(1, 0));
dbInitializerDal
.Setup(x => x.GetMigrationPatterns())
.Returns(migrationPatterns);
foreach (var m in migrationPatterns)
{
dbInitializerDal
.Setup(x => x.MigrateDbAsync(
It.Is<Version>(y => y == m.Item1),
It.Is<Version>(y => y == m.Item2)
))
.ReturnsAsync(m.Item2);
}
#endregion
var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object);
await dbInitializer.InitAndMigrateDbAsync();
#region Validations
dbInitializerDal.VerifyAll();
#endregion
}
[TestMethod]
public async Task DbInitAsync_HasDb_Migration_Test()
{
#region Stubs
var current = new Version(1, 7);
var mandatory = new Version(2, 3);
var migrationPatterns = new Tuple<Version, Version>[]
{
new Tuple<Version, Version>(new Version(1,0), new Version(1,7)),
new Tuple<Version, Version>(new Version(1,7), new Version(2,0)),
new Tuple<Version, Version>(new Version(2,0), new Version(2,3))
};
#endregion
#region Mocks
var dbInitializerDal = new Mock<IDbInitializerDal>(MockBehavior.Strict);
dbInitializerDal
.Setup(x => x.GetCurrentDbVersionAsync())
.ReturnsAsync(current);
dbInitializerDal
.Setup(x => x.GetMandatoryDbVersion())
.Returns(mandatory);
dbInitializerDal
.Setup(x => x.GetMigrationPatterns())
.Returns(migrationPatterns);
foreach (var m in migrationPatterns.Skip(1))
{
dbInitializerDal
.Setup(x => x.MigrateDbAsync(
It.Is<Version>(y => y == m.Item1),
It.Is<Version>(y => y == m.Item2)
))
.ReturnsAsync(m.Item2);
}
#endregion
var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object);
await dbInitializer.InitAndMigrateDbAsync();
#region Validations
dbInitializerDal.VerifyAll();
#endregion
}
[TestMethod]
[ExpectedException(typeof(Exception))]
public async Task DbInitAsync_NoDb_Migration_Error_Test()
{
#region Stubs
var current = (Version)null;
var mandatory = new Version(2, 3);
var migrationPatterns = new Tuple<Version, Version>[]
{
new Tuple<Version, Version>(new Version(1,0), new Version(1,7)),
new Tuple<Version, Version>(new Version(1,7), new Version(2,0)),
new Tuple<Version, Version>(new Version(2,0), new Version(2,2))
};
#endregion
#region Mocks
var dbInitializerDal = new Mock<IDbInitializerDal>(MockBehavior.Strict);
dbInitializerDal
.Setup(x => x.GetCurrentDbVersionAsync())
.ReturnsAsync(current);
dbInitializerDal
.Setup(x => x.GetMandatoryDbVersion())
.Returns(mandatory);
dbInitializerDal
.Setup(x => x.InitDbAsync())
.ReturnsAsync(new Version(1, 0));
dbInitializerDal
.Setup(x => x.GetMigrationPatterns())
.Returns(migrationPatterns);
foreach (var m in migrationPatterns)
{
dbInitializerDal
.Setup(x => x.MigrateDbAsync(
It.Is<Version>(y => y == m.Item1),
It.Is<Version>(y => y == m.Item2)
))
.ReturnsAsync(m.Item2);
}
#endregion
var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object);
try
{
await dbInitializer.InitAndMigrateDbAsync();
}
finally
{
#region Validations
dbInitializerDal.VerifyAll();
#endregion
}
}
}
}

4
src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj
View file

@ -18,4 +18,8 @@
<ProjectReference Include="..\..\BirdsiteLive.Pipeline\BirdsiteLive.Pipeline.csproj" />
</ItemGroup>
<ItemGroup>
<Folder Include="Tools\" />
</ItemGroup>
</Project>

4
src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs
View file

@ -1,3 +1,4 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
@ -54,7 +55,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
.Setup(x => x.UpdateTwitterUserAsync(
It.Is<int>(y => y == user1.Id),
It.Is<long>(y => y == tweets.Last().Id),
It.Is<long>(y => y == tweets.Last().Id)
It.Is<long>(y => y == tweets.Last().Id),
It.IsAny<DateTime>()
))
.Returns(Task.CompletedTask);
#endregion

88
src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs
View file

@ -3,9 +3,11 @@ using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Pipeline.Processors;
using BirdsiteLive.Pipeline.Tools;
using Microsoft.Extensions.Logging;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Moq;
@ -26,24 +28,32 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
new SyncTwitterUser(),
new SyncTwitterUser(),
};
var maxUsers = 1000;
#endregion
#region Mocks
var maxUsersNumberProviderMock = new Mock<IMaxUsersNumberProvider>(MockBehavior.Strict);
maxUsersNumberProviderMock
.Setup(x => x.GetMaxUsersNumberAsync())
.ReturnsAsync(maxUsers);
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync())
.Setup(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == maxUsers)))
.ReturnsAsync(users);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object);
processor.WaitFactor = 10;
processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
await Task.Delay(50);
#region Validations
maxUsersNumberProviderMock.VerifyAll();
twitterUserDalMock.VerifyAll();
Assert.AreEqual(3, buffer.Count);
buffer.TryReceive(out var result);
@ -60,25 +70,37 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
for (var i = 0; i < 30; i++)
users.Add(new SyncTwitterUser());
var maxUsers = 1000;
#endregion
#region Mocks
var maxUsersNumberProviderMock = new Mock<IMaxUsersNumberProvider>(MockBehavior.Strict);
maxUsersNumberProviderMock
.Setup(x => x.GetMaxUsersNumberAsync())
.ReturnsAsync(maxUsers);
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.SetupSequence(x => x.GetAllTwitterUsersAsync())
.SetupSequence(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == maxUsers)))
.ReturnsAsync(users.ToArray())
.ReturnsAsync(new SyncTwitterUser[0])
.ReturnsAsync(new SyncTwitterUser[0])
.ReturnsAsync(new SyncTwitterUser[0])
.ReturnsAsync(new SyncTwitterUser[0]);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object);
processor.WaitFactor = 2;
processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
await Task.Delay(200);
await Task.Delay(300);
#region Validations
maxUsersNumberProviderMock.VerifyAll();
twitterUserDalMock.VerifyAll();
Assert.AreEqual(15, buffer.Count);
buffer.TryReceive(out var result);
@ -95,25 +117,37 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
for (var i = 0; i < 31; i++)
users.Add(new SyncTwitterUser());
var maxUsers = 1000;
#endregion
#region Mocks
var maxUsersNumberProviderMock = new Mock<IMaxUsersNumberProvider>(MockBehavior.Strict);
maxUsersNumberProviderMock
.Setup(x => x.GetMaxUsersNumberAsync())
.ReturnsAsync(maxUsers);
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.SetupSequence(x => x.GetAllTwitterUsersAsync())
.SetupSequence(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == maxUsers)))
.ReturnsAsync(users.ToArray())
.ReturnsAsync(new SyncTwitterUser[0])
.ReturnsAsync(new SyncTwitterUser[0])
.ReturnsAsync(new SyncTwitterUser[0])
.ReturnsAsync(new SyncTwitterUser[0]);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object);
processor.WaitFactor = 2;
processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
await Task.Delay(200);
#region Validations
maxUsersNumberProviderMock.VerifyAll();
twitterUserDalMock.VerifyAll();
Assert.AreEqual(11, buffer.Count);
buffer.TryReceive(out var result);
@ -126,24 +160,33 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
{
#region Stubs
var buffer = new BufferBlock<SyncTwitterUser[]>();
var maxUsers = 1000;
#endregion
#region Mocks
var maxUsersNumberProviderMock = new Mock<IMaxUsersNumberProvider>(MockBehavior.Strict);
maxUsersNumberProviderMock
.Setup(x => x.GetMaxUsersNumberAsync())
.ReturnsAsync(maxUsers);
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync())
.Setup(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == maxUsers)))
.ReturnsAsync(new SyncTwitterUser[0]);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object);
processor.WaitFactor = 1;
processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
await Task.Delay(50);
#region Validations
maxUsersNumberProviderMock.VerifyAll();
twitterUserDalMock.VerifyAll();
Assert.AreEqual(0, buffer.Count);
#endregion
@ -154,24 +197,33 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
{
#region Stubs
var buffer = new BufferBlock<SyncTwitterUser[]>();
var maxUsers = 1000;
#endregion
#region Mocks
var maxUsersNumberProviderMock = new Mock<IMaxUsersNumberProvider>(MockBehavior.Strict);
maxUsersNumberProviderMock
.Setup(x => x.GetMaxUsersNumberAsync())
.ReturnsAsync(maxUsers);
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync())
.Setup(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == maxUsers)))
.Returns(async () => await DelayFaultedTask<SyncTwitterUser[]>(new Exception()));
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object);
processor.WaitFactor = 10;
var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
await Task.WhenAny(t, Task.Delay(50));
#region Validations
maxUsersNumberProviderMock.VerifyAll();
twitterUserDalMock.VerifyAll();
Assert.AreEqual(0, buffer.Count);
#endregion
@ -185,14 +237,22 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var buffer = new BufferBlock<SyncTwitterUser[]>();
var canTokenS = new CancellationTokenSource();
canTokenS.Cancel();
var maxUsers = 1000;
#endregion
#region Mocks
var maxUsersNumberProviderMock = new Mock<IMaxUsersNumberProvider>(MockBehavior.Strict);
maxUsersNumberProviderMock
.Setup(x => x.GetMaxUsersNumberAsync())
.ReturnsAsync(maxUsers);
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object);
processor.WaitFactor = 1;
await processor.GetTwitterUsersAsync(buffer, canTokenS.Token);
}

12
src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs
View file

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
@ -60,7 +61,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
.Setup(x => x.UpdateTwitterUserAsync(
It.Is<int>(y => y == user.Id),
It.Is<long>(y => y == tweet2.Id),
It.Is<long>(y => y == tweet2.Id)
It.Is<long>(y => y == tweet2.Id),
It.IsAny<DateTime>()
))
.Returns(Task.CompletedTask);
#endregion
@ -123,7 +125,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
.Setup(x => x.UpdateTwitterUserAsync(
It.Is<int>(y => y == user.Id),
It.Is<long>(y => y == tweet3.Id),
It.Is<long>(y => y == tweet2.Id)
It.Is<long>(y => y == tweet2.Id),
It.IsAny<DateTime>()
))
.Returns(Task.CompletedTask);
#endregion
@ -194,7 +197,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
.Setup(x => x.UpdateTwitterUserAsync(
It.Is<int>(y => y == user.Id),
It.Is<long>(y => y == tweet3.Id),
It.Is<long>(y => y == tweet2.Id)
It.Is<long>(y => y == tweet2.Id),
It.IsAny<DateTime>()
))
.Returns(Task.CompletedTask);
#endregion

396
src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs
View file

@ -4,6 +4,7 @@ using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using BirdsiteLive.ActivityPub.Models;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Domain;
@ -54,6 +55,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
InboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
};
var settings = new InstanceSettings
{
PublishReplies = false
};
#endregion
#region Mocks
@ -83,7 +89,239 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
#endregion
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, loggerMock.Object);
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser);
#region Validations
activityPubService.VerifyAll();
statusServiceMock.VerifyAll();
followersDalMock.VerifyAll();
#endregion
}
[TestMethod]
public async Task ExecuteAsync_SingleTweet_Reply_Test()
{
#region Stubs
var tweetId = 10;
var tweets = new List<ExtractedTweet>
{
new ExtractedTweet
{
Id = tweetId,
IsReply = true,
IsThread = false
}
};
var noteId = "noteId";
var note = new Note()
{
id = noteId
};
var twitterHandle = "Test";
var twitterUserId = 7;
var twitterUser = new SyncTwitterUser
{
Id = twitterUserId,
Acct = twitterHandle
};
var host = "domain.ext";
var inbox = "/user/inbox";
var follower = new Follower
{
Id = 1,
Host = host,
InboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
};
var settings = new InstanceSettings
{
PublishReplies = false
};
#endregion
#region Mocks
var activityPubService = new Mock<IActivityPubService>(MockBehavior.Strict);
var statusServiceMock = new Mock<IStatusService>(MockBehavior.Strict);
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
followersDalMock
.Setup(x => x.UpdateFollowerAsync(
It.Is<Follower>(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId)))
.Returns(Task.CompletedTask);
var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
#endregion
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser);
#region Validations
activityPubService.VerifyAll();
statusServiceMock.VerifyAll();
followersDalMock.VerifyAll();
#endregion
}
[TestMethod]
public async Task ExecuteAsync_SingleTweet_ReplyThread_Test()
{
#region Stubs
var tweetId = 10;
var tweets = new List<ExtractedTweet>
{
new ExtractedTweet
{
Id = tweetId,
IsReply = true,
IsThread = true
}
};
var noteId = "noteId";
var note = new Note()
{
id = noteId
};
var twitterHandle = "Test";
var twitterUserId = 7;
var twitterUser = new SyncTwitterUser
{
Id = twitterUserId,
Acct = twitterHandle
};
var host = "domain.ext";
var inbox = "/user/inbox";
var follower = new Follower
{
Id = 1,
Host = host,
InboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
};
var settings = new InstanceSettings
{
PublishReplies = false
};
#endregion
#region Mocks
var activityPubService = new Mock<IActivityPubService>(MockBehavior.Strict);
activityPubService
.Setup(x => x.PostNewNoteActivity(
It.Is<Note>(y => y.id == noteId),
It.Is<string>(y => y == twitterHandle),
It.Is<string>(y => y == tweetId.ToString()),
It.Is<string>(y => y == host),
It.Is<string>(y => y == inbox)))
.Returns(Task.CompletedTask);
var statusServiceMock = new Mock<IStatusService>(MockBehavior.Strict);
statusServiceMock
.Setup(x => x.GetStatus(
It.Is<string>(y => y == twitterHandle),
It.Is<ExtractedTweet>(y => y.Id == tweetId)))
.Returns(note);
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
followersDalMock
.Setup(x => x.UpdateFollowerAsync(
It.Is<Follower>(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId)))
.Returns(Task.CompletedTask);
var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
#endregion
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser);
#region Validations
activityPubService.VerifyAll();
statusServiceMock.VerifyAll();
followersDalMock.VerifyAll();
#endregion
}
[TestMethod]
public async Task ExecuteAsync_SingleTweet_PublishReply_Test()
{
#region Stubs
var tweetId = 10;
var tweets = new List<ExtractedTweet>
{
new ExtractedTweet
{
Id = tweetId,
IsReply = true,
IsThread = false
}
};
var noteId = "noteId";
var note = new Note()
{
id = noteId
};
var twitterHandle = "Test";
var twitterUserId = 7;
var twitterUser = new SyncTwitterUser
{
Id = twitterUserId,
Acct = twitterHandle
};
var host = "domain.ext";
var inbox = "/user/inbox";
var follower = new Follower
{
Id = 1,
Host = host,
InboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
};
var settings = new InstanceSettings
{
PublishReplies = true
};
#endregion
#region Mocks
var activityPubService = new Mock<IActivityPubService>(MockBehavior.Strict);
activityPubService
.Setup(x => x.PostNewNoteActivity(
It.Is<Note>(y => y.id == noteId),
It.Is<string>(y => y == twitterHandle),
It.Is<string>(y => y == tweetId.ToString()),
It.Is<string>(y => y == host),
It.Is<string>(y => y == inbox)))
.Returns(Task.CompletedTask);
var statusServiceMock = new Mock<IStatusService>(MockBehavior.Strict);
statusServiceMock
.Setup(x => x.GetStatus(
It.Is<string>(y => y == twitterHandle),
It.Is<ExtractedTweet>(y => y.Id == tweetId)))
.Returns(note);
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
followersDalMock
.Setup(x => x.UpdateFollowerAsync(
It.Is<Follower>(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId)))
.Returns(Task.CompletedTask);
var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
#endregion
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser);
#region Validations
@ -126,6 +364,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
InboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 10 } }
};
var settings = new InstanceSettings
{
PublishReplies = false
};
#endregion
#region Mocks
@ -161,7 +404,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
#endregion
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, loggerMock.Object);
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser);
#region Validations
@ -205,6 +448,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
InboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 10 } }
};
var settings = new InstanceSettings
{
PublishReplies = false
};
#endregion
#region Mocks
@ -247,7 +495,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
#endregion
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, loggerMock.Object);
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
try
{
@ -262,5 +510,147 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
#endregion
}
}
[TestMethod]
public async Task ExecuteAsync_SingleTweet_ParsingError_Test()
{
#region Stubs
var tweetId = 10;
var tweets = new List<ExtractedTweet>
{
new ExtractedTweet
{
Id = tweetId,
}
};
var noteId = "noteId";
var note = new Note()
{
id = noteId
};
var twitterHandle = "Test";
var twitterUserId = 7;
var twitterUser = new SyncTwitterUser
{
Id = twitterUserId,
Acct = twitterHandle
};
var host = "domain.ext";
var inbox = "/user/inbox";
var follower = new Follower
{
Id = 1,
Host = host,
InboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
};
var settings = new InstanceSettings
{
PublishReplies = false
};
#endregion
#region Mocks
var activityPubService = new Mock<IActivityPubService>(MockBehavior.Strict);
var statusServiceMock = new Mock<IStatusService>(MockBehavior.Strict);
statusServiceMock
.Setup(x => x.GetStatus(
It.Is<string>(y => y == twitterHandle),
It.Is<ExtractedTweet>(y => y.Id == tweetId)))
.Throws(new ArgumentException("Invalid pattern blabla at offset 9"));
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
followersDalMock
.Setup(x => x.UpdateFollowerAsync(
It.Is<Follower>(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId)))
.Returns(Task.CompletedTask);
var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
#endregion
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser);
#region Validations
activityPubService.VerifyAll();
statusServiceMock.VerifyAll();
followersDalMock.VerifyAll();
#endregion
}
[TestMethod]
[ExpectedException(typeof(ArgumentException))]
public async Task ExecuteAsync_SingleTweet_ArgumentException_Test()
{
#region Stubs
var tweetId = 10;
var tweets = new List<ExtractedTweet>
{
new ExtractedTweet
{
Id = tweetId,
}
};
var twitterHandle = "Test";
var twitterUserId = 7;
var twitterUser = new SyncTwitterUser
{
Id = twitterUserId,
Acct = twitterHandle
};
var host = "domain.ext";
var inbox = "/user/inbox";
var follower = new Follower
{
Id = 1,
Host = host,
InboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
};
var settings = new InstanceSettings
{
PublishReplies = false
};
#endregion
#region Mocks
var activityPubService = new Mock<IActivityPubService>(MockBehavior.Strict);
var statusServiceMock = new Mock<IStatusService>(MockBehavior.Strict);
statusServiceMock
.Setup(x => x.GetStatus(
It.Is<string>(y => y == twitterHandle),
It.Is<ExtractedTweet>(y => y.Id == tweetId)))
.Throws(new ArgumentException());
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
var loggerMock = new Mock<ILogger<SendTweetsToInboxTask>>();
#endregion
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
try
{
await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser);
}
finally
{
#region Validations
activityPubService.VerifyAll();
statusServiceMock.VerifyAll();
followersDalMock.VerifyAll();
#endregion
}
}
}
}

498
src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs
View file

@ -5,6 +5,7 @@ using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using BirdsiteLive.ActivityPub.Models;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Domain;
@ -72,6 +73,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 7 } }
}
};
var settings = new InstanceSettings
{
PublishReplies = false
};
#endregion
#region Mocks
@ -105,7 +111,303 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
#endregion
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, loggerMock.Object);
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray());
#region Validations
activityPubService.VerifyAll();
statusServiceMock.VerifyAll();
followersDalMock.VerifyAll();
#endregion
}
[TestMethod]
public async Task ExecuteAsync_SingleTweet_Reply_Test()
{
#region Stubs
var tweetId = 10;
var tweets = new List<ExtractedTweet>
{
new ExtractedTweet
{
Id = tweetId,
IsReply = true,
IsThread = false
}
};
var noteId = "noteId";
var note = new Note()
{
id = noteId
};
var twitterHandle = "Test";
var twitterUserId = 7;
var twitterUser = new SyncTwitterUser
{
Id = twitterUserId,
Acct = twitterHandle
};
var host = "domain.ext";
var inbox = "/inbox";
var followers = new List<Follower>
{
new Follower
{
Id = 1,
Host = host,
SharedInboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
},
new Follower
{
Id = 2,
Host = host,
SharedInboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 8 } }
},
new Follower
{
Id = 3,
Host = host,
SharedInboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 7 } }
}
};
var settings = new InstanceSettings
{
PublishReplies = false
};
#endregion
#region Mocks
var activityPubService = new Mock<IActivityPubService>(MockBehavior.Strict);
var statusServiceMock = new Mock<IStatusService>(MockBehavior.Strict);
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
foreach (var follower in followers)
{
followersDalMock
.Setup(x => x.UpdateFollowerAsync(
It.Is<Follower>(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId)))
.Returns(Task.CompletedTask);
}
var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
#endregion
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray());
#region Validations
activityPubService.VerifyAll();
statusServiceMock.VerifyAll();
followersDalMock.VerifyAll();
#endregion
}
[TestMethod]
public async Task ExecuteAsync_SingleTweet_ReplyThread_Test()
{
#region Stubs
var tweetId = 10;
var tweets = new List<ExtractedTweet>
{
new ExtractedTweet
{
Id = tweetId,
IsReply = true,
IsThread = true
}
};
var noteId = "noteId";
var note = new Note()
{
id = noteId
};
var twitterHandle = "Test";
var twitterUserId = 7;
var twitterUser = new SyncTwitterUser
{
Id = twitterUserId,
Acct = twitterHandle
};
var host = "domain.ext";
var inbox = "/inbox";
var followers = new List<Follower>
{
new Follower
{
Id = 1,
Host = host,
SharedInboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
},
new Follower
{
Id = 2,
Host = host,
SharedInboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 8 } }
},
new Follower
{
Id = 3,
Host = host,
SharedInboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 7 } }
}
};
var settings = new InstanceSettings
{
PublishReplies = false
};
#endregion
#region Mocks
var activityPubService = new Mock<IActivityPubService>(MockBehavior.Strict);
activityPubService
.Setup(x => x.PostNewNoteActivity(
It.Is<Note>(y => y.id == noteId),
It.Is<string>(y => y == twitterHandle),
It.Is<string>(y => y == tweetId.ToString()),
It.Is<string>(y => y == host),
It.Is<string>(y => y == inbox)))
.Returns(Task.CompletedTask);
var statusServiceMock = new Mock<IStatusService>(MockBehavior.Strict);
statusServiceMock
.Setup(x => x.GetStatus(
It.Is<string>(y => y == twitterHandle),
It.Is<ExtractedTweet>(y => y.Id == tweetId)))
.Returns(note);
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
foreach (var follower in followers)
{
followersDalMock
.Setup(x => x.UpdateFollowerAsync(
It.Is<Follower>(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId)))
.Returns(Task.CompletedTask);
}
var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
#endregion
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray());
#region Validations
activityPubService.VerifyAll();
statusServiceMock.VerifyAll();
followersDalMock.VerifyAll();
#endregion
}
[TestMethod]
public async Task ExecuteAsync_SingleTweet_PublishReply_Test()
{
#region Stubs
var tweetId = 10;
var tweets = new List<ExtractedTweet>
{
new ExtractedTweet
{
Id = tweetId,
IsReply = true,
IsThread = false
}
};
var noteId = "noteId";
var note = new Note()
{
id = noteId
};
var twitterHandle = "Test";
var twitterUserId = 7;
var twitterUser = new SyncTwitterUser
{
Id = twitterUserId,
Acct = twitterHandle
};
var host = "domain.ext";
var inbox = "/inbox";
var followers = new List<Follower>
{
new Follower
{
Id = 1,
Host = host,
SharedInboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
},
new Follower
{
Id = 2,
Host = host,
SharedInboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 8 } }
},
new Follower
{
Id = 3,
Host = host,
SharedInboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 7 } }
}
};
var settings = new InstanceSettings
{
PublishReplies = true
};
#endregion
#region Mocks
var activityPubService = new Mock<IActivityPubService>(MockBehavior.Strict);
activityPubService
.Setup(x => x.PostNewNoteActivity(
It.Is<Note>(y => y.id == noteId),
It.Is<string>(y => y == twitterHandle),
It.Is<string>(y => y == tweetId.ToString()),
It.Is<string>(y => y == host),
It.Is<string>(y => y == inbox)))
.Returns(Task.CompletedTask);
var statusServiceMock = new Mock<IStatusService>(MockBehavior.Strict);
statusServiceMock
.Setup(x => x.GetStatus(
It.Is<string>(y => y == twitterHandle),
It.Is<ExtractedTweet>(y => y.Id == tweetId)))
.Returns(note);
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
foreach (var follower in followers)
{
followersDalMock
.Setup(x => x.UpdateFollowerAsync(
It.Is<Follower>(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId)))
.Returns(Task.CompletedTask);
}
var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
#endregion
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray());
#region Validations
@ -165,6 +467,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
FollowingsSyncStatus = new Dictionary<int, long> {{twitterUserId, 7}}
}
};
var settings = new InstanceSettings
{
PublishReplies = false
};
#endregion
#region Mocks
@ -204,7 +511,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
#endregion
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, loggerMock.Object);
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray());
#region Validations
@ -265,6 +572,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
FollowingsSyncStatus = new Dictionary<int, long> {{twitterUserId, 7}}
}
};
var settings = new InstanceSettings
{
PublishReplies = false
};
#endregion
#region Mocks
@ -311,7 +623,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
#endregion
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, loggerMock.Object);
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
try
{
@ -326,5 +638,185 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
#endregion
}
}
[TestMethod]
public async Task ExecuteAsync_SingleTweet_ParsingError_Test()
{
#region Stubs
var tweetId = 10;
var tweets = new List<ExtractedTweet>
{
new ExtractedTweet
{
Id = tweetId,
}
};
var noteId = "noteId";
var note = new Note()
{
id = noteId
};
var twitterHandle = "Test";
var twitterUserId = 7;
var twitterUser = new SyncTwitterUser
{
Id = twitterUserId,
Acct = twitterHandle
};
var host = "domain.ext";
var inbox = "/inbox";
var followers = new List<Follower>
{
new Follower
{
Id = 1,
Host = host,
SharedInboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
},
new Follower
{
Id = 2,
Host = host,
SharedInboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 8 } }
},
new Follower
{
Id = 3,
Host = host,
SharedInboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 7 } }
}
};
var settings = new InstanceSettings
{
PublishReplies = false
};
#endregion
#region Mocks
var activityPubService = new Mock<IActivityPubService>(MockBehavior.Strict);
var statusServiceMock = new Mock<IStatusService>(MockBehavior.Strict);
statusServiceMock
.Setup(x => x.GetStatus(
It.Is<string>(y => y == twitterHandle),
It.Is<ExtractedTweet>(y => y.Id == tweetId)))
.Throws(new ArgumentException("Invalid pattern blabla at offset 9"));
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
foreach (var follower in followers)
{
followersDalMock
.Setup(x => x.UpdateFollowerAsync(
It.Is<Follower>(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId)))
.Returns(Task.CompletedTask);
}
var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
#endregion
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray());
#region Validations
activityPubService.VerifyAll();
statusServiceMock.VerifyAll();
followersDalMock.VerifyAll();
#endregion
}
[TestMethod]
[ExpectedException(typeof(ArgumentException))]
public async Task ExecuteAsync_SingleTweet_ArgumentException_Test()
{
#region Stubs
var tweetId = 10;
var tweets = new List<ExtractedTweet>
{
new ExtractedTweet
{
Id = tweetId,
}
};
var twitterHandle = "Test";
var twitterUserId = 7;
var twitterUser = new SyncTwitterUser
{
Id = twitterUserId,
Acct = twitterHandle
};
var host = "domain.ext";
var inbox = "/inbox";
var followers = new List<Follower>
{
new Follower
{
Id = 1,
Host = host,
SharedInboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
},
new Follower
{
Id = 2,
Host = host,
SharedInboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 8 } }
},
new Follower
{
Id = 3,
Host = host,
SharedInboxRoute = inbox,
FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 7 } }
}
};
var settings = new InstanceSettings
{
PublishReplies = false
};
#endregion
#region Mocks
var activityPubService = new Mock<IActivityPubService>(MockBehavior.Strict);
var statusServiceMock = new Mock<IStatusService>(MockBehavior.Strict);
statusServiceMock
.Setup(x => x.GetStatus(
It.Is<string>(y => y == twitterHandle),
It.Is<ExtractedTweet>(y => y.Id == tweetId)))
.Throws(new ArgumentException());
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
var loggerMock = new Mock<ILogger<SendTweetsToSharedInboxTask>>();
#endregion
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
try
{
await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray());
}
finally
{
#region Validations
activityPubService.VerifyAll();
statusServiceMock.VerifyAll();
followersDalMock.VerifyAll();
#endregion
}
}
}
}

79
src/Tests/BirdsiteLive.Pipeline.Tests/Tools/MaxUsersNumberProviderTests.cs Normal file
View file

@ -0,0 +1,79 @@
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.Pipeline.Tools;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Moq;
namespace BirdsiteLive.Pipeline.Tests.Tools
{
[TestClass]
public class MaxUsersNumberProviderTests
{
[TestMethod]
public async Task GetMaxUsersNumberAsync_WarmUp_Test()
{
#region Stubs
var settings = new InstanceSettings
{
MaxUsersCapacity = 1000
};
#endregion
#region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetTwitterUsersCountAsync())
.ReturnsAsync(1000);
#endregion
var provider = new MaxUsersNumberProvider(settings, twitterUserDalMock.Object);
var result = await provider.GetMaxUsersNumberAsync();
Assert.AreEqual(250, result);
result = await provider.GetMaxUsersNumberAsync();
Assert.AreEqual(250, result);
result = await provider.GetMaxUsersNumberAsync();
Assert.AreEqual(250, result);
result = await provider.GetMaxUsersNumberAsync();
Assert.AreEqual(250, result);
result = await provider.GetMaxUsersNumberAsync();
Assert.AreEqual(1000, result);
#region Validations
twitterUserDalMock.VerifyAll();
#endregion
}
[TestMethod]
public async Task GetMaxUsersNumberAsync_NoWarmUp_Test()
{
#region Stubs
var settings = new InstanceSettings
{
MaxUsersCapacity = 1000
};
#endregion
#region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetTwitterUsersCountAsync())
.ReturnsAsync(249);
#endregion
var provider = new MaxUsersNumberProvider(settings, twitterUserDalMock.Object);
var result = await provider.GetMaxUsersNumberAsync();
Assert.AreEqual(1000, result);
#region Validations
twitterUserDalMock.VerifyAll();
#endregion
}
}
}