From 3af2ef05d97b847d2b613ad4e5d6f894cd048285 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 16 Jan 2021 00:34:09 -0500 Subject: [PATCH 1/6] added better logging --- src/BirdsiteLive.Domain/ActivityPubService.cs | 11 +++++----- src/BirdsiteLive.Domain/UserService.cs | 4 ++-- .../BirdsiteLive.Pipeline.csproj | 1 + .../RetrieveTwitterUsersProcessor.cs | 8 ++++--- .../SendTweetsToFollowersProcessor.cs | 12 ++++++----- .../SubTasks/SendTweetsToInboxTask.cs | 8 ++----- .../SubTasks/SendTweetsToSharedInboxTask.cs | 11 +++------- .../StatusPublicationPipeline.cs | 21 ++++++++++--------- 8 files changed, 36 insertions(+), 40 deletions(-) diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs index 3fc17e6..8072075 100644 --- a/src/BirdsiteLive.Domain/ActivityPubService.cs +++ b/src/BirdsiteLive.Domain/ActivityPubService.cs @@ -18,7 +18,7 @@ namespace BirdsiteLive.Domain { Task GetUser(string objectId); Task PostDataAsync(T data, string targetHost, string actorUrl, string inbox = null); - Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, + Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox); } @@ -46,7 +46,7 @@ namespace BirdsiteLive.Domain } } - public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox) + public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox) { var actor = UrlFactory.GetActorUrl(_instanceSettings.Domain, username); var noteUri = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, noteId); @@ -67,7 +67,7 @@ namespace BirdsiteLive.Domain apObject = note }; - return await PostDataAsync(noteActivity, targetHost, actor, targetInbox); + await PostDataAsync(noteActivity, targetHost, actor, targetInbox); } public async Task PostDataAsync(T data, string targetHost, string actorUrl, string inbox = null) @@ -85,7 +85,7 @@ namespace BirdsiteLive.Domain var signature = _cryptoService.SignAndGetSignatureHeader(date, actorUrl, targetHost, digest, usedInbox); - var client = new HttpClient(); + var client = new HttpClient(); //TODO: remove this from here var httpRequestMessage = new HttpRequestMessage { Method = HttpMethod.Post, @@ -101,9 +101,8 @@ namespace BirdsiteLive.Domain }; var response = await client.SendAsync(httpRequestMessage); + response.EnsureSuccessStatusCode(); return response.StatusCode; } - - } } \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index 6131ca9..69cdb87 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -144,7 +144,7 @@ namespace BirdsiteLive.Domain } }; var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject); - return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; + return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; //TODO: revamp this for better error handling } private string OnlyKeepRoute(string inbox, string host) @@ -188,7 +188,7 @@ namespace BirdsiteLive.Domain } }; var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject.apObject); - return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; + return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; //TODO: revamp this for better error handling } private async Task ValidateSignature(string actor, string rawSig, string method, string path, string queryString, Dictionary requestHeaders, string body) diff --git a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj index 6b8b510..009da20 100644 --- a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj +++ b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj @@ -7,6 +7,7 @@ + diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs index adff434..f11b277 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs @@ -5,18 +5,21 @@ using System.Threading.Tasks.Dataflow; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Contracts; +using Microsoft.Extensions.Logging; namespace BirdsiteLive.Pipeline.Processors { public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor { private readonly ITwitterUserDal _twitterUserDal; + private readonly ILogger _logger; private const int SyncPeriod = 15; //in minutes #region Ctor - public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal) + public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, ILogger logger) { _twitterUserDal = twitterUserDal; + _logger = logger; } #endregion @@ -35,8 +38,7 @@ namespace BirdsiteLive.Pipeline.Processors } catch (Exception e) { - Console.WriteLine(e); - //TODO handle error + _logger.LogError(e, "Failing retrieving Twitter Users."); } await Task.Delay(SyncPeriod * 1000 * 60, ct); diff --git a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs index 95fd0c8..afdb00e 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs @@ -13,6 +13,7 @@ using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Pipeline.Processors.SubTasks; using BirdsiteLive.Twitter; using BirdsiteLive.Twitter.Models; +using Microsoft.Extensions.Logging; using Tweetinvi.Models; namespace BirdsiteLive.Pipeline.Processors @@ -21,12 +22,14 @@ namespace BirdsiteLive.Pipeline.Processors { private readonly ISendTweetsToInboxTask _sendTweetsToInboxTask; private readonly ISendTweetsToSharedInboxTask _sendTweetsToSharedInbox; + private readonly ILogger _logger; #region Ctor - public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox) + public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox, ILogger logger) { _sendTweetsToInboxTask = sendTweetsToInboxTask; _sendTweetsToSharedInbox = sendTweetsToSharedInbox; + _logger = logger; } #endregion @@ -61,8 +64,8 @@ namespace BirdsiteLive.Pipeline.Processors } catch (Exception e) { - Console.WriteLine(e); - //TODO handle error + var follower = followersPerInstance.First(); + _logger.LogError(e, "Posting to {Host}{Route} failed", follower.Host, follower.SharedInboxRoute); } } } @@ -77,8 +80,7 @@ namespace BirdsiteLive.Pipeline.Processors } catch (Exception e) { - Console.WriteLine(e); - //TODO handle error + _logger.LogError(e, "Posting to {Host}{Route} failed", follower.Host, follower.InboxRoute); } } } diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs index 77644cf..7e0835d 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs @@ -47,12 +47,8 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks foreach (var tweet in tweetsToSend) { var note = _statusService.GetStatus(user.Acct, tweet); - var result = await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox); - - if (result == HttpStatusCode.Accepted || result == HttpStatusCode.OK) - syncStatus = tweet.Id; - else - throw new Exception("Posting new note activity failed"); + await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox); + syncStatus = tweet.Id; } } finally diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs index 5c6af76..c620910 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs @@ -19,7 +19,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks private readonly IStatusService _statusService; private readonly IActivityPubService _activityPubService; private readonly IFollowersDal _followersDal; - + #region Ctor public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal) { @@ -48,13 +48,8 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks foreach (var tweet in tweetsToSend) { var note = _statusService.GetStatus(user.Acct, tweet); - var result = - await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox); - - if (result == HttpStatusCode.Accepted || result == HttpStatusCode.OK) - syncStatus = tweet.Id; - else - throw new Exception("Posting new note activity failed"); + await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox); + syncStatus = tweet.Id; } } finally diff --git a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs index bcb896f..8de272e 100644 --- a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs +++ b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks.Dataflow; using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Contracts; using BirdsiteLive.Pipeline.Models; +using Microsoft.Extensions.Logging; namespace BirdsiteLive.Pipeline { @@ -19,29 +20,31 @@ namespace BirdsiteLive.Pipeline private readonly IRetrieveTweetsProcessor _retrieveTweetsProcessor; private readonly IRetrieveFollowersProcessor _retrieveFollowersProcessor; private readonly ISendTweetsToFollowersProcessor _sendTweetsToFollowersProcessor; - + private readonly ILogger _logger; + #region Ctor - public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor) + public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ILogger logger) { _retrieveTweetsProcessor = retrieveTweetsProcessor; _retrieveTwitterAccountsProcessor = retrieveTwitterAccountsProcessor; _retrieveFollowersProcessor = retrieveFollowersProcessor; _sendTweetsToFollowersProcessor = sendTweetsToFollowersProcessor; + _logger = logger; } #endregion public async Task ExecuteAsync(CancellationToken ct) { // Create blocks - var twitterUsersBufferBlock = new BufferBlock(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct}); + var twitterUsersBufferBlock = new BufferBlock(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct }); var retrieveTweetsBlock = new TransformBlock(async x => await _retrieveTweetsProcessor.ProcessAsync(x, ct)); var retrieveTweetsBufferBlock = new BufferBlock(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct }); var retrieveFollowersBlock = new TransformManyBlock(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct)); var retrieveFollowersBufferBlock = new BufferBlock(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct }); - var sendTweetsToFollowersBlock = new ActionBlock(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct}); + var sendTweetsToFollowersBlock = new ActionBlock(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct }); // Link pipeline - twitterUsersBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions {PropagateCompletion = true}); + twitterUsersBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions { PropagateCompletion = true }); retrieveTweetsBlock.LinkTo(retrieveTweetsBufferBlock, new DataflowLinkOptions { PropagateCompletion = true }); retrieveTweetsBufferBlock.LinkTo(retrieveFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true }); retrieveFollowersBlock.LinkTo(retrieveFollowersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true }); @@ -51,12 +54,10 @@ namespace BirdsiteLive.Pipeline var retrieveTwitterAccountsTask = _retrieveTwitterAccountsProcessor.GetTwitterUsersAsync(twitterUsersBufferBlock, ct); // Wait - await Task.WhenAny(new []{ retrieveTwitterAccountsTask , sendTweetsToFollowersBlock.Completion}); + await Task.WhenAny(new[] { retrieveTwitterAccountsTask, sendTweetsToFollowersBlock.Completion }); - var foreground = Console.ForegroundColor; - Console.ForegroundColor = ConsoleColor.Red; - Console.WriteLine("An error occured, pipeline stopped"); - Console.ForegroundColor = foreground; + var ex = retrieveTwitterAccountsTask.IsFaulted ? retrieveTwitterAccountsTask.Exception : sendTweetsToFollowersBlock.Completion.Exception; + _logger.LogCritical(ex, "An error occurred, pipeline stopped"); } } } From a25cb31a620d162fc2eb92d65cf2fd63729e6147 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 16 Jan 2021 00:51:52 -0500 Subject: [PATCH 2/6] fix httpclient candid implementation --- src/BirdsiteLive.Cryptography/MagicKey.cs | 19 ------------------ src/BirdsiteLive.Domain/ActivityPubService.cs | 20 +++++++++---------- .../BirdsiteLive.Domain.csproj | 4 ++++ 3 files changed, 14 insertions(+), 29 deletions(-) diff --git a/src/BirdsiteLive.Cryptography/MagicKey.cs b/src/BirdsiteLive.Cryptography/MagicKey.cs index 24f7af2..23d478a 100644 --- a/src/BirdsiteLive.Cryptography/MagicKey.cs +++ b/src/BirdsiteLive.Cryptography/MagicKey.cs @@ -105,25 +105,6 @@ namespace BirdsiteLive.Cryptography return new MagicKey(JsonConvert.SerializeObject(RSAKeyParms.From(rsa.ExportParameters(true)))); } - //public static async Task KeyForAuthor(ASObject obj) - //{ - // var authorId = (string)obj["email"].FirstOrDefault()?.Primitive; - // if (authorId == null) - // { - // authorId = obj["name"].FirstOrDefault()?.Primitive + "@" + new Uri(obj.Id).Host; - // } - - // var domain = authorId.Split('@')[1]; - // var hc = new HttpClient(); - // var wf = JsonConvert.DeserializeObject(await hc.GetStringAsync($"https://{domain}/.well-known/webfinger?resource=acct:{Uri.EscapeDataString(authorId)}")); - // var link = wf.links.FirstOrDefault(a => a.rel == "magic-public-key"); - // if (link == null) return null; - - // if (!link.href.StartsWith("data:")) return null; // does this happen? - - // return new MagicKey(link.href.Split(new char[] { ',' }, 2)[1]); - //} - public byte[] BuildSignedData(string data, string dataType, string encoding, string algorithm) { var sig = data + "." + _encodeBase64Url(Encoding.UTF8.GetBytes(dataType)) + "." + _encodeBase64Url(Encoding.UTF8.GetBytes(encoding)) + "." + _encodeBase64Url(Encoding.UTF8.GetBytes(algorithm)); diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs index 8072075..4d8fc54 100644 --- a/src/BirdsiteLive.Domain/ActivityPubService.cs +++ b/src/BirdsiteLive.Domain/ActivityPubService.cs @@ -25,25 +25,25 @@ namespace BirdsiteLive.Domain public class ActivityPubService : IActivityPubService { private readonly InstanceSettings _instanceSettings; + private readonly IHttpClientFactory _httpClientFactory; private readonly ICryptoService _cryptoService; #region Ctor - public ActivityPubService(ICryptoService cryptoService, InstanceSettings instanceSettings) + public ActivityPubService(ICryptoService cryptoService, InstanceSettings instanceSettings, IHttpClientFactory httpClientFactory) { _cryptoService = cryptoService; _instanceSettings = instanceSettings; + _httpClientFactory = httpClientFactory; } #endregion public async Task GetUser(string objectId) { - using (var httpClient = new HttpClient()) - { - httpClient.DefaultRequestHeaders.Add("Accept", "application/json"); - var result = await httpClient.GetAsync(objectId); - var content = await result.Content.ReadAsStringAsync(); - return JsonConvert.DeserializeObject(content); - } + var httpClient = _httpClientFactory.CreateClient(); + httpClient.DefaultRequestHeaders.Add("Accept", "application/json"); + var result = await httpClient.GetAsync(objectId); + var content = await result.Content.ReadAsStringAsync(); + return JsonConvert.DeserializeObject(content); } public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox) @@ -53,7 +53,7 @@ namespace BirdsiteLive.Domain var now = DateTime.UtcNow; var nowString = now.ToString("s") + "Z"; - + var noteActivity = new ActivityCreateNote() { context = "https://www.w3.org/ns/activitystreams", @@ -85,7 +85,7 @@ namespace BirdsiteLive.Domain var signature = _cryptoService.SignAndGetSignatureHeader(date, actorUrl, targetHost, digest, usedInbox); - var client = new HttpClient(); //TODO: remove this from here + var client = _httpClientFactory.CreateClient(); var httpRequestMessage = new HttpRequestMessage { Method = HttpMethod.Post, diff --git a/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj b/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj index cb89578..8c601b4 100644 --- a/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj +++ b/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj @@ -4,6 +4,10 @@ netstandard2.0 + + + + From 12875b2c56af6e85b4cf2b9f3b63c9467f3770f3 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 16 Jan 2021 01:18:03 -0500 Subject: [PATCH 3/6] fix dependancies --- src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj | 2 +- src/BirdsiteLive/BirdsiteLive.csproj | 2 +- src/BirdsiteLive/Startup.cs | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj index 009da20..5d93cb1 100644 --- a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj +++ b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj @@ -6,7 +6,7 @@ - + diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index 041c8b3..aef5d85 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -8,7 +8,7 @@ - + diff --git a/src/BirdsiteLive/Startup.cs b/src/BirdsiteLive/Startup.cs index a6806b9..ee4b079 100644 --- a/src/BirdsiteLive/Startup.cs +++ b/src/BirdsiteLive/Startup.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net.Http; using System.Threading.Tasks; using BirdsiteLive.Common.Settings; using BirdsiteLive.Common.Structs; @@ -46,6 +47,8 @@ namespace BirdsiteLive } services.AddControllersWithViews(); + + services.AddHttpClient(); } public void ConfigureContainer(ServiceRegistry services) From 568c033f9c53fa57724ca7db411cb1a49a298b5f Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 16 Jan 2021 02:06:27 -0500 Subject: [PATCH 4/6] fix tests --- .../RetrieveTwitterUsersProcessorTests.cs | 16 +++++++++--- .../SendTweetsToFollowersProcessorTests.cs | 25 ++++++++++++++----- .../SubTasks/SendTweetsToInboxTaskTests.cs | 11 ++++---- .../SubTasks/SendTweetsToSharedInboxTests.cs | 11 ++++---- 4 files changed, 43 insertions(+), 20 deletions(-) diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs index b7a2e2b..356a479 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks.Dataflow; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Processors; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -31,9 +32,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors twitterUserDalMock .Setup(x => x.GetAllTwitterUsersAsync()) .ReturnsAsync(users); + + var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); processor.GetTwitterUsersAsync(buffer, CancellationToken.None); await Task.Delay(50); @@ -58,9 +61,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors twitterUserDalMock .Setup(x => x.GetAllTwitterUsersAsync()) .ReturnsAsync(new SyncTwitterUser[0]); + + var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); processor.GetTwitterUsersAsync(buffer, CancellationToken.None); await Task.Delay(50); @@ -84,9 +89,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors twitterUserDalMock .Setup(x => x.GetAllTwitterUsersAsync()) .Throws(new Exception()); + + var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None); await Task.WhenAny(t, Task.Delay(50)); @@ -110,9 +117,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); + var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); await processor.GetTwitterUsersAsync(buffer, canTokenS.Token); } } diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs index 034d8d2..ad35c3e 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs @@ -6,6 +6,7 @@ using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Pipeline.Processors; using BirdsiteLive.Pipeline.Processors.SubTasks; using BirdsiteLive.Twitter.Models; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -67,9 +68,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == host), It.Is(y => y.Length == 2))) .Returns(Task.CompletedTask); + + var loggerMock = new Mock>(); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, loggerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations @@ -135,9 +138,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y.Length == 1))) .Returns(Task.CompletedTask); } + + var loggerMock = new Mock>(); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, loggerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations @@ -208,9 +213,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == host2), It.Is(y => y.Length == 1))) .Throws(new Exception()); + + var loggerMock = new Mock>(); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, loggerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations @@ -274,9 +281,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors } var sendTweetsToSharedInboxTaskMock = new Mock(MockBehavior.Strict); + + var loggerMock = new Mock>(); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, loggerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations @@ -341,9 +350,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors } var sendTweetsToSharedInboxTaskMock = new Mock(MockBehavior.Strict); + + var loggerMock = new Mock>(); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, loggerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations @@ -412,9 +423,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Throws(new Exception()); var sendTweetsToSharedInboxTaskMock = new Mock(MockBehavior.Strict); + + var loggerMock = new Mock>(); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, loggerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs index 0596162..36688f6 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Net; +using System.Net.Http; using System.Threading.Tasks; using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.DAL.Contracts; @@ -63,7 +64,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks It.Is(y => y == tweetId.ToString()), It.Is(y => y == host), It.Is(y => y == inbox))) - .ReturnsAsync(HttpStatusCode.Accepted); + .Returns(Task.CompletedTask); var statusServiceMock = new Mock(MockBehavior.Strict); statusServiceMock @@ -136,7 +137,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks It.Is(y => y == tweetId.ToString()), It.Is(y => y == host), It.Is(y => y == inbox))) - .ReturnsAsync(HttpStatusCode.Accepted); + .Returns(Task.CompletedTask); } var statusServiceMock = new Mock(MockBehavior.Strict); @@ -168,7 +169,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks } [TestMethod] - [ExpectedException(typeof(Exception))] + [ExpectedException(typeof(HttpRequestException))] public async Task ExecuteAsync_MultipleTweets_Error_Test() { #region Stubs @@ -213,7 +214,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks It.Is(y => y == tweetId2.ToString()), It.Is(y => y == host), It.Is(y => y == inbox))) - .ReturnsAsync(HttpStatusCode.Accepted); + .Returns(Task.CompletedTask); activityPubService .Setup(x => x.PostNewNoteActivity( @@ -222,7 +223,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks It.Is(y => y == tweetId3.ToString()), It.Is(y => y == host), It.Is(y => y == inbox))) - .ReturnsAsync(HttpStatusCode.InternalServerError); + .Throws(new HttpRequestException()); var statusServiceMock = new Mock(MockBehavior.Strict); foreach (var tweetId in new[] { tweetId2, tweetId3 }) diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs index a052a5c..1909108 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using System.Net; +using System.Net.Http; using System.Threading.Tasks; using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.DAL.Contracts; @@ -81,7 +82,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks It.Is(y => y == tweetId.ToString()), It.Is(y => y == host), It.Is(y => y == inbox))) - .ReturnsAsync(HttpStatusCode.Accepted); + .Returns(Task.CompletedTask); var statusServiceMock = new Mock(MockBehavior.Strict); statusServiceMock @@ -174,7 +175,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks It.Is(y => y == tweetId.ToString()), It.Is(y => y == host), It.Is(y => y == inbox))) - .ReturnsAsync(HttpStatusCode.Accepted); + .Returns(Task.CompletedTask); } var statusServiceMock = new Mock(MockBehavior.Strict); @@ -209,7 +210,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks } [TestMethod] - [ExpectedException(typeof(Exception))] + [ExpectedException(typeof(HttpRequestException))] public async Task ExecuteAsync_MultipleTweets_Error_Test() { #region Stubs @@ -271,7 +272,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks It.Is(y => y == tweetId2.ToString()), It.Is(y => y == host), It.Is(y => y == inbox))) - .ReturnsAsync(HttpStatusCode.Accepted); + .Returns(Task.CompletedTask); activityPubService .Setup(x => x.PostNewNoteActivity( @@ -280,7 +281,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks It.Is(y => y == tweetId3.ToString()), It.Is(y => y == host), It.Is(y => y == inbox))) - .ReturnsAsync(HttpStatusCode.InternalServerError); + .Throws(new HttpRequestException()); var statusServiceMock = new Mock(MockBehavior.Strict); foreach (var tweetId in new[] { tweetId2, tweetId3 }) From 22df5d23566836f5dff9d0688e2af1a059b792a8 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sun, 17 Jan 2021 23:05:00 -0500 Subject: [PATCH 5/6] added user caching --- .../BirdsiteLive.Twitter.csproj | 1 + .../CachedTwitterService.cs | 52 +++++++++++++++++++ src/BirdsiteLive/Startup.cs | 8 ++- 3 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 src/BirdsiteLive.Twitter/CachedTwitterService.cs diff --git a/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj b/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj index 8b73ab8..377d04d 100644 --- a/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj +++ b/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj @@ -5,6 +5,7 @@ + diff --git a/src/BirdsiteLive.Twitter/CachedTwitterService.cs b/src/BirdsiteLive.Twitter/CachedTwitterService.cs new file mode 100644 index 0000000..a77fb12 --- /dev/null +++ b/src/BirdsiteLive.Twitter/CachedTwitterService.cs @@ -0,0 +1,52 @@ +using System; +using BirdsiteLive.Twitter.Models; +using Microsoft.Extensions.Caching.Memory; + +namespace BirdsiteLive.Twitter +{ + public class CachedTwitterService : ITwitterService + { + private readonly ITwitterService _twitterService; + + private MemoryCache _userCache = new MemoryCache(new MemoryCacheOptions() + { + SizeLimit = 5000 + }); + private MemoryCacheEntryOptions _cacheEntryOptions = new MemoryCacheEntryOptions() + .SetSize(1)//Size amount + //Priority on removing when reaching size limit (memory pressure) + .SetPriority(CacheItemPriority.High) + // Keep in cache for this time, reset time if accessed. + .SetSlidingExpiration(TimeSpan.FromHours(24)) + // Remove from cache after this time, regardless of sliding expiration + .SetAbsoluteExpiration(TimeSpan.FromDays(30)); + + #region Ctor + public CachedTwitterService(ITwitterService twitterService) + { + _twitterService = twitterService; + } + #endregion + + public TwitterUser GetUser(string username) + { + if (!_userCache.TryGetValue(username, out TwitterUser user)) + { + user = _twitterService.GetUser(username); + _userCache.Set(username, user, _cacheEntryOptions); + } + + return user; + } + + public ExtractedTweet GetTweet(long statusId) + { + return _twitterService.GetTweet(statusId); + } + + public ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1) + { + return _twitterService.GetTimeline(username, nberTweets, fromTweetId); + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Startup.cs b/src/BirdsiteLive/Startup.cs index ee4b079..9c37a54 100644 --- a/src/BirdsiteLive/Startup.cs +++ b/src/BirdsiteLive/Startup.cs @@ -9,6 +9,7 @@ using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Postgres.DataAccessLayers; using BirdsiteLive.DAL.Postgres.Settings; using BirdsiteLive.Models; +using BirdsiteLive.Twitter; using Lamar; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; @@ -64,7 +65,7 @@ namespace BirdsiteLive var logsSettings = Configuration.GetSection("Logging").Get(); services.For().Use(x => logsSettings); - + if (string.Equals(dbSettings.Type, DbTypes.Postgres, StringComparison.OrdinalIgnoreCase)) { var connString = $"Host={dbSettings.Host};Username={dbSettings.User};Password={dbSettings.Password};Database={dbSettings.Name}"; @@ -73,7 +74,7 @@ namespace BirdsiteLive ConnString = connString }; services.For().Use(x => postgresSettings); - + services.For().Use().Singleton(); services.For().Use().Singleton(); services.For().Use().Singleton(); @@ -82,6 +83,9 @@ namespace BirdsiteLive { throw new NotImplementedException($"{dbSettings.Type} is not supported"); } + + services.For().DecorateAllWith(); + services.For().Use().Singleton(); services.Scan(_ => { From b66e7cd85463ce44a2cf3a759ac850bc0c7305b1 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sun, 17 Jan 2021 23:06:33 -0500 Subject: [PATCH 6/6] road to 0.9.0 --- src/BirdsiteLive/BirdsiteLive.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index aef5d85..9cfe9db 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -4,7 +4,7 @@ netcoreapp3.1 d21486de-a812-47eb-a419-05682bb68856 Linux - 0.8.1 + 0.9.0