From 3af2ef05d97b847d2b613ad4e5d6f894cd048285 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 16 Jan 2021 00:34:09 -0500 Subject: [PATCH] added better logging --- src/BirdsiteLive.Domain/ActivityPubService.cs | 11 +++++----- src/BirdsiteLive.Domain/UserService.cs | 4 ++-- .../BirdsiteLive.Pipeline.csproj | 1 + .../RetrieveTwitterUsersProcessor.cs | 8 ++++--- .../SendTweetsToFollowersProcessor.cs | 12 ++++++----- .../SubTasks/SendTweetsToInboxTask.cs | 8 ++----- .../SubTasks/SendTweetsToSharedInboxTask.cs | 11 +++------- .../StatusPublicationPipeline.cs | 21 ++++++++++--------- 8 files changed, 36 insertions(+), 40 deletions(-) diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs index 3fc17e6..8072075 100644 --- a/src/BirdsiteLive.Domain/ActivityPubService.cs +++ b/src/BirdsiteLive.Domain/ActivityPubService.cs @@ -18,7 +18,7 @@ namespace BirdsiteLive.Domain { Task GetUser(string objectId); Task PostDataAsync(T data, string targetHost, string actorUrl, string inbox = null); - Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, + Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox); } @@ -46,7 +46,7 @@ namespace BirdsiteLive.Domain } } - public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox) + public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox) { var actor = UrlFactory.GetActorUrl(_instanceSettings.Domain, username); var noteUri = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, noteId); @@ -67,7 +67,7 @@ namespace BirdsiteLive.Domain apObject = note }; - return await PostDataAsync(noteActivity, targetHost, actor, targetInbox); + await PostDataAsync(noteActivity, targetHost, actor, targetInbox); } public async Task PostDataAsync(T data, string targetHost, string actorUrl, string inbox = null) @@ -85,7 +85,7 @@ namespace BirdsiteLive.Domain var signature = _cryptoService.SignAndGetSignatureHeader(date, actorUrl, targetHost, digest, usedInbox); - var client = new HttpClient(); + var client = new HttpClient(); //TODO: remove this from here var httpRequestMessage = new HttpRequestMessage { Method = HttpMethod.Post, @@ -101,9 +101,8 @@ namespace BirdsiteLive.Domain }; var response = await client.SendAsync(httpRequestMessage); + response.EnsureSuccessStatusCode(); return response.StatusCode; } - - } } \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index 6131ca9..69cdb87 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -144,7 +144,7 @@ namespace BirdsiteLive.Domain } }; var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject); - return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; + return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; //TODO: revamp this for better error handling } private string OnlyKeepRoute(string inbox, string host) @@ -188,7 +188,7 @@ namespace BirdsiteLive.Domain } }; var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject.apObject); - return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; + return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; //TODO: revamp this for better error handling } private async Task ValidateSignature(string actor, string rawSig, string method, string path, string queryString, Dictionary requestHeaders, string body) diff --git a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj index 6b8b510..009da20 100644 --- a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj +++ b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj @@ -7,6 +7,7 @@ + diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs index adff434..f11b277 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs @@ -5,18 +5,21 @@ using System.Threading.Tasks.Dataflow; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Contracts; +using Microsoft.Extensions.Logging; namespace BirdsiteLive.Pipeline.Processors { public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor { private readonly ITwitterUserDal _twitterUserDal; + private readonly ILogger _logger; private const int SyncPeriod = 15; //in minutes #region Ctor - public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal) + public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, ILogger logger) { _twitterUserDal = twitterUserDal; + _logger = logger; } #endregion @@ -35,8 +38,7 @@ namespace BirdsiteLive.Pipeline.Processors } catch (Exception e) { - Console.WriteLine(e); - //TODO handle error + _logger.LogError(e, "Failing retrieving Twitter Users."); } await Task.Delay(SyncPeriod * 1000 * 60, ct); diff --git a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs index 95fd0c8..afdb00e 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs @@ -13,6 +13,7 @@ using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Pipeline.Processors.SubTasks; using BirdsiteLive.Twitter; using BirdsiteLive.Twitter.Models; +using Microsoft.Extensions.Logging; using Tweetinvi.Models; namespace BirdsiteLive.Pipeline.Processors @@ -21,12 +22,14 @@ namespace BirdsiteLive.Pipeline.Processors { private readonly ISendTweetsToInboxTask _sendTweetsToInboxTask; private readonly ISendTweetsToSharedInboxTask _sendTweetsToSharedInbox; + private readonly ILogger _logger; #region Ctor - public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox) + public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox, ILogger logger) { _sendTweetsToInboxTask = sendTweetsToInboxTask; _sendTweetsToSharedInbox = sendTweetsToSharedInbox; + _logger = logger; } #endregion @@ -61,8 +64,8 @@ namespace BirdsiteLive.Pipeline.Processors } catch (Exception e) { - Console.WriteLine(e); - //TODO handle error + var follower = followersPerInstance.First(); + _logger.LogError(e, "Posting to {Host}{Route} failed", follower.Host, follower.SharedInboxRoute); } } } @@ -77,8 +80,7 @@ namespace BirdsiteLive.Pipeline.Processors } catch (Exception e) { - Console.WriteLine(e); - //TODO handle error + _logger.LogError(e, "Posting to {Host}{Route} failed", follower.Host, follower.InboxRoute); } } } diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs index 77644cf..7e0835d 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs @@ -47,12 +47,8 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks foreach (var tweet in tweetsToSend) { var note = _statusService.GetStatus(user.Acct, tweet); - var result = await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox); - - if (result == HttpStatusCode.Accepted || result == HttpStatusCode.OK) - syncStatus = tweet.Id; - else - throw new Exception("Posting new note activity failed"); + await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox); + syncStatus = tweet.Id; } } finally diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs index 5c6af76..c620910 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs @@ -19,7 +19,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks private readonly IStatusService _statusService; private readonly IActivityPubService _activityPubService; private readonly IFollowersDal _followersDal; - + #region Ctor public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal) { @@ -48,13 +48,8 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks foreach (var tweet in tweetsToSend) { var note = _statusService.GetStatus(user.Acct, tweet); - var result = - await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox); - - if (result == HttpStatusCode.Accepted || result == HttpStatusCode.OK) - syncStatus = tweet.Id; - else - throw new Exception("Posting new note activity failed"); + await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox); + syncStatus = tweet.Id; } } finally diff --git a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs index bcb896f..8de272e 100644 --- a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs +++ b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks.Dataflow; using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Contracts; using BirdsiteLive.Pipeline.Models; +using Microsoft.Extensions.Logging; namespace BirdsiteLive.Pipeline { @@ -19,29 +20,31 @@ namespace BirdsiteLive.Pipeline private readonly IRetrieveTweetsProcessor _retrieveTweetsProcessor; private readonly IRetrieveFollowersProcessor _retrieveFollowersProcessor; private readonly ISendTweetsToFollowersProcessor _sendTweetsToFollowersProcessor; - + private readonly ILogger _logger; + #region Ctor - public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor) + public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ILogger logger) { _retrieveTweetsProcessor = retrieveTweetsProcessor; _retrieveTwitterAccountsProcessor = retrieveTwitterAccountsProcessor; _retrieveFollowersProcessor = retrieveFollowersProcessor; _sendTweetsToFollowersProcessor = sendTweetsToFollowersProcessor; + _logger = logger; } #endregion public async Task ExecuteAsync(CancellationToken ct) { // Create blocks - var twitterUsersBufferBlock = new BufferBlock(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct}); + var twitterUsersBufferBlock = new BufferBlock(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct }); var retrieveTweetsBlock = new TransformBlock(async x => await _retrieveTweetsProcessor.ProcessAsync(x, ct)); var retrieveTweetsBufferBlock = new BufferBlock(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct }); var retrieveFollowersBlock = new TransformManyBlock(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct)); var retrieveFollowersBufferBlock = new BufferBlock(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct }); - var sendTweetsToFollowersBlock = new ActionBlock(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct}); + var sendTweetsToFollowersBlock = new ActionBlock(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct }); // Link pipeline - twitterUsersBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions {PropagateCompletion = true}); + twitterUsersBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions { PropagateCompletion = true }); retrieveTweetsBlock.LinkTo(retrieveTweetsBufferBlock, new DataflowLinkOptions { PropagateCompletion = true }); retrieveTweetsBufferBlock.LinkTo(retrieveFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true }); retrieveFollowersBlock.LinkTo(retrieveFollowersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true }); @@ -51,12 +54,10 @@ namespace BirdsiteLive.Pipeline var retrieveTwitterAccountsTask = _retrieveTwitterAccountsProcessor.GetTwitterUsersAsync(twitterUsersBufferBlock, ct); // Wait - await Task.WhenAny(new []{ retrieveTwitterAccountsTask , sendTweetsToFollowersBlock.Completion}); + await Task.WhenAny(new[] { retrieveTwitterAccountsTask, sendTweetsToFollowersBlock.Completion }); - var foreground = Console.ForegroundColor; - Console.ForegroundColor = ConsoleColor.Red; - Console.WriteLine("An error occured, pipeline stopped"); - Console.ForegroundColor = foreground; + var ex = retrieveTwitterAccountsTask.IsFaulted ? retrieveTwitterAccountsTask.Exception : sendTweetsToFollowersBlock.Completion.Exception; + _logger.LogCritical(ex, "An error occurred, pipeline stopped"); } } }