From c5fe304694514da86b6a1fa7109fdeeb28a745b9 Mon Sep 17 00:00:00 2001 From: Miss Pasture Date: Sun, 18 Jul 2021 14:56:15 -0400 Subject: [PATCH] Hashflags --- .../Settings/InstanceSettings.cs | 2 + src/BirdsiteLive.Domain/HashflagService.cs | 52 +++++++++++++++++++ .../Tools/StatusExtractor.cs | 25 ++++++++- .../IRetrieveTwitterUsersProcessor.cs | 2 +- .../RetrieveTwitterUsersProcessor.cs | 16 ++++-- .../StatusPublicationPipeline.cs | 2 +- .../Services/FederationService.cs | 8 ++- src/BirdsiteLive/Startup.cs | 6 ++- src/BirdsiteLive/appsettings.json | 3 +- .../RetrieveTwitterUsersProcessorTests.cs | 12 ++--- .../StatusPublicationPipelineTests.cs | 2 +- 11 files changed, 113 insertions(+), 17 deletions(-) create mode 100644 src/BirdsiteLive.Domain/HashflagService.cs diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs index de5a236..c93c817 100644 --- a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs +++ b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs @@ -23,5 +23,7 @@ public bool DiscloseInstanceRestrictions { get; set; } + public bool EnableHashflags { get; set; } + } } \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/HashflagService.cs b/src/BirdsiteLive.Domain/HashflagService.cs new file mode 100644 index 0000000..9ea1286 --- /dev/null +++ b/src/BirdsiteLive.Domain/HashflagService.cs @@ -0,0 +1,52 @@ +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace BirdsiteLive.Domain +{ + public interface IHashflagService + { + Task ExecuteAsync(); + Dictionary Hashflags { get; } + } + + public class HashflagService : IHashflagService + { + private DateTime lastFetch = default(DateTime); + + public Dictionary Hashflags { get; private set; } + + private readonly IHttpClientFactory _httpClientFactory; + private readonly ILogger _logger; + + public HashflagService(IHttpClientFactory httpClientFactory/* , ILogger logger */) + { + _httpClientFactory = httpClientFactory; + /* _logger = logger; */ + } + + public async Task ExecuteAsync() + { + if(DateTime.Now - lastFetch >= new TimeSpan(1, 0, 0)) + { + try + { + var client = _httpClientFactory.CreateClient(); + var result = await client.GetAsync("https://hashflags.blob.core.windows.net/json/activeHashflags"); + var content = await result.Content.ReadAsStringAsync(); + + Hashflags = JsonConvert.DeserializeObject>(content); + } catch(Exception e) + { + Console.WriteLine(e); + /* _logger.LogCritical("Error fetching hashflags: {exception}", e); */ + } + } + } + } +} diff --git a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs index 4e98baf..4ced3b8 100644 --- a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs +++ b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs @@ -18,12 +18,14 @@ namespace BirdsiteLive.Domain.Tools { private readonly InstanceSettings _instanceSettings; private readonly ILogger _logger; + private readonly IHashflagService _hashflagService; #region Ctor - public StatusExtractor(InstanceSettings instanceSettings, ILogger logger) + public StatusExtractor(InstanceSettings instanceSettings, ILogger logger, IHashflagService hashflagService) { _instanceSettings = instanceSettings; _logger = logger; + _hashflagService = hashflagService; } #endregion @@ -82,6 +84,8 @@ namespace BirdsiteLive.Domain.Tools var url = $"https://{_instanceSettings.Domain}/tags/{tag}"; + var flagsInPost = new List(); + if (tags.All(x => x.href != url)) { tags.Add(new Tag @@ -90,10 +94,27 @@ namespace BirdsiteLive.Domain.Tools href = url, type = "Hashtag" }); + + if(_hashflagService.Hashflags.TryGetValue(tag, out string hashflagUrl)) + { + tags.Add(new Tag + { + icon = new TagResource + { + url = hashflagUrl, + type = "Image" + }, + id = hashflagUrl, + name = $":{tag}:", + type = "Emoji" + }); + + flagsInPost.Add(tag); + } } messageContent = Regex.Replace(messageContent, Regex.Escape(m.Groups[0].ToString()), - $@"{m.Groups[1]}#{tag}{m.Groups[3]}"); + $@"{m.Groups[1]}#{tag}{(flagsInPost.IndexOf(tag) > -1 ? $" :{tag}:" : "")}{m.Groups[3]}"); } // Extract Mentions diff --git a/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterUsersProcessor.cs index b71ae93..7ee8f38 100644 --- a/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterUsersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterUsersProcessor.cs @@ -7,6 +7,6 @@ namespace BirdsiteLive.Pipeline.Contracts { public interface IRetrieveTwitterUsersProcessor { - Task GetTwitterUsersAsync(BufferBlock twitterUsersBufferBlock, CancellationToken ct); + Task UpdateTwitterAsync(BufferBlock twitterUsersBufferBlock, CancellationToken ct); } } \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs index ebb87fc..c9e1622 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs @@ -7,6 +7,7 @@ using BirdsiteLive.Common.Extensions; using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; +using BirdsiteLive.Domain; using BirdsiteLive.Pipeline.Contracts; using BirdsiteLive.Pipeline.Tools; using Microsoft.Extensions.Logging; @@ -18,19 +19,23 @@ namespace BirdsiteLive.Pipeline.Processors private readonly ITwitterUserDal _twitterUserDal; private readonly IMaxUsersNumberProvider _maxUsersNumberProvider; private readonly ILogger _logger; - + private readonly IHashflagService _hashflagService; + private readonly InstanceSettings _instanceSettings; + public int WaitFactor = 1000 * 60; //1 min #region Ctor - public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IMaxUsersNumberProvider maxUsersNumberProvider, ILogger logger) + public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IMaxUsersNumberProvider maxUsersNumberProvider, ILogger logger, IHashflagService hashflagService, InstanceSettings instanceSettings) { _twitterUserDal = twitterUserDal; _maxUsersNumberProvider = maxUsersNumberProvider; _logger = logger; + _hashflagService = hashflagService; + _instanceSettings = instanceSettings; } #endregion - public async Task GetTwitterUsersAsync(BufferBlock twitterUsersBufferBlock, CancellationToken ct) + public async Task UpdateTwitterAsync(BufferBlock twitterUsersBufferBlock, CancellationToken ct) { for (; ; ) { @@ -38,6 +43,11 @@ namespace BirdsiteLive.Pipeline.Processors try { + if(_instanceSettings.EnableHashflags) + { + await _hashflagService.ExecuteAsync(); + } + var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync(); var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber); diff --git a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs index d2436f0..57faeb1 100644 --- a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs +++ b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs @@ -58,7 +58,7 @@ namespace BirdsiteLive.Pipeline sendTweetsToFollowersBufferBlock.LinkTo(saveProgressionBlock, new DataflowLinkOptions { PropagateCompletion = true }); // Launch twitter user retriever - var retrieveTwitterAccountsTask = _retrieveTwitterAccountsProcessor.GetTwitterUsersAsync(twitterUsersBufferBlock, ct); + var retrieveTwitterAccountsTask = _retrieveTwitterAccountsProcessor.UpdateTwitterAsync(twitterUsersBufferBlock, ct); // Wait await Task.WhenAny(new[] { retrieveTwitterAccountsTask, saveProgressionBlock.Completion }); diff --git a/src/BirdsiteLive/Services/FederationService.cs b/src/BirdsiteLive/Services/FederationService.cs index 0b0faed..b34c4d3 100644 --- a/src/BirdsiteLive/Services/FederationService.cs +++ b/src/BirdsiteLive/Services/FederationService.cs @@ -2,8 +2,10 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL; using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.Domain; using BirdsiteLive.Moderation; using BirdsiteLive.Pipeline; using Microsoft.Extensions.Hosting; @@ -16,14 +18,18 @@ namespace BirdsiteLive.Services private readonly IModerationPipeline _moderationPipeline; private readonly IStatusPublicationPipeline _statusPublicationPipeline; private readonly IHostApplicationLifetime _applicationLifetime; + private readonly IHashflagService _hashflagService; + private readonly InstanceSettings _instanceSettings; #region Ctor - public FederationService(IDatabaseInitializer databaseInitializer, IModerationPipeline moderationPipeline, IStatusPublicationPipeline statusPublicationPipeline, IHostApplicationLifetime applicationLifetime) + public FederationService(IDatabaseInitializer databaseInitializer, IModerationPipeline moderationPipeline, IStatusPublicationPipeline statusPublicationPipeline, IHostApplicationLifetime applicationLifetime, IHashflagService hashflagService, InstanceSettings instanceSettings) { _databaseInitializer = databaseInitializer; _moderationPipeline = moderationPipeline; _statusPublicationPipeline = statusPublicationPipeline; _applicationLifetime = applicationLifetime; + _hashflagService = hashflagService; + _instanceSettings = instanceSettings; } #endregion diff --git a/src/BirdsiteLive/Startup.cs b/src/BirdsiteLive/Startup.cs index 9758c94..8bfc841 100644 --- a/src/BirdsiteLive/Startup.cs +++ b/src/BirdsiteLive/Startup.cs @@ -8,7 +8,9 @@ using BirdsiteLive.Common.Structs; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Postgres.DataAccessLayers; using BirdsiteLive.DAL.Postgres.Settings; +using BirdsiteLive.Domain; using BirdsiteLive.Models; +using BirdsiteLive.Services; using BirdsiteLive.Twitter; using BirdsiteLive.Twitter.Tools; using Lamar; @@ -55,6 +57,8 @@ namespace BirdsiteLive public void ConfigureContainer(ServiceRegistry services) { + services.For().Use().Singleton(); + var twitterSettings = Configuration.GetSection("Twitter").Get(); services.For().Use(x => twitterSettings); @@ -87,7 +91,7 @@ namespace BirdsiteLive { throw new NotImplementedException($"{dbSettings.Type} is not supported"); } - + services.For().DecorateAllWith(); services.For().Use().Singleton(); diff --git a/src/BirdsiteLive/appsettings.json b/src/BirdsiteLive/appsettings.json index 13d142c..18b60bc 100644 --- a/src/BirdsiteLive/appsettings.json +++ b/src/BirdsiteLive/appsettings.json @@ -27,7 +27,8 @@ "InfoBanner": "", "ShowAboutInstanceOnProfiles": true, "MaxFollowsPerUser": 0, - "DiscloseInstanceRestrictions": false + "DiscloseInstanceRestrictions": false, + "EnableHashflags": false }, "Db": { "Type": "postgres", diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs index 4d0e465..bba391e 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs @@ -48,7 +48,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 10; - var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None); + var t = processor.UpdateTwitterAsync(buffer, CancellationToken.None); await Task.WhenAny(t, Task.Delay(50)); @@ -95,7 +95,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 2; - var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None); + var t = processor.UpdateTwitterAsync(buffer, CancellationToken.None); await Task.WhenAny(t, Task.Delay(300)); @@ -142,7 +142,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 2; - var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None); + var t = processor.UpdateTwitterAsync(buffer, CancellationToken.None); var t2 = Task.Run(async () => { while (buffer.Count < 11) @@ -186,7 +186,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 1; - var t =processor.GetTwitterUsersAsync(buffer, CancellationToken.None); + var t =processor.UpdateTwitterAsync(buffer, CancellationToken.None); await Task.WhenAny(t, Task.Delay(50)); @@ -223,7 +223,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 10; - var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None); + var t = processor.UpdateTwitterAsync(buffer, CancellationToken.None); await Task.WhenAny(t, Task.Delay(50)); @@ -259,7 +259,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 1; - await processor.GetTwitterUsersAsync(buffer, canTokenS.Token); + await processor.UpdateTwitterAsync(buffer, canTokenS.Token); } private static async Task DelayFaultedTask(Exception e) diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/StatusPublicationPipelineTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/StatusPublicationPipelineTests.cs index 2a47b95..bfacb79 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/StatusPublicationPipelineTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/StatusPublicationPipelineTests.cs @@ -22,7 +22,7 @@ namespace BirdsiteLive.Pipeline.Tests #region Mocks var retrieveTwitterUsersProcessor = new Mock(MockBehavior.Strict); retrieveTwitterUsersProcessor - .Setup(x => x.GetTwitterUsersAsync( + .Setup(x => x.UpdateTwitterAsync( It.IsAny>(), It.IsAny())) .Returns(Task.Delay(0));