sam/BirdsiteLIVE
sam/BirdsiteLIVE
Archived
1
0
Fork 0
Code Issues 1 Pull requests 1 Projects Releases Packages 1 Wiki Activity

Hashflags

Browse source
This commit is contained in:
Miss Pasture 2021-07-18 14:56:15 -04:00
parent 6e9a3bc100
commit c5fe304694
11 changed files with 113 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/BirdsiteLive.Common/Settings/InstanceSettings.cs
View file

@ -23,5 +23,7 @@
public bool DiscloseInstanceRestrictions { get; set; }
public bool EnableHashflags { get; set; }
}
}

52
src/BirdsiteLive.Domain/HashflagService.cs Normal file
View file

@ -0,0 +1,52 @@
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace BirdsiteLive.Domain
{
public interface IHashflagService
{
Task ExecuteAsync();
Dictionary<string, string> Hashflags { get; }
}
public class HashflagService : IHashflagService
{
private DateTime lastFetch = default(DateTime);
public Dictionary<string, string> Hashflags { get; private set; }
private readonly IHttpClientFactory _httpClientFactory;
private readonly ILogger _logger;
public HashflagService(IHttpClientFactory httpClientFactory/* , ILogger logger */)
{
_httpClientFactory = httpClientFactory;
/* _logger = logger; */
}
public async Task ExecuteAsync()
{
if(DateTime.Now - lastFetch >= new TimeSpan(1, 0, 0))
{
try
{
var client = _httpClientFactory.CreateClient();
var result = await client.GetAsync("https://hashflags.blob.core.windows.net/json/activeHashflags");
var content = await result.Content.ReadAsStringAsync();
Hashflags = JsonConvert.DeserializeObject<Dictionary<string, string>>(content);
} catch(Exception e)
{
Console.WriteLine(e);
/* _logger.LogCritical("Error fetching hashflags: {exception}", e); */
}
}
}
}
}

25
src/BirdsiteLive.Domain/Tools/StatusExtractor.cs
View file

@ -18,12 +18,14 @@ namespace BirdsiteLive.Domain.Tools
{
private readonly InstanceSettings _instanceSettings;
private readonly ILogger<StatusExtractor> _logger;
private readonly IHashflagService _hashflagService;
#region Ctor
public StatusExtractor(InstanceSettings instanceSettings, ILogger<StatusExtractor> logger)
public StatusExtractor(InstanceSettings instanceSettings, ILogger<StatusExtractor> logger, IHashflagService hashflagService)
{
_instanceSettings = instanceSettings;
_logger = logger;
_hashflagService = hashflagService;
}
#endregion
@ -82,6 +84,8 @@ namespace BirdsiteLive.Domain.Tools
var url = $"https://{_instanceSettings.Domain}/tags/{tag}";
var flagsInPost = new List<string>();
if (tags.All(x => x.href != url))
{
tags.Add(new Tag
@ -90,10 +94,27 @@ namespace BirdsiteLive.Domain.Tools
href = url,
type = "Hashtag"
});
if(_hashflagService.Hashflags.TryGetValue(tag, out string hashflagUrl))
{
tags.Add(new Tag
{
icon = new TagResource
{
url = hashflagUrl,
type = "Image"
},
id = hashflagUrl,
name = $":{tag}:",
type = "Emoji"
});
flagsInPost.Add(tag);
}
}
messageContent = Regex.Replace(messageContent, Regex.Escape(m.Groups[0].ToString()),
$@"{m.Groups[1]}<a href=""{url}"" class=""mention hashtag"" rel=""tag"">#<span>{tag}</span></a>{m.Groups[3]}");
$@"{m.Groups[1]}<a href=""{url}"" class=""mention hashtag"" rel=""tag"">#<span>{tag}</span></a>{(flagsInPost.IndexOf(tag) > -1 ? $" :{tag}:" : "")}{m.Groups[3]}");
}
// Extract Mentions

2
src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterUsersProcessor.cs
View file

@ -7,6 +7,6 @@ namespace BirdsiteLive.Pipeline.Contracts
{
public interface IRetrieveTwitterUsersProcessor
{
Task GetTwitterUsersAsync(BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock, CancellationToken ct);
Task UpdateTwitterAsync(BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock, CancellationToken ct);
}
}

16
src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
View file

@ -7,6 +7,7 @@ using BirdsiteLive.Common.Extensions;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Domain;
using BirdsiteLive.Pipeline.Contracts;
using BirdsiteLive.Pipeline.Tools;
using Microsoft.Extensions.Logging;
@ -18,19 +19,23 @@ namespace BirdsiteLive.Pipeline.Processors
private readonly ITwitterUserDal _twitterUserDal;
private readonly IMaxUsersNumberProvider _maxUsersNumberProvider;
private readonly ILogger<RetrieveTwitterUsersProcessor> _logger;
private readonly IHashflagService _hashflagService;
private readonly InstanceSettings _instanceSettings;
public int WaitFactor = 1000 * 60; //1 min
#region Ctor
public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IMaxUsersNumberProvider maxUsersNumberProvider, ILogger<RetrieveTwitterUsersProcessor> logger)
public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IMaxUsersNumberProvider maxUsersNumberProvider, ILogger<RetrieveTwitterUsersProcessor> logger, IHashflagService hashflagService, InstanceSettings instanceSettings)
{
_twitterUserDal = twitterUserDal;
_maxUsersNumberProvider = maxUsersNumberProvider;
_logger = logger;
_hashflagService = hashflagService;
_instanceSettings = instanceSettings;
}
#endregion
public async Task GetTwitterUsersAsync(BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock, CancellationToken ct)
public async Task UpdateTwitterAsync(BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock, CancellationToken ct)
{
for (; ; )
{
@ -38,6 +43,11 @@ namespace BirdsiteLive.Pipeline.Processors
try
{
if(_instanceSettings.EnableHashflags)
{
await _hashflagService.ExecuteAsync();
}
var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync();
var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber);

2
src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs
View file

@ -58,7 +58,7 @@ namespace BirdsiteLive.Pipeline
sendTweetsToFollowersBufferBlock.LinkTo(saveProgressionBlock, new DataflowLinkOptions { PropagateCompletion = true });
// Launch twitter user retriever
var retrieveTwitterAccountsTask = _retrieveTwitterAccountsProcessor.GetTwitterUsersAsync(twitterUsersBufferBlock, ct);
var retrieveTwitterAccountsTask = _retrieveTwitterAccountsProcessor.UpdateTwitterAsync(twitterUsersBufferBlock, ct);
// Wait
await Task.WhenAny(new[] { retrieveTwitterAccountsTask, saveProgressionBlock.Completion });

8
src/BirdsiteLive/Services/FederationService.cs
View file

@ -2,8 +2,10 @@
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.Domain;
using BirdsiteLive.Moderation;
using BirdsiteLive.Pipeline;
using Microsoft.Extensions.Hosting;
@ -16,14 +18,18 @@ namespace BirdsiteLive.Services
private readonly IModerationPipeline _moderationPipeline;
private readonly IStatusPublicationPipeline _statusPublicationPipeline;
private readonly IHostApplicationLifetime _applicationLifetime;
private readonly IHashflagService _hashflagService;
private readonly InstanceSettings _instanceSettings;
#region Ctor
public FederationService(IDatabaseInitializer databaseInitializer, IModerationPipeline moderationPipeline, IStatusPublicationPipeline statusPublicationPipeline, IHostApplicationLifetime applicationLifetime)
public FederationService(IDatabaseInitializer databaseInitializer, IModerationPipeline moderationPipeline, IStatusPublicationPipeline statusPublicationPipeline, IHostApplicationLifetime applicationLifetime, IHashflagService hashflagService, InstanceSettings instanceSettings)
{
_databaseInitializer = databaseInitializer;
_moderationPipeline = moderationPipeline;
_statusPublicationPipeline = statusPublicationPipeline;
_applicationLifetime = applicationLifetime;
_hashflagService = hashflagService;
_instanceSettings = instanceSettings;
}
#endregion

6
src/BirdsiteLive/Startup.cs
View file

@ -8,7 +8,9 @@ using BirdsiteLive.Common.Structs;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Postgres.DataAccessLayers;
using BirdsiteLive.DAL.Postgres.Settings;
using BirdsiteLive.Domain;
using BirdsiteLive.Models;
using BirdsiteLive.Services;
using BirdsiteLive.Twitter;
using BirdsiteLive.Twitter.Tools;
using Lamar;
@ -55,6 +57,8 @@ namespace BirdsiteLive
public void ConfigureContainer(ServiceRegistry services)
{
services.For<IHashflagService>().Use<HashflagService>().Singleton();
var twitterSettings = Configuration.GetSection("Twitter").Get<TwitterSettings>();
services.For<TwitterSettings>().Use(x => twitterSettings);
@ -87,7 +91,7 @@ namespace BirdsiteLive
{
throw new NotImplementedException($"{dbSettings.Type} is not supported");
}
services.For<ITwitterUserService>().DecorateAllWith<CachedTwitterUserService>();
services.For<ITwitterUserService>().Use<TwitterUserService>().Singleton();

3
src/BirdsiteLive/appsettings.json
View file

@ -27,7 +27,8 @@
"InfoBanner": "",
"ShowAboutInstanceOnProfiles": true,
"MaxFollowsPerUser": 0,
"DiscloseInstanceRestrictions": false
"DiscloseInstanceRestrictions": false,
"EnableHashflags": false
},
"Db": {
"Type": "postgres",

12
src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs
View file

@ -48,7 +48,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object);
processor.WaitFactor = 10;
var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
var t = processor.UpdateTwitterAsync(buffer, CancellationToken.None);
await Task.WhenAny(t, Task.Delay(50));
@ -95,7 +95,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object);
processor.WaitFactor = 2;
var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
var t = processor.UpdateTwitterAsync(buffer, CancellationToken.None);
await Task.WhenAny(t, Task.Delay(300));
@ -142,7 +142,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object);
processor.WaitFactor = 2;
var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
var t = processor.UpdateTwitterAsync(buffer, CancellationToken.None);
var t2 = Task.Run(async () =>
{
while (buffer.Count < 11)
@ -186,7 +186,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object);
processor.WaitFactor = 1;
var t =processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
var t =processor.UpdateTwitterAsync(buffer, CancellationToken.None);
await Task.WhenAny(t, Task.Delay(50));
@ -223,7 +223,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object);
processor.WaitFactor = 10;
var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
var t = processor.UpdateTwitterAsync(buffer, CancellationToken.None);
await Task.WhenAny(t, Task.Delay(50));
@ -259,7 +259,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object);
processor.WaitFactor = 1;
await processor.GetTwitterUsersAsync(buffer, canTokenS.Token);
await processor.UpdateTwitterAsync(buffer, canTokenS.Token);
}
private static async Task<T> DelayFaultedTask<T>(Exception e)

2
src/Tests/BirdsiteLive.Pipeline.Tests/StatusPublicationPipelineTests.cs
View file

@ -22,7 +22,7 @@ namespace BirdsiteLive.Pipeline.Tests
#region Mocks
var retrieveTwitterUsersProcessor = new Mock<IRetrieveTwitterUsersProcessor>(MockBehavior.Strict);
retrieveTwitterUsersProcessor
.Setup(x => x.GetTwitterUsersAsync(
.Setup(x => x.UpdateTwitterAsync(
It.IsAny<BufferBlock<SyncTwitterUser[]>>(),
It.IsAny<CancellationToken>()))
.Returns(Task.Delay(0));