From a59ed49a9aec3a4bde022df545bc47f7d4273814 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 15 Jan 2021 01:42:05 -0500 Subject: [PATCH 1/3] added option to prevent mention extraction in user's description --- .../Settings/InstanceSettings.cs | 1 + src/BirdsiteLive.Domain/StatusService.cs | 2 +- .../Tools/StatusExtractor.cs | 33 ++++++++++-------- src/BirdsiteLive.Domain/UserService.cs | 2 +- src/BirdsiteLive/appsettings.json | 3 +- .../Tools/StatusExtractorTests.cs | 34 +++++++++---------- 6 files changed, 40 insertions(+), 35 deletions(-) diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs index ba2a517..2ce9715 100644 --- a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs +++ b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs @@ -4,5 +4,6 @@ { public string Domain { get; set; } public string AdminEmail { get; set; } + public bool ResolveMentionsInProfiles { get; set; } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/StatusService.cs b/src/BirdsiteLive.Domain/StatusService.cs index a1b3d38..c669779 100644 --- a/src/BirdsiteLive.Domain/StatusService.cs +++ b/src/BirdsiteLive.Domain/StatusService.cs @@ -43,7 +43,7 @@ namespace BirdsiteLive.Domain var to = $"{actorUrl}/followers"; var apPublic = "https://www.w3.org/ns/activitystreams#Public"; - var extractedTags = _statusExtractor.ExtractTags(tweet.MessageContent); + var extractedTags = _statusExtractor.Extract(tweet.MessageContent); _statisticsHandler.ExtractedStatus(extractedTags.tags.Count(x => x.type == "Mention")); string inReplyTo = null; diff --git a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs index ed401a6..778f6b5 100644 --- a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs +++ b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs @@ -8,7 +8,7 @@ namespace BirdsiteLive.Domain.Tools { public interface IStatusExtractor { - (string content, Tag[] tags) ExtractTags(string messageContent); + (string content, Tag[] tags) Extract(string messageContent, bool extractMentions = true); } public class StatusExtractor : IStatusExtractor @@ -36,7 +36,7 @@ namespace BirdsiteLive.Domain.Tools } #endregion - public (string content, Tag[] tags) ExtractTags(string messageContent) + public (string content, Tag[] tags) Extract(string messageContent, bool extractMentions = true) { var tags = new List(); messageContent = $" {messageContent} "; @@ -102,22 +102,25 @@ namespace BirdsiteLive.Domain.Tools } // Extract Mentions - var mentionMatch = OrderByLength(_mentionRegex.Matches(messageContent)); - foreach (Match m in mentionMatch.OrderByDescending(x => x.Length)) + if (extractMentions) { - var mention = m.ToString().Replace("@", string.Empty).Replace("\n", string.Empty).Trim(); - var url = $"https://{_instanceSettings.Domain}/users/{mention}"; - var name = $"@{mention}@{_instanceSettings.Domain}"; - - tags.Add(new Tag + var mentionMatch = OrderByLength(_mentionRegex.Matches(messageContent)); + foreach (Match m in mentionMatch.OrderByDescending(x => x.Length)) { - name = name, - href = url, - type = "Mention" - }); + var mention = m.ToString().Replace("@", string.Empty).Replace("\n", string.Empty).Trim(); + var url = $"https://{_instanceSettings.Domain}/users/{mention}"; + var name = $"@{mention}@{_instanceSettings.Domain}"; - messageContent = Regex.Replace(messageContent, m.ToString(), - $@" @{mention}"); + tags.Add(new Tag + { + name = name, + href = url, + type = "Mention" + }); + + messageContent = Regex.Replace(messageContent, m.ToString(), + $@" @{mention}"); + } } // Clean up return lines diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index 6e498af..6131ca9 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -59,7 +59,7 @@ namespace BirdsiteLive.Domain var description = twitterUser.Description; if (!string.IsNullOrWhiteSpace(description)) { - var extracted = _statusExtractor.ExtractTags(description); + var extracted = _statusExtractor.Extract(description, _instanceSettings.ResolveMentionsInProfiles); description = extracted.content; _statisticsHandler.ExtractedDescription(extracted.tags.Count(x => x.type == "Mention")); diff --git a/src/BirdsiteLive/appsettings.json b/src/BirdsiteLive/appsettings.json index f240094..d0c49bf 100644 --- a/src/BirdsiteLive/appsettings.json +++ b/src/BirdsiteLive/appsettings.json @@ -11,7 +11,8 @@ "AllowedHosts": "*", "Instance": { "Domain": "domain.name", - "AdminEmail": "me@domain.name" + "AdminEmail": "me@domain.name", + "ResolveMentionsInProfiles": true }, "Db": { "Type": "postgres", diff --git a/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs index 32d1faa..f5dc91a 100644 --- a/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs +++ b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs @@ -30,7 +30,7 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion var service = new StatusExtractor(_settings); - var result = service.ExtractTags(message); + var result = service.Extract(message); #region Validations Assert.IsTrue(result.content.Contains("Bla.")); @@ -46,7 +46,7 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion var service = new StatusExtractor(_settings); - var result = service.ExtractTags(message); + var result = service.Extract(message); #region Validations Assert.IsTrue(result.content.Contains("Bla.")); @@ -62,7 +62,7 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion var service = new StatusExtractor(_settings); - var result = service.ExtractTags(message); + var result = service.Extract(message); #region Validations Assert.AreEqual(0, result.tags.Length); @@ -80,7 +80,7 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion var service = new StatusExtractor(_settings); - var result = service.ExtractTags(message); + var result = service.Extract(message); #region Validations Assert.AreEqual(0, result.tags.Length); @@ -98,7 +98,7 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion var service = new StatusExtractor(_settings); - var result = service.ExtractTags(message); + var result = service.Extract(message); #region Validations Assert.AreEqual(0, result.tags.Length); @@ -116,7 +116,7 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion var service = new StatusExtractor(_settings); - var result = service.ExtractTags(message); + var result = service.Extract(message); #region Validations Assert.AreEqual(0, result.tags.Length); @@ -136,7 +136,7 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion var service = new StatusExtractor(_settings); - var result = service.ExtractTags(message); + var result = service.Extract(message); #region Validations Assert.AreEqual(1, result.tags.Length); @@ -157,7 +157,7 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion var service = new StatusExtractor(_settings); - var result = service.ExtractTags(message); + var result = service.Extract(message); #region Validations Assert.AreEqual(1, result.tags.Length); @@ -178,7 +178,7 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion var service = new StatusExtractor(_settings); - var result = service.ExtractTags(message); + var result = service.Extract(message); #region Validations Assert.AreEqual(1, result.tags.Length); @@ -199,7 +199,7 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion var service = new StatusExtractor(_settings); - var result = service.ExtractTags(message); + var result = service.Extract(message); #region Validations Assert.AreEqual(4, result.tags.Length); @@ -219,7 +219,7 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion var service = new StatusExtractor(_settings); - var result = service.ExtractTags(message); + var result = service.Extract(message); #region Validations Assert.AreEqual(1, result.tags.Length); @@ -240,7 +240,7 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion var service = new StatusExtractor(_settings); - var result = service.ExtractTags(message); + var result = service.Extract(message); #region Validations Assert.AreEqual(1, result.tags.Length); @@ -261,7 +261,7 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion var service = new StatusExtractor(_settings); - var result = service.ExtractTags(message); + var result = service.Extract(message); #region Validations Assert.AreEqual(1, result.tags.Length); @@ -282,7 +282,7 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion var service = new StatusExtractor(_settings); - var result = service.ExtractTags(message); + var result = service.Extract(message); #region Validations Assert.AreEqual(4, result.tags.Length); @@ -302,7 +302,7 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion var service = new StatusExtractor(_settings); - var result = service.ExtractTags(message); + var result = service.Extract(message); #region Validations Assert.AreEqual(5, result.tags.Length); @@ -325,7 +325,7 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion var service = new StatusExtractor(_settings); - var result = service.ExtractTags(message); + var result = service.Extract(message); #region Validations Assert.AreEqual(1, result.tags.Length); @@ -345,7 +345,7 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion var service = new StatusExtractor(_settings); - var result = service.ExtractTags(message); + var result = service.Extract(message); #region Validations Assert.AreEqual(1, result.tags.Length); From 532b87f7190f9807e1804124a968cf9bddb4c49e Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 15 Jan 2021 02:00:10 -0500 Subject: [PATCH 2/3] clean up --- .../Processors/RetrieveTwitterUsersProcessor.cs | 1 + src/BirdsiteLive/Controllers/UsersController.cs | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs index f8ea2a2..adff434 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs @@ -36,6 +36,7 @@ namespace BirdsiteLive.Pipeline.Processors catch (Exception e) { Console.WriteLine(e); + //TODO handle error } await Task.Delay(SyncPeriod * 1000 * 60, ct); diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index d85a6d9..d1d22c8 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -125,10 +125,7 @@ namespace BirdsiteLive.Controllers var activity = ApDeserializer.ProcessActivity(body); // Do something var signature = r.Headers["Signature"].First(); - - Console.WriteLine(body); - Console.WriteLine(); - + switch (activity?.type) { case "Follow": From 3dad498e78af5bd012c92b3438e9bd761a79c6f6 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 15 Jan 2021 17:29:35 -0500 Subject: [PATCH 3/3] road to 0.8.0 --- src/BirdsiteLive/BirdsiteLive.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index af20da9..f665db2 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -4,7 +4,7 @@ netcoreapp3.1 d21486de-a812-47eb-a419-05682bb68856 Linux - 0.7.0 + 0.8.0