From cee46df117c4fbbf696bdb927ad155de9fcfdc48 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Thu, 4 Feb 2021 19:06:37 -0500 Subject: [PATCH 01/20] fix RT not being detected in some cases --- src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs index 7fce30c..75ae645 100644 --- a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs +++ b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs @@ -57,7 +57,7 @@ namespace BirdsiteLive.Twitter.Extractors var message = tweet.FullText; var tweetUrls = tweet.Media.Select(x => x.URL).Distinct(); - if (tweet.IsRetweet && tweet.QuotedStatusId == null && message.StartsWith("RT") && tweet.RetweetedTweet != null) + if (tweet.IsRetweet && message.StartsWith("RT") && tweet.RetweetedTweet != null) { message = tweet.RetweetedTweet.FullText; tweetUrls = tweet.RetweetedTweet.Media.Select(x => x.URL).Distinct(); From 5077819722f323deee7062aee41eef2e188abf36 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Thu, 4 Feb 2021 19:07:00 -0500 Subject: [PATCH 02/20] road to 0.13.1 --- src/BirdsiteLive/BirdsiteLive.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index 607a195..e23687f 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -4,7 +4,7 @@ netcoreapp3.1 d21486de-a812-47eb-a419-05682bb68856 Linux - 0.13.0 + 0.13.1 From 130ad6ff63ae60dcbb383450e086a74325eee963 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 6 Feb 2021 00:11:11 -0500 Subject: [PATCH 03/20] fix sig and user retrieval, fix #84 --- .../Regexes/HeaderRegexes.cs | 9 ++++ src/BirdsiteLive.Domain/ActivityPubService.cs | 2 +- src/BirdsiteLive.Domain/UserService.cs | 7 +-- src/BirdsiteLive.sln | 9 +++- .../BirdsiteLive.Common.Tests.csproj | 20 +++++++ .../Regexes/HeaderRegexTests.cs | 53 +++++++++++++++++++ 6 files changed, 95 insertions(+), 5 deletions(-) create mode 100644 src/BirdsiteLive.Common/Regexes/HeaderRegexes.cs create mode 100644 src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj create mode 100644 src/Tests/BirdsiteLive.Common.Tests/Regexes/HeaderRegexTests.cs diff --git a/src/BirdsiteLive.Common/Regexes/HeaderRegexes.cs b/src/BirdsiteLive.Common/Regexes/HeaderRegexes.cs new file mode 100644 index 0000000..b98818c --- /dev/null +++ b/src/BirdsiteLive.Common/Regexes/HeaderRegexes.cs @@ -0,0 +1,9 @@ +ο»Ώusing System.Text.RegularExpressions; + +namespace BirdsiteLive.Common.Regexes +{ + public class HeaderRegexes + { + public static readonly Regex HeaderSignature = new Regex(@"^([a-zA-Z0-9]+)=""(.+)""$"); + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs index 4d8fc54..53c656c 100644 --- a/src/BirdsiteLive.Domain/ActivityPubService.cs +++ b/src/BirdsiteLive.Domain/ActivityPubService.cs @@ -40,7 +40,7 @@ namespace BirdsiteLive.Domain public async Task GetUser(string objectId) { var httpClient = _httpClientFactory.CreateClient(); - httpClient.DefaultRequestHeaders.Add("Accept", "application/json"); + httpClient.DefaultRequestHeaders.Add("Accept", "application/activity+json"); var result = await httpClient.GetAsync(objectId); var content = await result.Content.ReadAsStringAsync(); return JsonConvert.DeserializeObject(content); diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index b666c29..713563f 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -7,6 +7,7 @@ using System.Text; using System.Threading.Tasks; using BirdsiteLive.ActivityPub; using BirdsiteLive.ActivityPub.Converters; +using BirdsiteLive.Common.Regexes; using BirdsiteLive.Common.Settings; using BirdsiteLive.Cryptography; using BirdsiteLive.Domain.BusinessUseCases; @@ -239,11 +240,11 @@ namespace BirdsiteLive.Domain var signature_header = new Dictionary(); foreach (var signature in signatures) { - var splitSig = signature.Replace("\"", string.Empty).Split('='); - signature_header.Add(splitSig[0], splitSig[1]); + var m = HeaderRegexes.HeaderSignature.Match(signature); + signature_header.Add(m.Groups[1].ToString(), m.Groups[2].ToString()); } - signature_header["signature"] = signature_header["signature"] + "=="; + //signature_header["signature"] = signature_header["signature"] + "=="; var key_id = signature_header["keyId"]; var headers = signature_header["headers"]; diff --git a/src/BirdsiteLive.sln b/src/BirdsiteLive.sln index 0a35bf6..1e9d65b 100644 --- a/src/BirdsiteLive.sln +++ b/src/BirdsiteLive.sln @@ -39,7 +39,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Domain.Tests", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Pipeline.Tests", "Tests\BirdsiteLive.Pipeline.Tests\BirdsiteLive.Pipeline.Tests.csproj", "{BF51CA81-5A7A-46F8-B4FB-861C6BE59298}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdsiteLive.DAL.Tests", "Tests\BirdsiteLive.DAL.Tests\BirdsiteLive.DAL.Tests.csproj", "{5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.DAL.Tests", "Tests\BirdsiteLive.DAL.Tests\BirdsiteLive.DAL.Tests.csproj", "{5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Common.Tests", "Tests\BirdsiteLive.Common.Tests\BirdsiteLive.Common.Tests.csproj", "{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -107,6 +109,10 @@ Global {5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}.Debug|Any CPU.Build.0 = Debug|Any CPU {5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}.Release|Any CPU.ActiveCfg = Release|Any CPU {5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}.Release|Any CPU.Build.0 = Release|Any CPU + {C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -126,6 +132,7 @@ Global {F544D745-89A8-4DEA-B61C-A7E6C53C1D63} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94} {BF51CA81-5A7A-46F8-B4FB-861C6BE59298} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94} {5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94} + {C69F7582-6050-44DC-BAAB-7C8F0BDA525C} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {69E8DCAD-4C37-4010-858F-5F94E6FBABCE} diff --git a/src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj b/src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj new file mode 100644 index 0000000..0a52603 --- /dev/null +++ b/src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj @@ -0,0 +1,20 @@ + + + + netcoreapp3.1 + + false + + + + + + + + + + + + + + diff --git a/src/Tests/BirdsiteLive.Common.Tests/Regexes/HeaderRegexTests.cs b/src/Tests/BirdsiteLive.Common.Tests/Regexes/HeaderRegexTests.cs new file mode 100644 index 0000000..8c906ea --- /dev/null +++ b/src/Tests/BirdsiteLive.Common.Tests/Regexes/HeaderRegexTests.cs @@ -0,0 +1,53 @@ +using BirdsiteLive.Common.Regexes; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace BirdsiteLive.Common.Tests +{ + [TestClass] + public class HeaderRegexTests + { + [TestMethod] + public void KeyId_Test() + { + var input = @"keyId=""https://misskey.tdl/users/8hwf6zy2k1#main-key"""; + + Assert.IsTrue(HeaderRegexes.HeaderSignature.IsMatch(input)); + var result = HeaderRegexes.HeaderSignature.Match(input); + Assert.AreEqual("keyId", result.Groups[1].ToString()); + Assert.AreEqual("https://misskey.tdl/users/8hwf6zy2k1#main-key", result.Groups[2].ToString()); + } + + [TestMethod] + public void Algorithm_Test() + { + var input = @"algorithm=""rsa-sha256"""; + + Assert.IsTrue(HeaderRegexes.HeaderSignature.IsMatch(input)); + var result = HeaderRegexes.HeaderSignature.Match(input); + Assert.AreEqual("algorithm", result.Groups[1].ToString()); + Assert.AreEqual("rsa-sha256", result.Groups[2].ToString()); + } + + [TestMethod] + public void Target_Test() + { + var input = @"headers=""(request-target) date host digest"""; + + Assert.IsTrue(HeaderRegexes.HeaderSignature.IsMatch(input)); + var result = HeaderRegexes.HeaderSignature.Match(input); + Assert.AreEqual("headers", result.Groups[1].ToString()); + Assert.AreEqual("(request-target) date host digest", result.Groups[2].ToString()); + } + + [TestMethod] + public void Signature_Test() + { + var input = @"signature=""ISxp4HYhkGc83SsCC3zNZMVKBota7HqgWVg6KwaQVTQcUqt+UsWXPxB0XPhaYkqLnH3hJ+KVMdoEn3rbzcw8XZpjVt48o9OAKd0rsEZYkLoERnnFFhEw0GmVDEhdoU7gyoeOreWGsIca6Pf7TC0vGTtqez31zmvoeXvxHgqRhWQvlZM/ovFR2xN+vhmF7rZdkd6UaKOzy21K8B/Q84J7PWdbJ8i0rKieVPDIuTCy5B0iQpgs1TMaz6xKZR/KVzAr207m9Gkku2gnJ4YZHFuoa2ct5M5AtIPMPCsWTU8yaimTkPdNNezSOKV5a7T55HSvFeopLNcQKsWNMioKGpZP5hCIRKNk0Ekx0yDReE6xF/qliT7eSAGVJ/6sLQjBpBFMPKBNOrYTxueBJGtISjCZlxaIyTtJ1ErNuCrKHGjImpNvvJzTJOtu+vWnjTcUJL7N1Mw7PEreCZrNUyNuAldDWSMAFuD4HVA3+KZjpWCfjAbyelzVy2gs96CyE56o9FqJEaM5XVQhsMTpa8OSHdr2QZtKYw7Wng0d8vmbKEX1pdTVeEIhi4M9js39ZdzB4mb8JXSBE/GA6PoE5s+oH3+GoufzJYINCpk0Ulwo9g7HKm9NATnwEZZPq4NKto5mSYZKYRtqjZaa8lIALNhdvzv2+8+ifPLHlOigAUVqoupd9Aq="""; + + Assert.IsTrue(HeaderRegexes.HeaderSignature.IsMatch(input)); + var result = HeaderRegexes.HeaderSignature.Match(input); + Assert.AreEqual("signature", result.Groups[1].ToString()); + Assert.AreEqual("ISxp4HYhkGc83SsCC3zNZMVKBota7HqgWVg6KwaQVTQcUqt+UsWXPxB0XPhaYkqLnH3hJ+KVMdoEn3rbzcw8XZpjVt48o9OAKd0rsEZYkLoERnnFFhEw0GmVDEhdoU7gyoeOreWGsIca6Pf7TC0vGTtqez31zmvoeXvxHgqRhWQvlZM/ovFR2xN+vhmF7rZdkd6UaKOzy21K8B/Q84J7PWdbJ8i0rKieVPDIuTCy5B0iQpgs1TMaz6xKZR/KVzAr207m9Gkku2gnJ4YZHFuoa2ct5M5AtIPMPCsWTU8yaimTkPdNNezSOKV5a7T55HSvFeopLNcQKsWNMioKGpZP5hCIRKNk0Ekx0yDReE6xF/qliT7eSAGVJ/6sLQjBpBFMPKBNOrYTxueBJGtISjCZlxaIyTtJ1ErNuCrKHGjImpNvvJzTJOtu+vWnjTcUJL7N1Mw7PEreCZrNUyNuAldDWSMAFuD4HVA3+KZjpWCfjAbyelzVy2gs96CyE56o9FqJEaM5XVQhsMTpa8OSHdr2QZtKYw7Wng0d8vmbKEX1pdTVeEIhi4M9js39ZdzB4mb8JXSBE/GA6PoE5s+oH3+GoufzJYINCpk0Ulwo9g7HKm9NATnwEZZPq4NKto5mSYZKYRtqjZaa8lIALNhdvzv2+8+ifPLHlOigAUVqoupd9Aq=", result.Groups[2].ToString()); + } + } +} From 6d5fe3089ee8b661fd8284217ca4f3a344636b4a Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 6 Feb 2021 00:13:38 -0500 Subject: [PATCH 04/20] clean up --- src/BirdsiteLive.Domain/UserService.cs | 33 +++++++------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index 713563f..a72f017 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -244,29 +244,20 @@ namespace BirdsiteLive.Domain signature_header.Add(m.Groups[1].ToString(), m.Groups[2].ToString()); } - //signature_header["signature"] = signature_header["signature"] + "=="; - var key_id = signature_header["keyId"]; var headers = signature_header["headers"]; var algorithm = signature_header["algorithm"]; var sig = Convert.FromBase64String(signature_header["signature"]); - + // Retrieve User var remoteUser = await _activityPubService.GetUser(actor); + // Prepare Key data var toDecode = remoteUser.publicKey.publicKeyPem.Trim().Remove(0, remoteUser.publicKey.publicKeyPem.IndexOf('\n')); toDecode = toDecode.Remove(toDecode.LastIndexOf('\n')).Replace("\n", ""); var signKey = ASN1.ToRSA(Convert.FromBase64String(toDecode)); var toSign = new StringBuilder(); - //var comparisonString = headers.Split(' ').Select(signed_header_name => - //{ - // if (signed_header_name == "(request-target)") - // return "(request-target): post /inbox"; - // else - // return $"{signed_header_name}: {r.Headers[signed_header_name.ToUpperInvariant()]}"; - //}); - foreach (var headerKey in headers.Split(' ')) { if (headerKey == "(request-target)") toSign.Append($"(request-target): {method.ToLower()} {path}{queryString}\n"); @@ -274,21 +265,13 @@ namespace BirdsiteLive.Domain } toSign.Remove(toSign.Length - 1, 1); - //var signKey = ASN1.ToRSA(Convert.FromBase64String(toDecode)); - - //new RSACryptoServiceProvider(keyId.publicKey.publicKeyPem); - - //Create a new instance of RSACryptoServiceProvider. - RSACryptoServiceProvider key = new RSACryptoServiceProvider(); - - //Get an instance of RSAParameters from ExportParameters function. - RSAParameters RSAKeyInfo = key.ExportParameters(false); - - //Set RSAKeyInfo to the public key values. - RSAKeyInfo.Modulus = Convert.FromBase64String(toDecode); - - key.ImportParameters(RSAKeyInfo); + // Import key + var key = new RSACryptoServiceProvider(); + var rsaKeyInfo = key.ExportParameters(false); + rsaKeyInfo.Modulus = Convert.FromBase64String(toDecode); + key.ImportParameters(rsaKeyInfo); + // Trust and Verify var result = signKey.VerifyData(Encoding.UTF8.GetBytes(toSign.ToString()), sig, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); return new SignatureValidationResult() From c02b4804f5d0d186ded344692c40e0ee84b3750d Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 6 Feb 2021 00:14:14 -0500 Subject: [PATCH 05/20] road to 0.14.0 --- src/BirdsiteLive/BirdsiteLive.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index e23687f..346a936 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -4,7 +4,7 @@ netcoreapp3.1 d21486de-a812-47eb-a419-05682bb68856 Linux - 0.13.1 + 0.14.0 From 674fde74bda9d6d55f4c9025f77733626c0f4607 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 10 Feb 2021 00:19:12 -0500 Subject: [PATCH 06/20] fix small link parsing --- src/BirdsiteLive.Common/Regexes/UrlRegexes.cs | 2 +- .../Tools/StatusExtractor.cs | 9 +-- .../Tools/StatusExtractorTests.cs | 59 ++++++++++++++++++- 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/src/BirdsiteLive.Common/Regexes/UrlRegexes.cs b/src/BirdsiteLive.Common/Regexes/UrlRegexes.cs index ea3e5c2..1f2b279 100644 --- a/src/BirdsiteLive.Common/Regexes/UrlRegexes.cs +++ b/src/BirdsiteLive.Common/Regexes/UrlRegexes.cs @@ -4,6 +4,6 @@ namespace BirdsiteLive.Common.Regexes { public class UrlRegexes { - public static readonly Regex Url = new Regex(@"((http|ftp|https):\/\/[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])?)"); + public static readonly Regex Url = new Regex(@"(.?)(((http|ftp|https):\/\/)[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])?)"); } } \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs index a43ccb0..7df4c0e 100644 --- a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs +++ b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs @@ -44,11 +44,8 @@ namespace BirdsiteLive.Domain.Tools var urlMatch = UrlRegexes.Url.Matches(messageContent); foreach (Match m in urlMatch) { - var url = m.ToString().Replace("\n", string.Empty).Trim(); - - var protocol = "https://"; - if (url.StartsWith("http://")) protocol = "http://"; - else if (url.StartsWith("ftp://")) protocol = "ftp://"; + var url = m.Groups[2].ToString(); + var protocol = m.Groups[3].ToString(); var truncatedUrl = url.Replace(protocol, string.Empty); @@ -68,7 +65,7 @@ namespace BirdsiteLive.Domain.Tools } messageContent = Regex.Replace(messageContent, m.ToString(), - $@" {protocol}{firstPart}{secondPart}"); + $@"{m.Groups[1]}{protocol}{firstPart}{secondPart}"); } // Extract Hashtags diff --git a/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs index 924461f..06d8f9a 100644 --- a/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs +++ b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs @@ -136,7 +136,7 @@ namespace BirdsiteLive.Domain.Tests.Tools } [TestMethod] - public void Extract_MultiUrls__Test() + public void Extract_MultiUrls_Test() { #region Stubs var message = $"https://t.co/L8BpyHgg25 Bla!{Environment.NewLine}https://www.eff.org/deeplinks/2020/07/pact-act-not-solution-problem-harmful-online-content"; @@ -160,6 +160,63 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion } + [TestMethod] + public void Extract_SmallUrl_Test() + { + #region Stubs + var message = @"πŸš€ test http://GOV.UK date πŸŽ‰ data http://GOV.UK woopsi."; + #endregion + + #region Mocks + var logger = new Mock>(); + #endregion + + var service = new StatusExtractor(_settings, logger.Object); + var result = service.Extract(message); + + #region Validations + Assert.AreEqual(@"πŸš€ test http://GOV.UK date πŸŽ‰ data http://GOV.UK woopsi.", result.content); + #endregion + } + + [TestMethod] + public void Extract_SmallUrl_2_Test() + { + #region Stubs + var message = @"πŸš€http://GOV.UK"; + #endregion + + #region Mocks + var logger = new Mock>(); + #endregion + + var service = new StatusExtractor(_settings, logger.Object); + var result = service.Extract(message); + + #region Validations + Assert.AreEqual(@"πŸš€http://GOV.UK", result.content); + #endregion + } + + [TestMethod] + public void Extract_SmallUrl_3_Test() + { + #region Stubs + var message = @"πŸš€http://GOV.UK."; + #endregion + + #region Mocks + var logger = new Mock>(); + #endregion + + var service = new StatusExtractor(_settings, logger.Object); + var result = service.Extract(message); + + #region Validations + Assert.AreEqual(@"πŸš€http://GOV.UK.", result.content); + #endregion + } + [TestMethod] public void Extract_SingleHashTag_Test() { From 567f02a4e2fadc76d184061101ae75fdc5c948a9 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 10 Feb 2021 01:04:00 -0500 Subject: [PATCH 07/20] road to 0.14.1 --- src/BirdsiteLive/BirdsiteLive.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index 346a936..cb2c708 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -4,7 +4,7 @@ netcoreapp3.1 d21486de-a812-47eb-a419-05682bb68856 Linux - 0.14.0 + 0.14.1 From 5f480b4baac427b258fac304a3224528ff7e80ad Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 10 Feb 2021 01:35:29 -0500 Subject: [PATCH 08/20] don't retrieve TL if user is protected --- .../Processors/RetrieveTweetsProcessor.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs index e60c121..bb5e026 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs @@ -68,6 +68,10 @@ namespace BirdsiteLive.Pipeline.Processors { var tweets = new ExtractedTweet[0]; + // Don't retrieve TL if protected + var userView = _twitterUserService.GetUser(user.Acct); + if (userView == null || userView.Protected) return tweets; + try { if (user.LastTweetPostedId == -1) From 7fec7e765e51399ed24b725a7bd6d3309be83bb2 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 10 Feb 2021 01:41:55 -0500 Subject: [PATCH 09/20] fix tests --- .../Processors/RetrieveTweetsProcessorTests.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs index 02fd7bd..38b750c 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs @@ -62,6 +62,9 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); var twitterUserServiceMock = new Mock(MockBehavior.Strict); + twitterUserServiceMock + .Setup(x => x.GetUser(It.Is(y => y == user1.Acct))) + .Returns(new TwitterUser {Protected = false}); var logger = new Mock>(MockBehavior.Strict); #endregion @@ -126,6 +129,9 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); var twitterUserServiceMock = new Mock(MockBehavior.Strict); + twitterUserServiceMock + .Setup(x => x.GetUser(It.Is(y => y == user1.Acct))) + .Returns(new TwitterUser { Protected = false }); var logger = new Mock>(MockBehavior.Strict); #endregion @@ -193,6 +199,9 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); var twitterUserServiceMock = new Mock(MockBehavior.Strict); + twitterUserServiceMock + .Setup(x => x.GetUser(It.Is(y => y == user1.Acct))) + .Returns(new TwitterUser { Protected = false }); var logger = new Mock>(MockBehavior.Strict); #endregion From 5aa8deb47b4aa589c34455896f68cc765a96f9c7 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 10 Feb 2021 23:31:29 -0500 Subject: [PATCH 10/20] make sure avatar is served in https, fix #87 --- src/BirdsiteLive.Twitter/TwitterUserService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive.Twitter/TwitterUserService.cs b/src/BirdsiteLive.Twitter/TwitterUserService.cs index dd80c4d..2370cea 100644 --- a/src/BirdsiteLive.Twitter/TwitterUserService.cs +++ b/src/BirdsiteLive.Twitter/TwitterUserService.cs @@ -64,7 +64,7 @@ namespace BirdsiteLive.Twitter Name = user.Name, Description = description, Url = $"https://twitter.com/{username}", - ProfileImageUrl = user.ProfileImageUrlFullSize, + ProfileImageUrl = user.ProfileImageUrlFullSize.Replace("http://", "https://"), ProfileBackgroundImageUrl = user.ProfileBackgroundImageUrlHttps, ProfileBannerURL = user.ProfileBannerURL, Protected = user.Protected From d660dc990c85a5df9eb9dfeef58ad118abc596d3 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 10 Feb 2021 23:32:57 -0500 Subject: [PATCH 11/20] road to 0.14.2 --- src/BirdsiteLive/BirdsiteLive.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index cb2c708..7c24384 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -4,7 +4,7 @@ netcoreapp3.1 d21486de-a812-47eb-a419-05682bb68856 Linux - 0.14.1 + 0.14.2 From cd36c629358f050ca5e85d982f41536fba418f3b Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Thu, 11 Feb 2021 00:45:55 -0500 Subject: [PATCH 12/20] escape URL regex --- .../Tools/StatusExtractor.cs | 2 +- .../Tools/StatusExtractorTests.cs | 21 ++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs index 7df4c0e..5cbff68 100644 --- a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs +++ b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs @@ -64,7 +64,7 @@ namespace BirdsiteLive.Domain.Tools secondPart = truncatedUrl.Substring(30); } - messageContent = Regex.Replace(messageContent, m.ToString(), + messageContent = Regex.Replace(messageContent, Regex.Escape(m.ToString()), $@"{m.Groups[1]}{protocol}{firstPart}{secondPart}"); } diff --git a/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs index 06d8f9a..a279dd0 100644 --- a/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs +++ b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs @@ -217,6 +217,25 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion } + [TestMethod] + public void Extract_UrlRegexChars_Test() + { + #region Stubs + var message = @"🐣 juniors & tech(http://tech.guru maker)"; + #endregion + + #region Mocks + var logger = new Mock>(); + #endregion + + var service = new StatusExtractor(_settings, logger.Object); + var result = service.Extract(message); + + #region Validations + Assert.AreEqual(@"🐣 juniors & tech(http://tech.guru maker)", result.content); + #endregion + } + [TestMethod] public void Extract_SingleHashTag_Test() { @@ -345,7 +364,7 @@ namespace BirdsiteLive.Domain.Tests.Tools Assert.IsTrue(result.content.Contains(@"@mynickname")); #endregion } - + [TestMethod] public void Extract_SingleMentionTag_RT_Test() { From 9bdca4e202c303d9f8835d385ed87a225fac5b58 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Thu, 11 Feb 2021 20:11:32 -0500 Subject: [PATCH 13/20] road to 0.14.3 --- src/BirdsiteLive/BirdsiteLive.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index 7c24384..26df777 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -4,7 +4,7 @@ netcoreapp3.1 d21486de-a812-47eb-a419-05682bb68856 Linux - 0.14.2 + 0.14.3 From 2181230d96e2ca7d69b071493380d7bf8337cc17 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 12 Feb 2021 18:22:43 -0500 Subject: [PATCH 14/20] added auto-update section --- INSTALLATION.md | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/INSTALLATION.md b/INSTALLATION.md index 67cb10a..6ad0af6 100644 --- a/INSTALLATION.md +++ b/INSTALLATION.md @@ -138,11 +138,11 @@ sudo ufw status You should now have an up and running BirdsiteLIVE instance! -## Upgrading +## Updating Make sure your data belong outside the containers before migrating (set by default). -To upgrade your installation to the latest release: +To update your installation to the latest release: ``` # Edit `docker-compose.yml` to update the version, if you have one specified @@ -152,6 +152,37 @@ docker-compose pull docker-compose up -d ``` +## Auto-Updating + +To set auto-updates on your deployment, add to the `docker-compose.yml` file this section: + +```diff +version: "3" + +networks: + birdsitelivenetwork: + external: false + +services: + server: + image: nicolasconstant/birdsitelive:latest + [...] + + db: + image: postgres:9.6 + [...] + ++ watchtower: ++ image: containrrr/watchtower ++ restart: always ++ container_name: watchtower ++ environment: ++ - WATCHTOWER_CLEANUP=true ++ volumes: ++ - /var/run/docker.sock:/var/run/docker.sock ++ command: --interval 300 +``` + ## More options -You can find more options available [here](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md) \ No newline at end of file +You can find more options available [here](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md) From 30bd16447f3b0ea9e942fbedb1046eb6ebf12a6e Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 13 Feb 2021 21:17:48 -0500 Subject: [PATCH 15/20] stop application if worker fails --- src/BirdsiteLive/Services/FederationService.cs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/BirdsiteLive/Services/FederationService.cs b/src/BirdsiteLive/Services/FederationService.cs index 9acab41..7ff4c1f 100644 --- a/src/BirdsiteLive/Services/FederationService.cs +++ b/src/BirdsiteLive/Services/FederationService.cs @@ -13,19 +13,28 @@ namespace BirdsiteLive.Services { private readonly IDatabaseInitializer _databaseInitializer; private readonly IStatusPublicationPipeline _statusPublicationPipeline; + private readonly IHostApplicationLifetime _applicationLifetime; #region Ctor - public FederationService(IDatabaseInitializer databaseInitializer, IStatusPublicationPipeline statusPublicationPipeline) + public FederationService(IDatabaseInitializer databaseInitializer, IStatusPublicationPipeline statusPublicationPipeline, IHostApplicationLifetime applicationLifetime) { _databaseInitializer = databaseInitializer; _statusPublicationPipeline = statusPublicationPipeline; + _applicationLifetime = applicationLifetime; } #endregion protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - await _databaseInitializer.InitAndMigrateDbAsync(); - await _statusPublicationPipeline.ExecuteAsync(stoppingToken); + try + { + await _databaseInitializer.InitAndMigrateDbAsync(); + await _statusPublicationPipeline.ExecuteAsync(stoppingToken); + } + finally + { + _applicationLifetime.StopApplication(); + } } } } \ No newline at end of file From bc6fe941015bc8baf63065a86c261385ac1f0599 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 13 Feb 2021 21:18:28 -0500 Subject: [PATCH 16/20] road to 0.14.4 --- src/BirdsiteLive/BirdsiteLive.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index 26df777..8b8febc 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -4,7 +4,7 @@ netcoreapp3.1 d21486de-a812-47eb-a419-05682bb68856 Linux - 0.14.3 + 0.14.4 From dad118d2228cd7927ac0c01fb38469b17d693e0a Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 13 Feb 2021 22:42:05 -0500 Subject: [PATCH 17/20] fine tune capacity --- src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs | 2 +- src/BirdsiteLive/appsettings.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs b/src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs index afea0d0..04a5649 100644 --- a/src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs +++ b/src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs @@ -87,7 +87,7 @@ namespace BirdsiteLive.Statistics.Domain UserCallsCountMin = userCalls.Any() ? userCalls.Min() : 0, UserCallsCountAvg = userCalls.Any() ? (int)userCalls.Average() : 0, UserCallsCountMax = userCalls.Any() ? userCalls.Max() : 0, - UserCallsMax = 900, + UserCallsMax = 300, TweetCallsCountMin = tweetCalls.Any() ? tweetCalls.Min() : 0, TweetCallsCountAvg = tweetCalls.Any() ? (int)tweetCalls.Average() : 0, TweetCallsCountMax = tweetCalls.Any() ? tweetCalls.Max() : 0, diff --git a/src/BirdsiteLive/appsettings.json b/src/BirdsiteLive/appsettings.json index 3dc47ff..106b4e5 100644 --- a/src/BirdsiteLive/appsettings.json +++ b/src/BirdsiteLive/appsettings.json @@ -15,7 +15,7 @@ "AdminEmail": "me@domain.name", "ResolveMentionsInProfiles": true, "PublishReplies": false, - "MaxUsersCapacity": 1400 + "MaxUsersCapacity": 800 }, "Db": { "Type": "postgres", From 922de367a8b4c9b3bd879b6fdda221c148ef5cd1 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Mon, 15 Feb 2021 18:28:48 -0500 Subject: [PATCH 18/20] added delay before stopping app --- src/BirdsiteLive/Services/FederationService.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/BirdsiteLive/Services/FederationService.cs b/src/BirdsiteLive/Services/FederationService.cs index 7ff4c1f..26583f9 100644 --- a/src/BirdsiteLive/Services/FederationService.cs +++ b/src/BirdsiteLive/Services/FederationService.cs @@ -33,6 +33,7 @@ namespace BirdsiteLive.Services } finally { + await Task.Delay(1000 * 30); _applicationLifetime.StopApplication(); } } From f0dc54ad0159c3b1d9f160b4cdf0b573542352fe Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Mon, 15 Feb 2021 18:54:52 -0500 Subject: [PATCH 19/20] added log and resiliency --- .../Processors/SaveProgressionProcessor.cs | 44 ++++++++++++++++--- .../SaveProgressionProcessorTests.cs | 16 +++++-- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs index c7cbc36..c2f3ff5 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs @@ -5,27 +5,57 @@ using System.Threading.Tasks; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.Pipeline.Contracts; using BirdsiteLive.Pipeline.Models; +using Microsoft.Extensions.Logging; namespace BirdsiteLive.Pipeline.Processors { public class SaveProgressionProcessor : ISaveProgressionProcessor { private readonly ITwitterUserDal _twitterUserDal; - + private readonly ILogger _logger; + #region Ctor - public SaveProgressionProcessor(ITwitterUserDal twitterUserDal) + public SaveProgressionProcessor(ITwitterUserDal twitterUserDal, ILogger logger) { _twitterUserDal = twitterUserDal; + _logger = logger; } #endregion public async Task ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct) { - var userId = userWithTweetsToSync.User.Id; - var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max(); - var minimumSync = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).Min(); - var now = DateTime.UtcNow; - await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, now); + try + { + if (userWithTweetsToSync.Tweets.Length == 0) + { + _logger.LogWarning("No tweets synchronized"); + return; + } + if(userWithTweetsToSync.Followers.Length == 0) + { + _logger.LogWarning("No Followers found for {User}", userWithTweetsToSync.User.Acct); + return; + } + + var userId = userWithTweetsToSync.User.Id; + var followingSyncStatuses = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).ToList(); + + if (followingSyncStatuses.Count == 0) + { + _logger.LogWarning("No Followers sync found for {User}, Id: {UserId}", userWithTweetsToSync.User.Acct, userId); + return; + } + + var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max(); + var minimumSync = followingSyncStatuses.Min(); + var now = DateTime.UtcNow; + await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, now); + } + catch (Exception e) + { + _logger.LogError(e, "SaveProgressionProcessor.ProcessAsync() Exception"); + throw; + } } } } \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs index b2a99b9..c95eed6 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs @@ -8,6 +8,7 @@ using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Pipeline.Processors; using BirdsiteLive.Twitter.Models; using Castle.DynamicProxy.Contributors; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -53,6 +54,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors }, User = user }; + + var loggerMock = new Mock>(); #endregion #region Mocks @@ -67,11 +70,12 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); #endregion - var processor = new SaveProgressionProcessor(twitterUserDalMock.Object); + var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object); await processor.ProcessAsync(usersWithTweets, CancellationToken.None); #region Validations twitterUserDalMock.VerifyAll(); + loggerMock.VerifyAll(); #endregion } @@ -129,13 +133,16 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.IsAny() )) .Returns(Task.CompletedTask); + + var loggerMock = new Mock>(); #endregion - var processor = new SaveProgressionProcessor(twitterUserDalMock.Object); + var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object); await processor.ProcessAsync(usersWithTweets, CancellationToken.None); #region Validations twitterUserDalMock.VerifyAll(); + loggerMock.VerifyAll(); #endregion } @@ -201,13 +208,16 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.IsAny() )) .Returns(Task.CompletedTask); + + var loggerMock = new Mock>(); #endregion - var processor = new SaveProgressionProcessor(twitterUserDalMock.Object); + var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object); await processor.ProcessAsync(usersWithTweets, CancellationToken.None); #region Validations twitterUserDalMock.VerifyAll(); + loggerMock.VerifyAll(); #endregion } } From bde9c0c34d1463a0c470bbd4d18d9e482a4eb802 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Mon, 15 Feb 2021 19:46:44 -0500 Subject: [PATCH 20/20] road to 0.14.5 --- src/BirdsiteLive/BirdsiteLive.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index 8b8febc..038392d 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -4,7 +4,7 @@ netcoreapp3.1 d21486de-a812-47eb-a419-05682bb68856 Linux - 0.14.4 + 0.14.5