Merge pull request #81 from NicolasConstant/develop

0.13.0 PR

src/BirdsiteLive.Common/Regexes/HashtagRegexes.cs

@@ -0,0 +1,10 @@
+using System.Text.RegularExpressions;
+
+namespace BirdsiteLive.Common.Regexes
+{
+    public class HashtagRegexes
+    {
+        public static readonly Regex HashtagName = new Regex(@"^[a-zA-Z0-9_]+$");
+        public static readonly Regex Hashtag = new Regex(@"(.?)#([a-zA-Z0-9_]+)(\s|$|[\[\]<>.,;:!?/|-])");
+    }
+}

src/BirdsiteLive.Common/Regexes/UrlRegexes.cs

@@ -0,0 +1,9 @@
+using System.Text.RegularExpressions;
+
+namespace BirdsiteLive.Common.Regexes
+{
+    public class UrlRegexes
+    {
+        public static readonly Regex Url = new Regex(@"((http|ftp|https):\/\/[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])?)");
+    }
+}

src/BirdsiteLive.Common/Regexes/UserRegexes.cs

@@ -0,0 +1,10 @@
+using System.Text.RegularExpressions;
+
+namespace BirdsiteLive.Common.Regexes
+{
+    public class UserRegexes
+    {
+        public static readonly Regex TwitterAccount = new Regex(@"^[a-zA-Z0-9_]+$");
+        public static readonly Regex Mention = new Regex(@"(.?)@([a-zA-Z0-9_]+)(\s|$|[\[\]<>,;:!?/|-]|(. ))");
+    }
+}

src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs

@@ -9,6 +9,7 @@ using BirdsiteLive.Pipeline.Contracts;
 using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
+using Microsoft.Extensions.Logging;
 using Tweetinvi.Models;
 
 namespace BirdsiteLive.Pipeline.Processors
@@ -16,13 +17,17 @@ namespace BirdsiteLive.Pipeline.Processors
     public class RetrieveTweetsProcessor : IRetrieveTweetsProcessor
     {
         private readonly ITwitterTweetsService _twitterTweetsService;
+        private readonly ICachedTwitterUserService _twitterUserService;
         private readonly ITwitterUserDal _twitterUserDal;
+        private readonly ILogger<RetrieveTweetsProcessor> _logger;
 
         #region Ctor
-        public RetrieveTweetsProcessor(ITwitterTweetsService twitterTweetsService, ITwitterUserDal twitterUserDal)
+        public RetrieveTweetsProcessor(ITwitterTweetsService twitterTweetsService, ITwitterUserDal twitterUserDal, ICachedTwitterUserService twitterUserService, ILogger<RetrieveTweetsProcessor> logger)
         {
             _twitterTweetsService = twitterTweetsService;
             _twitterUserDal = twitterUserDal;
+            _twitterUserService = twitterUserService;
+            _logger = logger;
         }
         #endregion
 
@@ -61,11 +66,20 @@ namespace BirdsiteLive.Pipeline.Processors
 
         private ExtractedTweet[] RetrieveNewTweets(SyncTwitterUser user)
         {
-            ExtractedTweet[] tweets;
-            if (user.LastTweetPostedId == -1)
-                tweets = _twitterTweetsService.GetTimeline(user.Acct, 1);
-            else
-                tweets = _twitterTweetsService.GetTimeline(user.Acct, 200, user.LastTweetSynchronizedForAllFollowersId);
+            var tweets = new ExtractedTweet[0];
+
+            try
+            {
+                if (user.LastTweetPostedId == -1)
+                    tweets = _twitterTweetsService.GetTimeline(user.Acct, 1);
+                else
+                    tweets = _twitterTweetsService.GetTimeline(user.Acct, 200, user.LastTweetSynchronizedForAllFollowersId);
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, "Error retrieving TL of {Username} from {LastTweetPostedId}, purging user from cache", user.Acct, user.LastTweetPostedId);
+                _twitterUserService.PurgeUser(user.Acct);
+            }
 
             return tweets;
         }

src/BirdsiteLive.Twitter/CachedTwitterService.cs

@@ -4,7 +4,12 @@ using Microsoft.Extensions.Caching.Memory;
 
 namespace BirdsiteLive.Twitter
 {
-    public class CachedTwitterUserService : ITwitterUserService
+    public interface ICachedTwitterUserService : ITwitterUserService
+    {
+        void PurgeUser(string username);
+    }
+
+    public class CachedTwitterUserService : ICachedTwitterUserService
     {
         private readonly ITwitterUserService _twitterService;
 
@@ -38,5 +43,10 @@ namespace BirdsiteLive.Twitter
 
             return user;
         }
+
+        public void PurgeUser(string username)
+        {
+            _userCache.Remove(username);
+        }
     }
 }

src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs

@@ -23,7 +23,7 @@ namespace BirdsiteLive.Twitter.Extractors
                 InReplyToStatusId = tweet.InReplyToStatusId,
                 InReplyToAccount = tweet.InReplyToScreenName,
                 MessageContent = ExtractMessage(tweet),
-                Media = ExtractMedia(tweet.Media),
+                Media = ExtractMedia(tweet),
                 CreatedAt = tweet.CreatedAt.ToUniversalTime(),
                 IsReply = tweet.InReplyToUserId != null,
                 IsThread = tweet.InReplyToUserId != null && tweet.InReplyToUserId == tweet.CreatedBy.Id,
@@ -36,10 +36,17 @@ namespace BirdsiteLive.Twitter.Extractors
 
         private string ExtractRetweetUrl(ITweet tweet)
         {
-            if (tweet.IsRetweet && tweet.FullText.Contains("https://t.co/"))
+            if (tweet.IsRetweet)
             {
-                var retweetId = tweet.FullText.Split(new[] { "https://t.co/" }, StringSplitOptions.RemoveEmptyEntries).Last();
-                return $"https://t.co/{retweetId}";
+                if (tweet.RetweetedTweet != null)
+                {
+                    return tweet.RetweetedTweet.Url;
+                }
+                if (tweet.FullText.Contains("https://t.co/"))
+                {
+                    var retweetId = tweet.FullText.Split(new[] { "https://t.co/" }, StringSplitOptions.RemoveEmptyEntries).Last();
+                    return $"https://t.co/{retweetId}";
+                }
             }
 
             return null;
@@ -47,8 +54,15 @@ namespace BirdsiteLive.Twitter.Extractors
 
         public string ExtractMessage(ITweet tweet)
         {
-            var tweetUrls = tweet.Media.Select(x => x.URL).Distinct();
             var message = tweet.FullText;
+            var tweetUrls = tweet.Media.Select(x => x.URL).Distinct();
+            
+            if (tweet.IsRetweet && tweet.QuotedStatusId == null && message.StartsWith("RT") && tweet.RetweetedTweet != null)
+            {
+                message = tweet.RetweetedTweet.FullText;
+                tweetUrls = tweet.RetweetedTweet.Media.Select(x => x.URL).Distinct();
+            }
+
             foreach (var tweetUrl in tweetUrls)
             {
                 if(tweet.IsRetweet)
@@ -60,8 +74,10 @@ namespace BirdsiteLive.Twitter.Extractors
             if (tweet.QuotedTweet != null) message = $"[Quote {{RT}}]{Environment.NewLine}{message}";
             if (tweet.IsRetweet)
             {
-                if (tweet.RetweetedTweet != null)
+                if (tweet.RetweetedTweet != null && !message.StartsWith("RT"))
                     message = $"[{{RT}} @{tweet.RetweetedTweet.CreatedBy.ScreenName}]{Environment.NewLine}{message}";
+                else if (tweet.RetweetedTweet != null && message.StartsWith($"RT @{tweet.RetweetedTweet.CreatedBy.ScreenName}:"))
+                    message = message.Replace($"RT @{tweet.RetweetedTweet.CreatedBy.ScreenName}:", $"[{{RT}} @{tweet.RetweetedTweet.CreatedBy.ScreenName}]{Environment.NewLine}");
                 else
                     message = message.Replace("RT", "[{{RT}}]");
             }
@@ -73,10 +89,13 @@ namespace BirdsiteLive.Twitter.Extractors
             return message;
         }
 
-        public ExtractedMedia[] ExtractMedia(List<IMediaEntity> media)
+        public ExtractedMedia[] ExtractMedia(ITweet tweet)
         {
-            var result = new List<ExtractedMedia>();
+            var media = tweet.Media;
+            if (tweet.IsRetweet && tweet.RetweetedTweet != null)
+                media = tweet.RetweetedTweet.Media;
 
+            var result = new List<ExtractedMedia>();
             foreach (var m in media)
             {
                 var mediaUrl = GetMediaUrl(m);

src/BirdsiteLive.Twitter/TwitterTweetsService.cs

@@ -61,36 +61,30 @@ namespace BirdsiteLive.Twitter
         public ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1)
         {
             var tweets = new List<ITweet>();
-            try
+
+            _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized();
+            ExceptionHandler.SwallowWebExceptions = false;
+            TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended;
+
+            var user = _twitterUserService.GetUser(username);
+            if (user == null || user.Protected) return new ExtractedTweet[0];
+
+            if (fromTweetId == -1)
             {
-                _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized();
-                ExceptionHandler.SwallowWebExceptions = false;
-                TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended;
-
-                var user = _twitterUserService.GetUser(username);
-                if (user == null || user.Protected) return new ExtractedTweet[0];
-
-                if (fromTweetId == -1)
-                {
-                    var timeline = Timeline.GetUserTimeline(user.Id, nberTweets);
-                    _statisticsHandler.CalledTimelineApi();
-                    if (timeline != null) tweets.AddRange(timeline);
-                }
-                else
-                {
-                    var timelineRequestParameters = new UserTimelineParameters
-                    {
-                        SinceId = fromTweetId,
-                        MaximumNumberOfTweetsToRetrieve = nberTweets
-                    };
-                    var timeline = Timeline.GetUserTimeline(user.Id, timelineRequestParameters);
-                    _statisticsHandler.CalledTimelineApi();
-                    if (timeline != null) tweets.AddRange(timeline);
-                }
+                var timeline = Timeline.GetUserTimeline(user.Id, nberTweets);
+                _statisticsHandler.CalledTimelineApi();
+                if (timeline != null) tweets.AddRange(timeline);
             }
-            catch (Exception e)
+            else
             {
-                _logger.LogError(e, "Error retrieving timeline from {Username}, from {TweetId}", username, fromTweetId);
+                var timelineRequestParameters = new UserTimelineParameters
+                {
+                    SinceId = fromTweetId,
+                    MaximumNumberOfTweetsToRetrieve = nberTweets
+                };
+                var timeline = Timeline.GetUserTimeline(user.Id, timelineRequestParameters);
+                _statisticsHandler.CalledTimelineApi();
+                if (timeline != null) tweets.AddRange(timeline);
             }
 
             return tweets.Select(_tweetExtractor.Extract).ToArray();

src/BirdsiteLive/BirdsiteLive.csproj

@@ -4,7 +4,7 @@
     <TargetFramework>netcoreapp3.1</TargetFramework>
     <UserSecretsId>d21486de-a812-47eb-a419-05682bb68856</UserSecretsId>
     <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
-    <Version>0.12.2</Version>
+    <Version>0.13.0</Version>
   </PropertyGroup>
 
   <ItemGroup>

src/BirdsiteLive/Controllers/UsersController.cs

@@ -9,6 +9,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using BirdsiteLive.ActivityPub;
 using BirdsiteLive.ActivityPub.Models;
+using BirdsiteLive.Common.Regexes;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Domain;
 using BirdsiteLive.Models;
@@ -28,7 +29,6 @@ namespace BirdsiteLive.Controllers
         private readonly IUserService _userService;
         private readonly IStatusService _statusService;
         private readonly InstanceSettings _instanceSettings;
-        private readonly Regex _twitterAccountRegex = new Regex(@"^[a-zA-Z0-9_]+$");
 
         #region Ctor
         public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ITwitterTweetsService twitterTweetService)
@@ -62,7 +62,7 @@ namespace BirdsiteLive.Controllers
             // Ensure valid username 
             // https://help.twitter.com/en/managing-your-account/twitter-username-rules
             TwitterUser user = null;
-            if (!string.IsNullOrWhiteSpace(id) && _twitterAccountRegex.IsMatch(id) && id.Length <= 15)
+            if (!string.IsNullOrWhiteSpace(id) && UserRegexes.TwitterAccount.IsMatch(id) && id.Length <= 15)
                 user = _twitterUserService.GetUser(id);
 
             var acceptHeaders = Request.Headers["Accept"];

src/BirdsiteLive/Controllers/WellKnownController.cs

@@ -4,6 +4,7 @@ using System.Linq;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using BirdsiteLive.ActivityPub.Converters;
+using BirdsiteLive.Common.Regexes;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.Models;
@@ -20,7 +21,6 @@ namespace BirdsiteLive.Controllers
         private readonly ITwitterUserService _twitterUserService;
         private readonly ITwitterUserDal _twitterUserDal;
         private readonly InstanceSettings _settings;
-        private readonly Regex _twitterAccountRegex = new Regex(@"^[a-zA-Z0-9_]+$");
 
         #region Ctor
         public WellKnownController(InstanceSettings settings, ITwitterUserService twitterUserService, ITwitterUserDal twitterUserDal)
@@ -164,7 +164,7 @@ namespace BirdsiteLive.Controllers
 
             // Ensure valid username 
             // https://help.twitter.com/en/managing-your-account/twitter-username-rules
-            if (string.IsNullOrWhiteSpace(name) || !_twitterAccountRegex.IsMatch(name) || name.Length > 15 )
+            if (string.IsNullOrWhiteSpace(name) || !UserRegexes.TwitterAccount.IsMatch(name) || name.Length > 15 )
                 return NotFound();
 
             if (!string.IsNullOrWhiteSpace(domain) && domain != _settings.Domain)

src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs

@@ -3,7 +3,9 @@ using System.Linq;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Domain.Tools;
 using BirdsiteLive.Twitter.Models;
+using Microsoft.Extensions.Logging;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Moq;
 
 namespace BirdsiteLive.Domain.Tests.Tools
 {
@@ -28,11 +30,16 @@ namespace BirdsiteLive.Domain.Tests.Tools
             #region Stubs
             var message = "Bla.\n\n@Mention blo. https://t.co/pgtrJi9600";
             #endregion
-            
-            var service = new StatusExtractor(_settings);
+
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
             var result = service.Extract(message);
 
             #region Validations
+            logger.VerifyAll();
             Assert.IsTrue(result.content.Contains("Bla."));
             Assert.IsTrue(result.content.Contains("</p><p>"));
             #endregion
@@ -45,10 +52,15 @@ namespace BirdsiteLive.Domain.Tests.Tools
             var message = "Bla.\n@Mention blo. https://t.co/pgtrJi9600";
             #endregion
 
-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
             var result = service.Extract(message);
 
             #region Validations
+            logger.VerifyAll();
             Assert.IsTrue(result.content.Contains("Bla."));
             Assert.IsTrue(result.content.Contains("<br/>"));
             #endregion
@@ -61,10 +73,15 @@ namespace BirdsiteLive.Domain.Tests.Tools
             var message = $"Bla!{Environment.NewLine}https://t.co/L8BpyHgg25";
             #endregion
 
-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
             var result = service.Extract(message);
 
             #region Validations
+            logger.VerifyAll();
             Assert.AreEqual(0, result.tags.Length);
 
             Assert.IsTrue(result.content.Contains("Bla!"));
@@ -79,10 +96,15 @@ namespace BirdsiteLive.Domain.Tests.Tools
             var message = $"Bla!{Environment.NewLine}https://www.eff.org/deeplinks/2020/07/pact-act-not-solution-problem-harmful-online-content";
             #endregion
 
-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
             var result = service.Extract(message);
 
             #region Validations
+            logger.VerifyAll();
             Assert.AreEqual(0, result.tags.Length);
 
             Assert.IsTrue(result.content.Contains("Bla!"));
@@ -97,10 +119,15 @@ namespace BirdsiteLive.Domain.Tests.Tools
             var message = $"Bla!{Environment.NewLine}https://www.eff.org/deeplinks/2020/07/pact";
             #endregion
 
-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
             var result = service.Extract(message);
 
             #region Validations
+            logger.VerifyAll();
             Assert.AreEqual(0, result.tags.Length);
 
             Assert.IsTrue(result.content.Contains("Bla!"));
@@ -115,10 +142,15 @@ namespace BirdsiteLive.Domain.Tests.Tools
             var message = $"https://t.co/L8BpyHgg25 Bla!{Environment.NewLine}https://www.eff.org/deeplinks/2020/07/pact-act-not-solution-problem-harmful-online-content";
             #endregion
 
-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
             var result = service.Extract(message);
 
             #region Validations
+            logger.VerifyAll();
             Assert.AreEqual(0, result.tags.Length);
 
             Assert.IsTrue(result.content.Contains("Bla!"));
@@ -132,13 +164,18 @@ namespace BirdsiteLive.Domain.Tests.Tools
         public void Extract_SingleHashTag_Test()
         {
             #region Stubs
-            var message = $"Bla!{Environment.NewLine}#mytag⁠";
+            var message = $"Bla!{Environment.NewLine}#mytag";
             #endregion
 
-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
             var result = service.Extract(message);
 
             #region Validations
+            logger.VerifyAll();
             Assert.AreEqual(1, result.tags.Length);
             Assert.AreEqual("#mytag", result.tags.First().name);
             Assert.AreEqual("Hashtag", result.tags.First().type);
@@ -153,13 +190,18 @@ namespace BirdsiteLive.Domain.Tests.Tools
         public void Extract_SingleHashTag_AtStart_Test()
         {
             #region Stubs
-            var message = $"#mytag⁠ Bla!";
+            var message = "#mytag Bla!";
             #endregion
 
-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
             var result = service.Extract(message);
 
             #region Validations
+            logger.VerifyAll();
             Assert.AreEqual(1, result.tags.Length);
             Assert.AreEqual("#mytag", result.tags.First().name);
             Assert.AreEqual("Hashtag", result.tags.First().type);
@@ -174,20 +216,25 @@ namespace BirdsiteLive.Domain.Tests.Tools
         public void Extract_SingleHashTag_SpecialChar_Test()
         {
             #region Stubs
-            var message = $"Bla!{Environment.NewLine}#COVIDー19⁠";
+            var message = $"Bla!{Environment.NewLine}#COVID_19";
             #endregion
 
-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
             var result = service.Extract(message);
 
             #region Validations
+            logger.VerifyAll();
             Assert.AreEqual(1, result.tags.Length);
-            Assert.AreEqual("#COVIDー19", result.tags.First().name);
+            Assert.AreEqual("#COVID_19", result.tags.First().name);
             Assert.AreEqual("Hashtag", result.tags.First().type);
-            Assert.AreEqual("https://domain.name/tags/COVIDー19", result.tags.First().href);
+            Assert.AreEqual("https://domain.name/tags/COVID_19", result.tags.First().href);
 
             Assert.IsTrue(result.content.Contains("Bla!"));
-            Assert.IsTrue(result.content.Contains(@"<a href=""https://domain.name/tags/COVIDー19"" class=""mention hashtag"" rel=""tag"">#<span>COVIDー19</span></a>"));
+            Assert.IsTrue(result.content.Contains(@"<a href=""https://domain.name/tags/COVID_19"" class=""mention hashtag"" rel=""tag"">#<span>COVID_19</span></a>"));
             #endregion
         }
 
@@ -195,13 +242,18 @@ namespace BirdsiteLive.Domain.Tests.Tools
         public void Extract_MultiHashTags_Test()
         {
             #region Stubs
-            var message = $"Bla!{Environment.NewLine}#mytag #mytag2 #mytag3⁠{Environment.NewLine}Test #bal Test";
+            var message = $"Bla!{Environment.NewLine}#mytag #mytag2 #mytag3{Environment.NewLine}Test #bal Test";
             #endregion
 
-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
             var result = service.Extract(message);
 
             #region Validations
+            logger.VerifyAll();
             Assert.AreEqual(4, result.tags.Length);
             Assert.IsTrue(result.content.Contains("Bla!"));
             Assert.IsTrue(result.content.Contains(@"<a href=""https://domain.name/tags/mytag"" class=""mention hashtag"" rel=""tag"">#<span>mytag</span></a>"));
@@ -215,13 +267,18 @@ namespace BirdsiteLive.Domain.Tests.Tools
         public void Extract_SingleMentionTag_Test()
         {
             #region Stubs
-            var message = $"Bla!{Environment.NewLine}@mynickname⁠";
+            var message = $"Bla!{Environment.NewLine}@mynickname";
             #endregion
 
-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
             var result = service.Extract(message);
 
             #region Validations
+            logger.VerifyAll();
             Assert.AreEqual(1, result.tags.Length);
             Assert.AreEqual("@mynickname@domain.name", result.tags.First().name);
             Assert.AreEqual("Mention", result.tags.First().type);
@@ -233,16 +290,74 @@ namespace BirdsiteLive.Domain.Tests.Tools
         }
 
         [TestMethod]
-        public void Extract_SingleMentionTag_SpecialChar_Test()
+        public void Extract_SingleMentionTag_RT_Test()
         {
             #region Stubs
-            var message = $"Bla!{Environment.NewLine}@my___nickname⁠";
+            var message = $"[RT @mynickname]{Environment.NewLine}Bla!";
             #endregion
 
-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
             var result = service.Extract(message);
 
             #region Validations
+            logger.VerifyAll();
+            Assert.AreEqual(1, result.tags.Length);
+            Assert.AreEqual("@mynickname@domain.name", result.tags.First().name);
+            Assert.AreEqual("Mention", result.tags.First().type);
+            Assert.AreEqual("https://domain.name/users/mynickname", result.tags.First().href);
+
+            Assert.IsTrue(result.content.Contains("Bla!"));
+            Assert.IsTrue(result.content.Contains(@"<span class=""h-card""><a href=""https://domain.name/@mynickname"" class=""u-url mention"">@<span>mynickname</span></a></span>"));
+            #endregion
+        }
+
+        [TestMethod]
+        public void Extract_SingleMentionTag_Dot_Test()
+        {
+            #region Stubs
+            var message = $".@mynickname Bla!{Environment.NewLine}Blo";
+            #endregion
+
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
+            var result = service.Extract(message);
+
+            #region Validations
+            logger.VerifyAll();
+            Assert.AreEqual(1, result.tags.Length);
+            Assert.AreEqual("@mynickname@domain.name", result.tags.First().name);
+            Assert.AreEqual("Mention", result.tags.First().type);
+            Assert.AreEqual("https://domain.name/users/mynickname", result.tags.First().href);
+
+            Assert.IsTrue(result.content.Contains("Bla!"));
+            Assert.IsTrue(result.content.Contains("Blo"));
+            Assert.IsTrue(result.content.Contains(@"<span class=""h-card""><a href=""https://domain.name/@mynickname"" class=""u-url mention"">@<span>mynickname</span></a></span>"));
+            #endregion
+        }
+
+        [TestMethod]
+        public void Extract_SingleMentionTag_SpecialChar_Test()
+        {
+            #region Stubs
+            var message = $"Bla!{Environment.NewLine}@my___nickname";
+            #endregion
+
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
+            var result = service.Extract(message);
+
+            #region Validations
+            logger.VerifyAll();
             Assert.AreEqual(1, result.tags.Length);
             Assert.AreEqual("@my___nickname@domain.name", result.tags.First().name);
             Assert.AreEqual("Mention", result.tags.First().type);
@@ -260,10 +375,15 @@ namespace BirdsiteLive.Domain.Tests.Tools
             var message = $"@mynickname Bla!";
             #endregion
 
-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
             var result = service.Extract(message);
 
             #region Validations
+            logger.VerifyAll();
             Assert.AreEqual(1, result.tags.Length);
             Assert.AreEqual("@mynickname@domain.name", result.tags.First().name);
             Assert.AreEqual("Mention", result.tags.First().type);
@@ -281,10 +401,15 @@ namespace BirdsiteLive.Domain.Tests.Tools
             var message = $"Bla!{Environment.NewLine}@mynickname⁠ @mynickname2 @mynickname3{Environment.NewLine}Test @dada Test";
             #endregion
 
-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
             var result = service.Extract(message);
 
             #region Validations
+            logger.VerifyAll();
             Assert.AreEqual(4, result.tags.Length);
             Assert.IsTrue(result.content.Contains("Bla!"));
             Assert.IsTrue(result.content.Contains(@"<span class=""h-card""><a href=""https://domain.name/@mynickname"" class=""u-url mention"">@<span>mynickname</span></a></span>"));
@@ -301,10 +426,15 @@ namespace BirdsiteLive.Domain.Tests.Tools
             var message = $"Bla!{Environment.NewLine}@mynickname⁠ #mytag2 @mynickname3{Environment.NewLine}Test @dada #dada Test";
             #endregion
 
-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
             var result = service.Extract(message);
 
             #region Validations
+            logger.VerifyAll();
             Assert.AreEqual(5, result.tags.Length);
             Assert.IsTrue(result.content.Contains("Bla!"));
             Assert.IsTrue(result.content.Contains(@"<span class=""h-card""><a href=""https://domain.name/@mynickname"" class=""u-url mention"">@<span>mynickname</span></a></span>"));
@@ -314,8 +444,7 @@ namespace BirdsiteLive.Domain.Tests.Tools
             Assert.IsTrue(result.content.Contains(@"<a href=""https://domain.name/tags/dada"" class=""mention hashtag"" rel=""tag"">#<span>dada</span></a>"));
             #endregion
         }
-
-
+        
         [TestMethod]
         public void Extract_Emoji_Test()
         {
@@ -324,15 +453,20 @@ namespace BirdsiteLive.Domain.Tests.Tools
             //var message = $"tests@mynickname";
             #endregion
 
-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
             var result = service.Extract(message);
 
             #region Validations
+            logger.VerifyAll();
             Assert.AreEqual(1, result.tags.Length);
             Assert.IsTrue(result.content.Contains(
-                @"😤 <span class=""h-card""><a href=""https://domain.name/@mynickname"" class=""u-url mention"">@<span>mynickname</span></a></span>"));
+                @"😤<span class=""h-card""><a href=""https://domain.name/@mynickname"" class=""u-url mention"">@<span>mynickname</span></a></span>"));
 
-            Assert.IsTrue(result.content.Contains(@"😎 😍 🤗 🤩 😘"));
+            Assert.IsTrue(result.content.Contains(@"😎😍🤗🤩😘"));
             #endregion
         }
 
@@ -344,12 +478,17 @@ namespace BirdsiteLive.Domain.Tests.Tools
             //var message = $"tests@mynickname";
             #endregion
 
-            var service = new StatusExtractor(_settings);
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
             var result = service.Extract(message);
 
             #region Validations
+            logger.VerifyAll();
             Assert.AreEqual(1, result.tags.Length);
-            Assert.IsTrue(result.content.Equals(@"bla ( <span class=""h-card""><a href=""https://domain.name/@mynickname"" class=""u-url mention"">@<span>mynickname</span></a></span> test)"));
+            Assert.IsTrue(result.content.Equals(@"bla (<span class=""h-card""><a href=""https://domain.name/@mynickname"" class=""u-url mention"">@<span>mynickname</span></a></span> test)"));
             #endregion
         }
     }

src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj

@@ -14,12 +14,14 @@
     <PackageReference Include="coverlet.collector" Version="1.2.0" />
   </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\..\BirdsiteLive.Pipeline\BirdsiteLive.Pipeline.csproj" />
-  </ItemGroup>
-
   <ItemGroup>
     <Folder Include="Tools\" />
   </ItemGroup>
 
+  <ItemGroup>
+    <ProjectReference Include="..\..\BirdsiteLive.Pipeline\BirdsiteLive.Pipeline.csproj" />
+    <ProjectReference Include="..\..\BirdsiteLive.Twitter\BirdsiteLive.Twitter.csproj" />
+    <ProjectReference Include="..\..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
+  </ItemGroup>
+
 </Project>

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs

@@ -7,6 +7,7 @@ using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Pipeline.Processors;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
+using Microsoft.Extensions.Logging;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Moq;
 
@@ -59,14 +60,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                     It.IsAny<DateTime>()
                 ))
                 .Returns(Task.CompletedTask);
+
+            var twitterUserServiceMock = new Mock<ICachedTwitterUserService>(MockBehavior.Strict);
+
+            var logger = new Mock<ILogger<RetrieveTweetsProcessor>>(MockBehavior.Strict);
             #endregion
 
-            var processor = new RetrieveTweetsProcessor(twitterServiceMock.Object, twitterUserDalMock.Object);
+            var processor = new RetrieveTweetsProcessor(twitterServiceMock.Object, twitterUserDalMock.Object, twitterUserServiceMock.Object, logger.Object);
             var usersResult = await processor.ProcessAsync(users, CancellationToken.None);
 
             #region Validations
             twitterServiceMock.VerifyAll();
             twitterUserDalMock.VerifyAll();
+            twitterUserServiceMock.VerifyAll();
+            logger.VerifyAll();
 
             Assert.AreEqual(0, usersResult.Length);
             #endregion
@@ -117,14 +124,21 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 .Returns(tweets);
 
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
+
+            var twitterUserServiceMock = new Mock<ICachedTwitterUserService>(MockBehavior.Strict);
+
+            var logger = new Mock<ILogger<RetrieveTweetsProcessor>>(MockBehavior.Strict);
             #endregion
 
-            var processor = new RetrieveTweetsProcessor(twitterServiceMock.Object, twitterUserDalMock.Object);
+            var processor = new RetrieveTweetsProcessor(twitterServiceMock.Object, twitterUserDalMock.Object, twitterUserServiceMock.Object, logger.Object);
             var usersResult = await processor.ProcessAsync(users, CancellationToken.None);
 
             #region Validations
             twitterServiceMock.VerifyAll();
             twitterUserDalMock.VerifyAll();
+            twitterUserServiceMock.VerifyAll();
+            logger.VerifyAll();
+
 
             Assert.AreEqual(users.Length, usersResult.Length);
             Assert.AreEqual(users[0].Acct, usersResult[0].User.Acct);
@@ -177,14 +191,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 .Returns(tweets);
 
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
+
+            var twitterUserServiceMock = new Mock<ICachedTwitterUserService>(MockBehavior.Strict);
+
+            var logger = new Mock<ILogger<RetrieveTweetsProcessor>>(MockBehavior.Strict);
             #endregion
 
-            var processor = new RetrieveTweetsProcessor(twitterServiceMock.Object, twitterUserDalMock.Object);
+            var processor = new RetrieveTweetsProcessor(twitterServiceMock.Object, twitterUserDalMock.Object, twitterUserServiceMock.Object, logger.Object);
             var usersResult = await processor.ProcessAsync(users, CancellationToken.None);
 
             #region Validations
             twitterServiceMock.VerifyAll();
             twitterUserDalMock.VerifyAll();
+            twitterUserServiceMock.VerifyAll();
+            logger.VerifyAll();
 
             Assert.AreEqual(users.Length, usersResult.Length);
             Assert.AreEqual(users[0].Acct, usersResult[0].User.Acct);