From 1d5df9a83b54e73c2fad4479f50f22081dd3fece Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 29 Jan 2021 23:10:02 -0500 Subject: [PATCH 1/6] make user retrieval more resilient --- src/BirdsiteLive.Twitter/TwitterTweetsService.cs | 8 ++++---- src/BirdsiteLive.Twitter/TwitterUserService.cs | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs index e67b125..38d1d68 100644 --- a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs +++ b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs @@ -59,13 +59,13 @@ namespace BirdsiteLive.Twitter public ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1) { TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended; - - var user = _twitterUserService.GetUser(username); - if (user.Protected) return new ExtractedTweet[0]; - + var tweets = new List(); try { + var user = _twitterUserService.GetUser(username); + if (user == null || user.Protected) return new ExtractedTweet[0]; + if (fromTweetId == -1) { var timeline = Timeline.GetUserTimeline(user.Id, nberTweets); diff --git a/src/BirdsiteLive.Twitter/TwitterUserService.cs b/src/BirdsiteLive.Twitter/TwitterUserService.cs index e7d44c3..e8729a6 100644 --- a/src/BirdsiteLive.Twitter/TwitterUserService.cs +++ b/src/BirdsiteLive.Twitter/TwitterUserService.cs @@ -38,7 +38,11 @@ namespace BirdsiteLive.Twitter { user = User.GetUserFromScreenName(username); _statisticsHandler.CalledUserApi(); - if (user == null) return null; + if (user == null) + { + _logger.LogWarning("User {username} not found", username); + return null; + } } catch (Exception e) { From 62a88e40c69f98879aa365a4f840f8b92272a9ec Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 30 Jan 2021 00:05:52 -0500 Subject: [PATCH 2/6] ensure to not swallowExceptions (thread safety) --- src/BirdsiteLive.Twitter/TwitterTweetsService.cs | 3 ++- src/BirdsiteLive.Twitter/TwitterUserService.cs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs index 38d1d68..7fdf391 100644 --- a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs +++ b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs @@ -35,7 +35,6 @@ namespace BirdsiteLive.Twitter _twitterUserService = twitterUserService; _logger = logger; Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true); - ExceptionHandler.SwallowWebExceptions = false; } #endregion @@ -43,6 +42,7 @@ namespace BirdsiteLive.Twitter { try { + ExceptionHandler.SwallowWebExceptions = false; TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended; var tweet = Tweet.GetTweet(statusId); _statisticsHandler.CalledTweetApi(); @@ -58,6 +58,7 @@ namespace BirdsiteLive.Twitter public ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1) { + ExceptionHandler.SwallowWebExceptions = false; TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended; var tweets = new List(); diff --git a/src/BirdsiteLive.Twitter/TwitterUserService.cs b/src/BirdsiteLive.Twitter/TwitterUserService.cs index e8729a6..4bb9448 100644 --- a/src/BirdsiteLive.Twitter/TwitterUserService.cs +++ b/src/BirdsiteLive.Twitter/TwitterUserService.cs @@ -27,12 +27,13 @@ namespace BirdsiteLive.Twitter _statisticsHandler = statisticsHandler; _logger = logger; Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true); - ExceptionHandler.SwallowWebExceptions = false; } #endregion public TwitterUser GetUser(string username) { + ExceptionHandler.SwallowWebExceptions = false; + IUser user; try { From 6a867f23055a30085a1adc579664a49178510d26 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 30 Jan 2021 00:22:29 -0500 Subject: [PATCH 3/6] removed dangerous initialization --- .../BirdsiteLive.Twitter.csproj | 4 ++ .../Tools/TwitterAuthenticationInitializer.cs | 64 +++++++++++++++++++ .../TwitterTweetsService.cs | 17 +++-- .../TwitterUserService.cs | 9 +-- src/BirdsiteLive/Startup.cs | 3 + 5 files changed, 86 insertions(+), 11 deletions(-) create mode 100644 src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs diff --git a/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj b/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj index 377d04d..438b4e1 100644 --- a/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj +++ b/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj @@ -13,4 +13,8 @@ + + + + diff --git a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs new file mode 100644 index 0000000..28f8fee --- /dev/null +++ b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs @@ -0,0 +1,64 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.Common.Settings; +using Microsoft.Extensions.Logging; +using Tweetinvi; + +namespace BirdsiteLive.Twitter.Tools +{ + public interface ITwitterAuthenticationInitializer + { + void EnsureAuthenticationIsInitialized(); + } + + public class TwitterAuthenticationInitializer : ITwitterAuthenticationInitializer + { + private readonly TwitterSettings _settings; + private readonly ILogger _logger; + private static bool _initialized; + private readonly SemaphoreSlim _semaphoregate = new SemaphoreSlim(1); + + #region Ctor + public TwitterAuthenticationInitializer(TwitterSettings settings, ILogger logger) + { + _settings = settings; + _logger = logger; + } + #endregion + + public void EnsureAuthenticationIsInitialized() + { + if (_initialized) return; + _semaphoregate.Wait(); + if (_initialized) return; + + try + { + InitTwitterCredentials(); + } + finally + { + _semaphoregate.Release(); + } + } + + private void InitTwitterCredentials() + { + for (;;) + { + try + { + Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true); + _initialized = true; + return; + } + catch (Exception e) + { + _logger.LogError(e, "Twitter Authentication Failed"); + Thread.Sleep(250); + } + } + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs index 7fdf391..49684e8 100644 --- a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs +++ b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs @@ -5,6 +5,7 @@ using BirdsiteLive.Common.Settings; using BirdsiteLive.Statistics.Domain; using BirdsiteLive.Twitter.Extractors; using BirdsiteLive.Twitter.Models; +using BirdsiteLive.Twitter.Tools; using Microsoft.Extensions.Logging; using Tweetinvi; using Tweetinvi.Models; @@ -20,21 +21,20 @@ namespace BirdsiteLive.Twitter public class TwitterTweetsService : ITwitterTweetsService { - private readonly TwitterSettings _settings; + private readonly ITwitterAuthenticationInitializer _twitterAuthenticationInitializer; private readonly ITweetExtractor _tweetExtractor; private readonly ITwitterStatisticsHandler _statisticsHandler; private readonly ITwitterUserService _twitterUserService; private readonly ILogger _logger; #region Ctor - public TwitterTweetsService(TwitterSettings settings, ITweetExtractor tweetExtractor, ITwitterStatisticsHandler statisticsHandler, ITwitterUserService twitterUserService, ILogger logger) + public TwitterTweetsService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITweetExtractor tweetExtractor, ITwitterStatisticsHandler statisticsHandler, ITwitterUserService twitterUserService, ILogger logger) { - _settings = settings; + _twitterAuthenticationInitializer = twitterAuthenticationInitializer; _tweetExtractor = tweetExtractor; _statisticsHandler = statisticsHandler; _twitterUserService = twitterUserService; _logger = logger; - Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true); } #endregion @@ -42,8 +42,10 @@ namespace BirdsiteLive.Twitter { try { + _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized(); ExceptionHandler.SwallowWebExceptions = false; TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended; + var tweet = Tweet.GetTweet(statusId); _statisticsHandler.CalledTweetApi(); if (tweet == null) return null; //TODO: test this @@ -58,12 +60,13 @@ namespace BirdsiteLive.Twitter public ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1) { - ExceptionHandler.SwallowWebExceptions = false; - TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended; - var tweets = new List(); try { + _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized(); + ExceptionHandler.SwallowWebExceptions = false; + TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended; + var user = _twitterUserService.GetUser(username); if (user == null || user.Protected) return new ExtractedTweet[0]; diff --git a/src/BirdsiteLive.Twitter/TwitterUserService.cs b/src/BirdsiteLive.Twitter/TwitterUserService.cs index 4bb9448..dd80c4d 100644 --- a/src/BirdsiteLive.Twitter/TwitterUserService.cs +++ b/src/BirdsiteLive.Twitter/TwitterUserService.cs @@ -3,6 +3,7 @@ using System.Linq; using BirdsiteLive.Common.Settings; using BirdsiteLive.Statistics.Domain; using BirdsiteLive.Twitter.Models; +using BirdsiteLive.Twitter.Tools; using Microsoft.Extensions.Logging; using Tweetinvi; using Tweetinvi.Models; @@ -16,22 +17,22 @@ namespace BirdsiteLive.Twitter public class TwitterUserService : ITwitterUserService { - private readonly TwitterSettings _settings; + private readonly ITwitterAuthenticationInitializer _twitterAuthenticationInitializer; private readonly ITwitterStatisticsHandler _statisticsHandler; private readonly ILogger _logger; #region Ctor - public TwitterUserService(TwitterSettings settings, ITwitterStatisticsHandler statisticsHandler, ILogger logger) + public TwitterUserService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITwitterStatisticsHandler statisticsHandler, ILogger logger) { - _settings = settings; + _twitterAuthenticationInitializer = twitterAuthenticationInitializer; _statisticsHandler = statisticsHandler; _logger = logger; - Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true); } #endregion public TwitterUser GetUser(string username) { + _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized(); ExceptionHandler.SwallowWebExceptions = false; IUser user; diff --git a/src/BirdsiteLive/Startup.cs b/src/BirdsiteLive/Startup.cs index 371e969..ab50932 100644 --- a/src/BirdsiteLive/Startup.cs +++ b/src/BirdsiteLive/Startup.cs @@ -10,6 +10,7 @@ using BirdsiteLive.DAL.Postgres.DataAccessLayers; using BirdsiteLive.DAL.Postgres.Settings; using BirdsiteLive.Models; using BirdsiteLive.Twitter; +using BirdsiteLive.Twitter.Tools; using Lamar; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; @@ -87,6 +88,8 @@ namespace BirdsiteLive services.For().DecorateAllWith(); services.For().Use().Singleton(); + services.For().Use().Singleton(); + services.Scan(_ => { _.Assembly("BirdsiteLive.Twitter"); From cd7bc3b21624453bdc23cbf3408a28e122a48a63 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 30 Jan 2021 00:46:45 -0500 Subject: [PATCH 4/6] liberate the semaphore! --- .../Tools/TwitterAuthenticationInitializer.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs index 28f8fee..8484c3d 100644 --- a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs +++ b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs @@ -31,10 +31,10 @@ namespace BirdsiteLive.Twitter.Tools { if (_initialized) return; _semaphoregate.Wait(); - if (_initialized) return; - + try { + if (_initialized) return; InitTwitterCredentials(); } finally From 10c1da4a3492d2c6f8b987a3a4ea8478d314e4c0 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 30 Jan 2021 00:59:21 -0500 Subject: [PATCH 5/6] road to 0.12.2 --- src/BirdsiteLive/BirdsiteLive.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index 3766f3a..cef4948 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -4,7 +4,7 @@ netcoreapp3.1 d21486de-a812-47eb-a419-05682bb68856 Linux - 0.12.1 + 0.12.2 From 52e2868debcb9aabcec0c7b7b459f9ed8efb7742 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 30 Jan 2021 01:28:20 -0500 Subject: [PATCH 6/6] ensure valide username pattern, fix #75 --- src/BirdsiteLive/Controllers/UsersController.cs | 10 +++++++++- src/BirdsiteLive/Controllers/WellKnownController.cs | 7 +++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index d291c67..a22ae73 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -4,6 +4,7 @@ using System.IO; using System.Linq; using System.Net.Mime; using System.Runtime.InteropServices.WindowsRuntime; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using BirdsiteLive.ActivityPub; @@ -12,6 +13,7 @@ using BirdsiteLive.Common.Settings; using BirdsiteLive.Domain; using BirdsiteLive.Models; using BirdsiteLive.Twitter; +using BirdsiteLive.Twitter.Models; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Primitives; @@ -26,6 +28,7 @@ namespace BirdsiteLive.Controllers private readonly IUserService _userService; private readonly IStatusService _statusService; private readonly InstanceSettings _instanceSettings; + private readonly Regex _twitterAccountRegex = new Regex(@"^[a-zA-Z0-9_]+$"); #region Ctor public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ITwitterTweetsService twitterTweetService) @@ -55,7 +58,12 @@ namespace BirdsiteLive.Controllers public IActionResult Index(string id) { id = id.Trim(new[] { ' ', '@' }).ToLowerInvariant(); - var user = _twitterUserService.GetUser(id); + + // Ensure valid username + // https://help.twitter.com/en/managing-your-account/twitter-username-rules + TwitterUser user = null; + if (!string.IsNullOrWhiteSpace(id) && _twitterAccountRegex.IsMatch(id) && id.Length <= 15) + user = _twitterUserService.GetUser(id); var acceptHeaders = Request.Headers["Accept"]; if (acceptHeaders.Any()) diff --git a/src/BirdsiteLive/Controllers/WellKnownController.cs b/src/BirdsiteLive/Controllers/WellKnownController.cs index 553d8e7..3f060a7 100644 --- a/src/BirdsiteLive/Controllers/WellKnownController.cs +++ b/src/BirdsiteLive/Controllers/WellKnownController.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.RegularExpressions; using System.Threading.Tasks; using BirdsiteLive.ActivityPub.Converters; using BirdsiteLive.Common.Settings; @@ -19,6 +20,7 @@ namespace BirdsiteLive.Controllers private readonly ITwitterUserService _twitterUserService; private readonly ITwitterUserDal _twitterUserDal; private readonly InstanceSettings _settings; + private readonly Regex _twitterAccountRegex = new Regex(@"^[a-zA-Z0-9_]+$"); #region Ctor public WellKnownController(InstanceSettings settings, ITwitterUserService twitterUserService, ITwitterUserDal twitterUserDal) @@ -160,6 +162,11 @@ namespace BirdsiteLive.Controllers // Ensure lowercase name = name.ToLowerInvariant(); + // Ensure valid username + // https://help.twitter.com/en/managing-your-account/twitter-username-rules + if (string.IsNullOrWhiteSpace(name) || !_twitterAccountRegex.IsMatch(name) || name.Length > 15 ) + return NotFound(); + if (!string.IsNullOrWhiteSpace(domain) && domain != _settings.Domain) return NotFound();