diff --git a/src/BirdsiteLive.Domain/Statistics/ExtractionStatisticsHandler.cs b/src/BirdsiteLive.Domain/Statistics/ExtractionStatisticsHandler.cs new file mode 100644 index 0000000..b9446df --- /dev/null +++ b/src/BirdsiteLive.Domain/Statistics/ExtractionStatisticsHandler.cs @@ -0,0 +1,80 @@ +using System.Threading; +using System.Timers; + +namespace BirdsiteLive.Domain.Statistics +{ + public interface IExtractionStatisticsHandler + { + void ExtractedDescription(int mentionsCount); + void ExtractedStatus(int mentionsCount); + ExtractionStatistics GetStatistics(); + } + + public class ExtractionStatisticsHandler : IExtractionStatisticsHandler + { + private static int _lastDescriptionMentionsExtracted; + private static int _lastStatusMentionsExtracted; + + private static int _descriptionMentionsExtracted; + private static int _statusMentionsExtracted; + + private static System.Timers.Timer _resetTimer; + + #region Ctor + public ExtractionStatisticsHandler() + { + if (_resetTimer == null) + { + _resetTimer = new System.Timers.Timer(); + _resetTimer.Elapsed += OnTimeResetEvent; + _resetTimer.Interval = 24 * 60 * 60 * 1000; // 24h + _resetTimer.Enabled = true; + } + } + #endregion + + private void OnTimeResetEvent(object sender, ElapsedEventArgs e) + { + _lastDescriptionMentionsExtracted = _descriptionMentionsExtracted; + _lastStatusMentionsExtracted = _statusMentionsExtracted; + + // Reset + Interlocked.Exchange(ref _descriptionMentionsExtracted, 0); + Interlocked.Exchange(ref _statusMentionsExtracted, 0); + } + + public void ExtractedDescription(int mentionsCount) + { + Interlocked.Increment(ref _descriptionMentionsExtracted); + } + + public void ExtractedStatus(int mentionsCount) + { + Interlocked.Increment(ref _statusMentionsExtracted); + } + + public ExtractionStatistics GetStatistics() + { + return new ExtractionStatistics(_descriptionMentionsExtracted, _statusMentionsExtracted, _lastDescriptionMentionsExtracted, _lastStatusMentionsExtracted); + } + } + + public class ExtractionStatistics + { + #region Ctor + public ExtractionStatistics(int mentionsInDescriptionsExtraction, int mentionsInStatusesExtraction, int lastMentionsInDescriptionsExtraction, int lastMentionsInStatusesExtraction) + { + MentionsInDescriptionsExtraction = mentionsInDescriptionsExtraction; + MentionsInStatusesExtraction = mentionsInStatusesExtraction; + LastMentionsInDescriptionsExtraction = lastMentionsInDescriptionsExtraction; + LastMentionsInStatusesExtraction = lastMentionsInStatusesExtraction; + } + #endregion + + public int MentionsInDescriptionsExtraction { get; } + public int MentionsInStatusesExtraction { get; } + + public int LastMentionsInDescriptionsExtraction { get; } + public int LastMentionsInStatusesExtraction { get; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/StatusService.cs b/src/BirdsiteLive.Domain/StatusService.cs index ab48e92..a1b3d38 100644 --- a/src/BirdsiteLive.Domain/StatusService.cs +++ b/src/BirdsiteLive.Domain/StatusService.cs @@ -7,6 +7,7 @@ using BirdsiteLive.ActivityPub; using BirdsiteLive.ActivityPub.Converters; using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.Common.Settings; +using BirdsiteLive.Domain.Statistics; using BirdsiteLive.Domain.Tools; using BirdsiteLive.Twitter.Models; using Tweetinvi.Models; @@ -23,12 +24,14 @@ namespace BirdsiteLive.Domain { private readonly InstanceSettings _instanceSettings; private readonly IStatusExtractor _statusExtractor; - + private readonly IExtractionStatisticsHandler _statisticsHandler; + #region Ctor - public StatusService(InstanceSettings instanceSettings, IStatusExtractor statusExtractor) + public StatusService(InstanceSettings instanceSettings, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler) { _instanceSettings = instanceSettings; _statusExtractor = statusExtractor; + _statisticsHandler = statisticsHandler; } #endregion @@ -41,6 +44,7 @@ namespace BirdsiteLive.Domain var apPublic = "https://www.w3.org/ns/activitystreams#Public"; var extractedTags = _statusExtractor.ExtractTags(tweet.MessageContent); + _statisticsHandler.ExtractedStatus(extractedTags.tags.Count(x => x.type == "Mention")); string inReplyTo = null; if (tweet.InReplyToStatusId != default) diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index 3961bf4..6e498af 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -10,6 +10,7 @@ using BirdsiteLive.ActivityPub.Converters; using BirdsiteLive.Common.Settings; using BirdsiteLive.Cryptography; using BirdsiteLive.Domain.BusinessUseCases; +using BirdsiteLive.Domain.Statistics; using BirdsiteLive.Domain.Tools; using BirdsiteLive.Twitter.Models; using Tweetinvi.Core.Exceptions; @@ -33,9 +34,10 @@ namespace BirdsiteLive.Domain private readonly ICryptoService _cryptoService; private readonly IActivityPubService _activityPubService; private readonly IStatusExtractor _statusExtractor; + private readonly IExtractionStatisticsHandler _statisticsHandler; #region Ctor - public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor) + public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler) { _instanceSettings = instanceSettings; _cryptoService = cryptoService; @@ -43,6 +45,7 @@ namespace BirdsiteLive.Domain _processFollowUser = processFollowUser; _processUndoFollowUser = processUndoFollowUser; _statusExtractor = statusExtractor; + _statisticsHandler = statisticsHandler; //_host = $"https://{instanceSettings.Domain.Replace("https://",string.Empty).Replace("http://", string.Empty).TrimEnd('/')}"; } #endregion @@ -58,6 +61,8 @@ namespace BirdsiteLive.Domain { var extracted = _statusExtractor.ExtractTags(description); description = extracted.content; + + _statisticsHandler.ExtractedDescription(extracted.tags.Count(x => x.type == "Mention")); } var user = new Actor diff --git a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs index 2cb7dde..1d27ddb 100644 --- a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs +++ b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs @@ -45,6 +45,10 @@ namespace BirdsiteLive.Twitter.Extractors message = message.Replace("RT", "[RT]"); } + // Expand URLs + foreach (var url in tweet.Urls.OrderByDescending(x => x.URL.Length)) + message = message.Replace(url.URL, url.ExpandedURL); + return message; } diff --git a/src/BirdsiteLive.Twitter/Models/ApiStatistics.cs b/src/BirdsiteLive.Twitter/Models/ApiStatistics.cs index 55325ad..4d5b194 100644 --- a/src/BirdsiteLive.Twitter/Models/ApiStatistics.cs +++ b/src/BirdsiteLive.Twitter/Models/ApiStatistics.cs @@ -2,11 +2,17 @@ { public class ApiStatistics { - public int UserCallsCount { get; set; } + public int UserCallsCountMin { get; set; } + public int UserCallsCountAvg { get; set; } + public int UserCallsCountMax { get; set; } public int UserCallsMax { get; set; } - public int TweetCallsCount { get; set; } + public int TweetCallsCountMin { get; set; } + public int TweetCallsCountAvg { get; set; } + public int TweetCallsCountMax { get; set; } public int TweetCallsMax { get; set; } - public int TimelineCallsCount { get; set; } + public int TimelineCallsCountMin { get; set; } + public int TimelineCallsCountAvg { get; set; } + public int TimelineCallsCountMax { get; set; } public int TimelineCallsMax { get; set; } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs b/src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs index ef64fd1..afea0d0 100644 --- a/src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs +++ b/src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs @@ -1,4 +1,7 @@ -using System.Threading; +using System; +using System.Collections.Concurrent; +using System.Linq; +using System.Threading; using System.Timers; using BirdsiteLive.Twitter.Models; @@ -15,14 +18,12 @@ namespace BirdsiteLive.Statistics.Domain //Rate limits: https://developer.twitter.com/en/docs/twitter-api/v1/rate-limits public class TwitterStatisticsHandler : ITwitterStatisticsHandler { - private static int _previousUserCalls; - private static int _previousTweetCalls; - private static int _previousTimelineCalls; - private static int _userCalls; private static int _tweetCalls; private static int _timelineCalls; + private static ConcurrentDictionary _snapshots = new ConcurrentDictionary(); + private static System.Timers.Timer _resetTimer; #region Ctor @@ -40,13 +41,23 @@ namespace BirdsiteLive.Statistics.Domain private void OnTimeResetEvent(object sender, ElapsedEventArgs e) { - _previousUserCalls = _userCalls; - _previousTweetCalls = _tweetCalls; - _previousTimelineCalls = _timelineCalls; - + // Add snapshot + var snapshot = new ApiStatisticsSnapshot(_userCalls, _tweetCalls, _timelineCalls); + bool success; + do + { + success = _snapshots.TryAdd(snapshot.SnapshotDate, snapshot); + } while (!success); + + // Reset Interlocked.Exchange(ref _userCalls, 0); Interlocked.Exchange(ref _tweetCalls, 0); Interlocked.Exchange(ref _timelineCalls, 0); + + // Clean up + var now = DateTime.UtcNow; + var oldSnapshots = _snapshots.Keys.Where(x => (now - x).TotalHours > 24).ToList(); + foreach (var old in oldSnapshots) _snapshots.TryRemove(old, out var data); } public void CalledUserApi() //GET users/show - 900/15mins @@ -66,15 +77,44 @@ namespace BirdsiteLive.Statistics.Domain public ApiStatistics GetStatistics() { + var snapshots = _snapshots.Values.ToList(); + var userCalls = snapshots.Select(x => x.UserCalls).ToList(); + var tweetCalls = snapshots.Select(x => x.TweetCalls).ToList(); + var timelineCalls = snapshots.Select(x => x.TimelineCalls).ToList(); + return new ApiStatistics { - UserCallsCount = _previousUserCalls, + UserCallsCountMin = userCalls.Any() ? userCalls.Min() : 0, + UserCallsCountAvg = userCalls.Any() ? (int)userCalls.Average() : 0, + UserCallsCountMax = userCalls.Any() ? userCalls.Max() : 0, UserCallsMax = 900, - TweetCallsCount = _previousTweetCalls, + TweetCallsCountMin = tweetCalls.Any() ? tweetCalls.Min() : 0, + TweetCallsCountAvg = tweetCalls.Any() ? (int)tweetCalls.Average() : 0, + TweetCallsCountMax = tweetCalls.Any() ? tweetCalls.Max() : 0, TweetCallsMax = 300, - TimelineCallsCount = _previousTimelineCalls, + TimelineCallsCountMin = timelineCalls.Any() ? timelineCalls.Min() : 0, + TimelineCallsCountAvg = timelineCalls.Any() ? (int)timelineCalls.Average() : 0, + TimelineCallsCountMax = timelineCalls.Any() ? timelineCalls.Max() : 0, TimelineCallsMax = 1500 }; } } + + internal class ApiStatisticsSnapshot + { + #region Ctor + public ApiStatisticsSnapshot(int userCalls, int tweetCalls, int timelineCalls) + { + UserCalls = userCalls; + TweetCalls = tweetCalls; + TimelineCalls = timelineCalls; + SnapshotDate = DateTime.UtcNow; + } + #endregion + + public DateTime SnapshotDate { get; } + public int UserCalls { get; set; } + public int TweetCalls { get; set; } + public int TimelineCalls { get; set; } + } } \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/TwitterService.cs b/src/BirdsiteLive.Twitter/TwitterService.cs index 47f799c..b4fdd92 100644 --- a/src/BirdsiteLive.Twitter/TwitterService.cs +++ b/src/BirdsiteLive.Twitter/TwitterService.cs @@ -43,11 +43,16 @@ namespace BirdsiteLive.Twitter _statisticsHandler.CalledUserApi(); if (user == null) return null; + // Expand URLs + var description = user.Description; + foreach (var descriptionUrl in user.Entities?.Description?.Urls?.OrderByDescending(x => x.URL.Length)) + description = description.Replace(descriptionUrl.URL, descriptionUrl.ExpandedURL); + return new TwitterUser { Acct = username, Name = user.Name, - Description = user.Description, + Description = description, Url = $"https://twitter.com/{username}", ProfileImageUrl = user.ProfileImageUrlFullSize, ProfileBackgroundImageUrl = user.ProfileBackgroundImageUrlHttps, diff --git a/src/BirdsiteLive/Controllers/StatisticsController.cs b/src/BirdsiteLive/Controllers/StatisticsController.cs index 58e4a7a..5a9aef8 100644 --- a/src/BirdsiteLive/Controllers/StatisticsController.cs +++ b/src/BirdsiteLive/Controllers/StatisticsController.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.Domain.Statistics; using BirdsiteLive.Statistics.Domain; using Microsoft.AspNetCore.Mvc; @@ -13,13 +14,16 @@ namespace BirdsiteLive.Controllers private readonly ITwitterUserDal _twitterUserDal; private readonly IFollowersDal _followersDal; private readonly ITwitterStatisticsHandler _twitterStatistics; + private readonly IExtractionStatisticsHandler _extractionStatistics; + #region Ctor - public StatisticsController(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, ITwitterStatisticsHandler twitterStatistics) + public StatisticsController(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, ITwitterStatisticsHandler twitterStatistics, IExtractionStatisticsHandler extractionStatistics) { _twitterUserDal = twitterUserDal; _followersDal = followersDal; _twitterStatistics = twitterStatistics; + _extractionStatistics = extractionStatistics; } #endregion @@ -29,7 +33,8 @@ namespace BirdsiteLive.Controllers { FollowersCount = await _followersDal.GetFollowersCountAsync(), TwitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync(), - TwitterStatistics = _twitterStatistics.GetStatistics() + TwitterStatistics = _twitterStatistics.GetStatistics(), + ExtractionStatistics = _extractionStatistics.GetStatistics(), }; return View(stats); } diff --git a/src/BirdsiteLive/Models/StatisticsModels/Statistics.cs b/src/BirdsiteLive/Models/StatisticsModels/Statistics.cs index 59cff91..a2dd693 100644 --- a/src/BirdsiteLive/Models/StatisticsModels/Statistics.cs +++ b/src/BirdsiteLive/Models/StatisticsModels/Statistics.cs @@ -1,4 +1,5 @@ -using BirdsiteLive.Twitter.Models; +using BirdsiteLive.Domain.Statistics; +using BirdsiteLive.Twitter.Models; namespace BirdsiteLive.Models.StatisticsModels { @@ -7,5 +8,6 @@ namespace BirdsiteLive.Models.StatisticsModels public int FollowersCount { get; set; } public int TwitterUserCount { get; set; } public ApiStatistics TwitterStatistics { get; set; } + public ExtractionStatistics ExtractionStatistics { get; set; } } } \ No newline at end of file diff --git a/src/BirdsiteLive/Views/Statistics/Index.cshtml b/src/BirdsiteLive/Views/Statistics/Index.cshtml index 90a5e02..9a0b536 100644 --- a/src/BirdsiteLive/Views/Statistics/Index.cshtml +++ b/src/BirdsiteLive/Views/Statistics/Index.cshtml @@ -12,9 +12,17 @@
  • Followers: @Model.FollowersCount
    • -

    Twitter API

    +

    Twitter API (Min, Avg, Max for the last 24h)

    + +

    Mentions Extraction

    + \ No newline at end of file