Merge pull request #50 from NicolasConstant/develop

0.7.0 PR
2021-01-14 06:42:45 +01:00 · 2021-01-14 06:42:45 +01:00 · 72f6337b8e
commit 72f6337b8e
parent ce5c843be6 f0a0bb9f43
11 changed files with 186 additions and 27 deletions
--- a/src/BirdsiteLive.Domain/Statistics/ExtractionStatisticsHandler.cs
+++ b/src/BirdsiteLive.Domain/Statistics/ExtractionStatisticsHandler.cs
@ -0,0 +1,80 @@
+using System.Threading;
+using System.Timers;
+
+namespace BirdsiteLive.Domain.Statistics
+{
+    public interface IExtractionStatisticsHandler
+    {
+        void ExtractedDescription(int mentionsCount);
+        void ExtractedStatus(int mentionsCount);
+        ExtractionStatistics GetStatistics();
+    }
+    
+    public class ExtractionStatisticsHandler : IExtractionStatisticsHandler
+    {
+        private static int _lastDescriptionMentionsExtracted;
+        private static int _lastStatusMentionsExtracted;
+
+        private static int _descriptionMentionsExtracted;
+        private static int _statusMentionsExtracted;
+        
+        private static System.Timers.Timer _resetTimer;
+
+        #region Ctor
+        public ExtractionStatisticsHandler()
+        {
+            if (_resetTimer == null)
+            {
+                _resetTimer = new System.Timers.Timer();
+                _resetTimer.Elapsed += OnTimeResetEvent;
+                _resetTimer.Interval = 24 * 60 * 60 * 1000; // 24h
+                _resetTimer.Enabled = true;
+            }
+        }
+        #endregion
+
+        private void OnTimeResetEvent(object sender, ElapsedEventArgs e)
+        {
+            _lastDescriptionMentionsExtracted = _descriptionMentionsExtracted;
+            _lastStatusMentionsExtracted = _statusMentionsExtracted;
+
+            // Reset
+            Interlocked.Exchange(ref _descriptionMentionsExtracted, 0);
+            Interlocked.Exchange(ref _statusMentionsExtracted, 0);
+        }
+
+        public void ExtractedDescription(int mentionsCount)
+        {
+            Interlocked.Increment(ref _descriptionMentionsExtracted);
+        }
+
+        public void ExtractedStatus(int mentionsCount)
+        {
+            Interlocked.Increment(ref _statusMentionsExtracted);
+        }
+
+        public ExtractionStatistics GetStatistics()
+        {
+            return new ExtractionStatistics(_descriptionMentionsExtracted, _statusMentionsExtracted, _lastDescriptionMentionsExtracted, _lastStatusMentionsExtracted);
+        }
+    }
+
+    public class ExtractionStatistics
+    {
+        #region Ctor
+        public ExtractionStatistics(int mentionsInDescriptionsExtraction, int mentionsInStatusesExtraction, int lastMentionsInDescriptionsExtraction, int lastMentionsInStatusesExtraction)
+        {
+            MentionsInDescriptionsExtraction = mentionsInDescriptionsExtraction;
+            MentionsInStatusesExtraction = mentionsInStatusesExtraction;
+            LastMentionsInDescriptionsExtraction = lastMentionsInDescriptionsExtraction;
+            LastMentionsInStatusesExtraction = lastMentionsInStatusesExtraction;
+        }
+        #endregion
+
+        public int MentionsInDescriptionsExtraction { get; }
+        public int MentionsInStatusesExtraction { get; }
+
+        public int LastMentionsInDescriptionsExtraction { get; }
+        public int LastMentionsInStatusesExtraction { get; }
+    }
+}
--- a/src/BirdsiteLive.Domain/StatusService.cs
+++ b/src/BirdsiteLive.Domain/StatusService.cs
@ -7,6 +7,7 @@ using BirdsiteLive.ActivityPub;
 using BirdsiteLive.ActivityPub.Converters;
 using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.Common.Settings;
+using BirdsiteLive.Domain.Statistics;
 using BirdsiteLive.Domain.Tools;
 using BirdsiteLive.Twitter.Models;
 using Tweetinvi.Models;
@ -23,12 +24,14 @@ namespace BirdsiteLive.Domain
    {
        private readonly InstanceSettings _instanceSettings;
        private readonly IStatusExtractor _statusExtractor;
-
+        private readonly IExtractionStatisticsHandler _statisticsHandler;
+        
        #region Ctor
-        public StatusService(InstanceSettings instanceSettings, IStatusExtractor statusExtractor)
+        public StatusService(InstanceSettings instanceSettings, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler)
        {
            _instanceSettings = instanceSettings;
            _statusExtractor = statusExtractor;
+            _statisticsHandler = statisticsHandler;
        }
        #endregion

@ -41,6 +44,7 @@ namespace BirdsiteLive.Domain
            var apPublic = "https://www.w3.org/ns/activitystreams#Public";

            var extractedTags = _statusExtractor.ExtractTags(tweet.MessageContent);
+            _statisticsHandler.ExtractedStatus(extractedTags.tags.Count(x => x.type == "Mention"));

            string inReplyTo = null;
            if (tweet.InReplyToStatusId != default)
--- a/src/BirdsiteLive.Domain/UserService.cs
+++ b/src/BirdsiteLive.Domain/UserService.cs
@ -10,6 +10,7 @@ using BirdsiteLive.ActivityPub.Converters;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Cryptography;
 using BirdsiteLive.Domain.BusinessUseCases;
+using BirdsiteLive.Domain.Statistics;
 using BirdsiteLive.Domain.Tools;
 using BirdsiteLive.Twitter.Models;
 using Tweetinvi.Core.Exceptions;
@ -33,9 +34,10 @@ namespace BirdsiteLive.Domain
        private readonly ICryptoService _cryptoService;
        private readonly IActivityPubService _activityPubService;
        private readonly IStatusExtractor _statusExtractor;
+        private readonly IExtractionStatisticsHandler _statisticsHandler;

        #region Ctor
-        public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor)
+        public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler)
        {
            _instanceSettings = instanceSettings;
            _cryptoService = cryptoService;
@ -43,6 +45,7 @@ namespace BirdsiteLive.Domain
            _processFollowUser = processFollowUser;
            _processUndoFollowUser = processUndoFollowUser;
            _statusExtractor = statusExtractor;
+            _statisticsHandler = statisticsHandler;
            //_host = $"https://{instanceSettings.Domain.Replace("https://",string.Empty).Replace("http://", string.Empty).TrimEnd('/')}";
        }
        #endregion
@ -58,6 +61,8 @@ namespace BirdsiteLive.Domain
            {
                var extracted = _statusExtractor.ExtractTags(description);
                description = extracted.content;
+
+                _statisticsHandler.ExtractedDescription(extracted.tags.Count(x => x.type == "Mention"));
            }

            var user = new Actor
--- a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs
+++ b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs
@ -45,6 +45,10 @@ namespace BirdsiteLive.Twitter.Extractors
                    message = message.Replace("RT", "[RT]");
            }

+            // Expand URLs
+            foreach (var url in tweet.Urls.OrderByDescending(x => x.URL.Length))
+                message = message.Replace(url.URL, url.ExpandedURL);
+
            return message;
        }

--- a/src/BirdsiteLive.Twitter/Models/ApiStatistics.cs
+++ b/src/BirdsiteLive.Twitter/Models/ApiStatistics.cs
@ -2,11 +2,17 @@
 {
    public class ApiStatistics
    {
-        public int UserCallsCount { get; set; }
+        public int UserCallsCountMin { get; set; }
+        public int UserCallsCountAvg { get; set; }
+        public int UserCallsCountMax { get; set; }
        public int UserCallsMax { get; set; }
-        public int TweetCallsCount { get; set; }
+        public int TweetCallsCountMin { get; set; }
+        public int TweetCallsCountAvg { get; set; }
+        public int TweetCallsCountMax { get; set; }
        public int TweetCallsMax { get; set; }
-        public int TimelineCallsCount { get; set; }
+        public int TimelineCallsCountMin { get; set; }
+        public int TimelineCallsCountAvg { get; set; }
+        public int TimelineCallsCountMax { get; set; }
        public int TimelineCallsMax { get; set; }
    }
 }
--- a/src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs
+++ b/src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs
@ -1,4 +1,7 @@
-using System.Threading;
+using System;
+using System.Collections.Concurrent;
+using System.Linq;
+using System.Threading;
 using System.Timers;
 using BirdsiteLive.Twitter.Models;

@ -15,14 +18,12 @@ namespace BirdsiteLive.Statistics.Domain
    //Rate limits: https://developer.twitter.com/en/docs/twitter-api/v1/rate-limits
    public class TwitterStatisticsHandler : ITwitterStatisticsHandler
    {
-        private static int _previousUserCalls;
-        private static int _previousTweetCalls;
-        private static int _previousTimelineCalls;
-
        private static int _userCalls;
        private static int _tweetCalls;
        private static int _timelineCalls;

+        private static ConcurrentDictionary<DateTime, ApiStatisticsSnapshot> _snapshots = new ConcurrentDictionary<DateTime, ApiStatisticsSnapshot>();
+
        private static System.Timers.Timer _resetTimer;

        #region Ctor
@ -40,13 +41,23 @@ namespace BirdsiteLive.Statistics.Domain

        private void OnTimeResetEvent(object sender, ElapsedEventArgs e)
        {
-            _previousUserCalls = _userCalls;
-            _previousTweetCalls = _tweetCalls;
-            _previousTimelineCalls = _timelineCalls;
-
+            // Add snapshot
+            var snapshot = new ApiStatisticsSnapshot(_userCalls, _tweetCalls, _timelineCalls);
+            bool success;
+            do
+            {
+                success = _snapshots.TryAdd(snapshot.SnapshotDate, snapshot);
+            } while (!success);
+            
+            // Reset
            Interlocked.Exchange(ref _userCalls, 0);
            Interlocked.Exchange(ref _tweetCalls, 0);
            Interlocked.Exchange(ref _timelineCalls, 0);
+
+            // Clean up 
+            var now = DateTime.UtcNow;
+            var oldSnapshots = _snapshots.Keys.Where(x => (now - x).TotalHours > 24).ToList();
+            foreach (var old in oldSnapshots) _snapshots.TryRemove(old, out var data);
        }

        public void CalledUserApi()  //GET users/show - 900/15mins
@ -66,15 +77,44 @@ namespace BirdsiteLive.Statistics.Domain

        public ApiStatistics GetStatistics()
        {
+            var snapshots = _snapshots.Values.ToList();
+            var userCalls = snapshots.Select(x => x.UserCalls).ToList();
+            var tweetCalls = snapshots.Select(x => x.TweetCalls).ToList();
+            var timelineCalls = snapshots.Select(x => x.TimelineCalls).ToList();
+
            return new ApiStatistics
            {
-                UserCallsCount = _previousUserCalls,
+                UserCallsCountMin = userCalls.Any() ? userCalls.Min() : 0,
+                UserCallsCountAvg = userCalls.Any() ? (int)userCalls.Average() : 0,
+                UserCallsCountMax = userCalls.Any() ? userCalls.Max() : 0,
                UserCallsMax = 900,
-                TweetCallsCount = _previousTweetCalls,
+                TweetCallsCountMin = tweetCalls.Any() ? tweetCalls.Min() : 0,
+                TweetCallsCountAvg = tweetCalls.Any() ? (int)tweetCalls.Average() : 0,
+                TweetCallsCountMax = tweetCalls.Any() ? tweetCalls.Max() : 0,
                TweetCallsMax = 300,
-                TimelineCallsCount = _previousTimelineCalls,
+                TimelineCallsCountMin = timelineCalls.Any() ? timelineCalls.Min() : 0,
+                TimelineCallsCountAvg = timelineCalls.Any() ? (int)timelineCalls.Average() : 0,
+                TimelineCallsCountMax = timelineCalls.Any() ? timelineCalls.Max() : 0,
                TimelineCallsMax = 1500
            };
        }
    }
+
+    internal class ApiStatisticsSnapshot 
+    {
+        #region Ctor
+        public ApiStatisticsSnapshot(int userCalls, int tweetCalls, int timelineCalls)
+        {
+            UserCalls = userCalls;
+            TweetCalls = tweetCalls;
+            TimelineCalls = timelineCalls;
+            SnapshotDate = DateTime.UtcNow;
+        }
+        #endregion
+
+        public DateTime SnapshotDate { get;  }
+        public int UserCalls { get; set; }
+        public int TweetCalls { get; set; }
+        public int TimelineCalls { get; set; }
+    }
 }
--- a/src/BirdsiteLive.Twitter/TwitterService.cs
+++ b/src/BirdsiteLive.Twitter/TwitterService.cs
@ -43,11 +43,16 @@ namespace BirdsiteLive.Twitter
            _statisticsHandler.CalledUserApi();
            if (user == null) return null;

+            // Expand URLs
+            var description = user.Description;
+            foreach (var descriptionUrl in user.Entities?.Description?.Urls?.OrderByDescending(x => x.URL.Length))
+                description = description.Replace(descriptionUrl.URL, descriptionUrl.ExpandedURL);
+
            return new TwitterUser
            {
                Acct = username,
                Name = user.Name,
-                Description = user.Description,
+                Description = description,
                Url = $"https://twitter.com/{username}",
                ProfileImageUrl = user.ProfileImageUrlFullSize,
                ProfileBackgroundImageUrl = user.ProfileBackgroundImageUrlHttps,
--- a/src/BirdsiteLive/BirdsiteLive.csproj
+++ b/src/BirdsiteLive/BirdsiteLive.csproj
@ -4,7 +4,7 @@
    <TargetFramework>netcoreapp3.1</TargetFramework>
    <UserSecretsId>d21486de-a812-47eb-a419-05682bb68856</UserSecretsId>
    <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
-    <Version>0.6.1</Version>
+    <Version>0.7.0</Version>
  </PropertyGroup>

  <ItemGroup>
--- a/src/BirdsiteLive/Controllers/StatisticsController.cs
+++ b/src/BirdsiteLive/Controllers/StatisticsController.cs
@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.Domain.Statistics;
 using BirdsiteLive.Statistics.Domain;
 using Microsoft.AspNetCore.Mvc;

@ -13,13 +14,16 @@ namespace BirdsiteLive.Controllers
        private readonly ITwitterUserDal _twitterUserDal;
        private readonly IFollowersDal _followersDal;
        private readonly ITwitterStatisticsHandler _twitterStatistics;
+        private readonly IExtractionStatisticsHandler _extractionStatistics;
+

        #region Ctor
-        public StatisticsController(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, ITwitterStatisticsHandler twitterStatistics)
+        public StatisticsController(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, ITwitterStatisticsHandler twitterStatistics, IExtractionStatisticsHandler extractionStatistics)
        {
            _twitterUserDal = twitterUserDal;
            _followersDal = followersDal;
            _twitterStatistics = twitterStatistics;
+            _extractionStatistics = extractionStatistics;
        }
        #endregion

@ -29,7 +33,8 @@ namespace BirdsiteLive.Controllers
            {
                FollowersCount = await _followersDal.GetFollowersCountAsync(),
                TwitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync(),
-                TwitterStatistics = _twitterStatistics.GetStatistics()
+                TwitterStatistics = _twitterStatistics.GetStatistics(),
+                ExtractionStatistics = _extractionStatistics.GetStatistics(),
            };
            return View(stats);
        }
--- a/src/BirdsiteLive/Models/StatisticsModels/Statistics.cs
+++ b/src/BirdsiteLive/Models/StatisticsModels/Statistics.cs
@ -1,4 +1,5 @@
-using BirdsiteLive.Twitter.Models;
+using BirdsiteLive.Domain.Statistics;
+using BirdsiteLive.Twitter.Models;

 namespace BirdsiteLive.Models.StatisticsModels
 {
@ -7,5 +8,6 @@ namespace BirdsiteLive.Models.StatisticsModels
        public int FollowersCount { get; set; }
        public int TwitterUserCount { get; set; }
        public ApiStatistics TwitterStatistics { get; set; }
+        public ExtractionStatistics ExtractionStatistics { get; set; }
    }
 }
--- a/src/BirdsiteLive/Views/Statistics/Index.cshtml
+++ b/src/BirdsiteLive/Views/Statistics/Index.cshtml
@ -12,9 +12,17 @@
    <li>Followers: @Model.FollowersCount</li>
 </ul>

-<h4>Twitter API</h4>
+<h4>Twitter API (Min, Avg, Max for the last 24h)</h4>
 <ul>
-    <li>Users Calls: @Model.TwitterStatistics.UserCallsCount / @Model.TwitterStatistics.UserCallsMax</li>
-    <li>Tweets Calls: @Model.TwitterStatistics.TweetCallsCount / @Model.TwitterStatistics.TweetCallsMax</li>
-    <li>Timeline Calls: @Model.TwitterStatistics.TimelineCallsCount / @Model.TwitterStatistics.TimelineCallsMax</li>
+    <li>Users Calls: @Model.TwitterStatistics.UserCallsCountMin ;  @Model.TwitterStatistics.UserCallsCountAvg ;  @Model.TwitterStatistics.UserCallsCountMax / @Model.TwitterStatistics.UserCallsMax</li>
+    <li>Tweets Calls: @Model.TwitterStatistics.TweetCallsCountMin ; @Model.TwitterStatistics.TweetCallsCountAvg ; @Model.TwitterStatistics.TweetCallsCountMax / @Model.TwitterStatistics.TweetCallsMax</li>
+    <li>Timeline Calls: @Model.TwitterStatistics.TimelineCallsCountMin ; @Model.TwitterStatistics.TimelineCallsCountAvg ; @Model.TwitterStatistics.TimelineCallsCountMax / @Model.TwitterStatistics.TimelineCallsMax</li>
+</ul>
+
+<h4>Mentions Extraction</h4>
+<ul>
+    <li>Current day in descriptions:  @Model.ExtractionStatistics.MentionsInDescriptionsExtraction</li>
+    <li>Current day in statuses: @Model.ExtractionStatistics.MentionsInStatusesExtraction</li>
+    <li>Last day in descriptions: @Model.ExtractionStatistics.LastMentionsInDescriptionsExtraction</li>
+    <li>Last day in statuses: @Model.ExtractionStatistics.LastMentionsInStatusesExtraction</li>
 </ul>