sam/BirdsiteLIVE
sam/BirdsiteLIVE
Archived
1
0
Fork 0
Code Issues 1 Pull requests 1 Packages Projects Releases Wiki Activity

Merge pull request #49 from NicolasConstant/topic_add-api-statistics

Browse source 
Topic add api statistics
This commit is contained in:
Nicolas Constant 2021-01-14 06:40:51 +01:00 committed by GitHub
parent 45f75935ed dea3089c06
commit f0a0bb9f43
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 185 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

80
src/BirdsiteLive.Domain/Statistics/ExtractionStatisticsHandler.cs Normal file
View file

@ -0,0 +1,80 @@
using System.Threading;
using System.Timers;
namespace BirdsiteLive.Domain.Statistics
{
public interface IExtractionStatisticsHandler
{
void ExtractedDescription(int mentionsCount);
void ExtractedStatus(int mentionsCount);
ExtractionStatistics GetStatistics();
}
public class ExtractionStatisticsHandler : IExtractionStatisticsHandler
{
private static int _lastDescriptionMentionsExtracted;
private static int _lastStatusMentionsExtracted;
private static int _descriptionMentionsExtracted;
private static int _statusMentionsExtracted;
private static System.Timers.Timer _resetTimer;
#region Ctor
public ExtractionStatisticsHandler()
{
if (_resetTimer == null)
{
_resetTimer = new System.Timers.Timer();
_resetTimer.Elapsed += OnTimeResetEvent;
_resetTimer.Interval = 24 * 60 * 60 * 1000; // 24h
_resetTimer.Enabled = true;
}
}
#endregion
private void OnTimeResetEvent(object sender, ElapsedEventArgs e)
{
_lastDescriptionMentionsExtracted = _descriptionMentionsExtracted;
_lastStatusMentionsExtracted = _statusMentionsExtracted;
// Reset
Interlocked.Exchange(ref _descriptionMentionsExtracted, 0);
Interlocked.Exchange(ref _statusMentionsExtracted, 0);
}
public void ExtractedDescription(int mentionsCount)
{
Interlocked.Increment(ref _descriptionMentionsExtracted);
}
public void ExtractedStatus(int mentionsCount)
{
Interlocked.Increment(ref _statusMentionsExtracted);
}
public ExtractionStatistics GetStatistics()
{
return new ExtractionStatistics(_descriptionMentionsExtracted, _statusMentionsExtracted, _lastDescriptionMentionsExtracted, _lastStatusMentionsExtracted);
}
}
public class ExtractionStatistics
{
#region Ctor
public ExtractionStatistics(int mentionsInDescriptionsExtraction, int mentionsInStatusesExtraction, int lastMentionsInDescriptionsExtraction, int lastMentionsInStatusesExtraction)
{
MentionsInDescriptionsExtraction = mentionsInDescriptionsExtraction;
MentionsInStatusesExtraction = mentionsInStatusesExtraction;
LastMentionsInDescriptionsExtraction = lastMentionsInDescriptionsExtraction;
LastMentionsInStatusesExtraction = lastMentionsInStatusesExtraction;
}
#endregion
public int MentionsInDescriptionsExtraction { get; }
public int MentionsInStatusesExtraction { get; }
public int LastMentionsInDescriptionsExtraction { get; }
public int LastMentionsInStatusesExtraction { get; }
}
}

8
src/BirdsiteLive.Domain/StatusService.cs
View file

@ -7,6 +7,7 @@ using BirdsiteLive.ActivityPub;
using BirdsiteLive.ActivityPub.Converters;
using BirdsiteLive.ActivityPub.Models;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.Domain.Statistics;
using BirdsiteLive.Domain.Tools;
using BirdsiteLive.Twitter.Models;
using Tweetinvi.Models;
@ -23,12 +24,14 @@ namespace BirdsiteLive.Domain
{
private readonly InstanceSettings _instanceSettings;
private readonly IStatusExtractor _statusExtractor;
private readonly IExtractionStatisticsHandler _statisticsHandler;
#region Ctor
public StatusService(InstanceSettings instanceSettings, IStatusExtractor statusExtractor)
public StatusService(InstanceSettings instanceSettings, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler)
{
_instanceSettings = instanceSettings;
_statusExtractor = statusExtractor;
_statisticsHandler = statisticsHandler;
}
#endregion
@ -41,6 +44,7 @@ namespace BirdsiteLive.Domain
var apPublic = "https://www.w3.org/ns/activitystreams#Public";
var extractedTags = _statusExtractor.ExtractTags(tweet.MessageContent);
_statisticsHandler.ExtractedStatus(extractedTags.tags.Count(x => x.type == "Mention"));
string inReplyTo = null;
if (tweet.InReplyToStatusId != default)

7
src/BirdsiteLive.Domain/UserService.cs
View file

@ -10,6 +10,7 @@ using BirdsiteLive.ActivityPub.Converters;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.Cryptography;
using BirdsiteLive.Domain.BusinessUseCases;
using BirdsiteLive.Domain.Statistics;
using BirdsiteLive.Domain.Tools;
using BirdsiteLive.Twitter.Models;
using Tweetinvi.Core.Exceptions;
@ -33,9 +34,10 @@ namespace BirdsiteLive.Domain
private readonly ICryptoService _cryptoService;
private readonly IActivityPubService _activityPubService;
private readonly IStatusExtractor _statusExtractor;
private readonly IExtractionStatisticsHandler _statisticsHandler;
#region Ctor
public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor)
public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler)
{
_instanceSettings = instanceSettings;
_cryptoService = cryptoService;
@ -43,6 +45,7 @@ namespace BirdsiteLive.Domain
_processFollowUser = processFollowUser;
_processUndoFollowUser = processUndoFollowUser;
_statusExtractor = statusExtractor;
_statisticsHandler = statisticsHandler;
//_host = $"https://{instanceSettings.Domain.Replace("https://",string.Empty).Replace("http://", string.Empty).TrimEnd('/')}";
}
#endregion
@ -58,6 +61,8 @@ namespace BirdsiteLive.Domain
{
var extracted = _statusExtractor.ExtractTags(description);
description = extracted.content;
_statisticsHandler.ExtractedDescription(extracted.tags.Count(x => x.type == "Mention"));
}
var user = new Actor

4
src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs
View file

@ -45,6 +45,10 @@ namespace BirdsiteLive.Twitter.Extractors
message = message.Replace("RT", "[RT]");
}
// Expand URLs
foreach (var url in tweet.Urls.OrderByDescending(x => x.URL.Length))
message = message.Replace(url.URL, url.ExpandedURL);
return message;
}

12
src/BirdsiteLive.Twitter/Models/ApiStatistics.cs
View file

@ -2,11 +2,17 @@
{
public class ApiStatistics
{
public int UserCallsCount { get; set; }
public int UserCallsCountMin { get; set; }
public int UserCallsCountAvg { get; set; }
public int UserCallsCountMax { get; set; }
public int UserCallsMax { get; set; }
public int TweetCallsCount { get; set; }
public int TweetCallsCountMin { get; set; }
public int TweetCallsCountAvg { get; set; }
public int TweetCallsCountMax { get; set; }
public int TweetCallsMax { get; set; }
public int TimelineCallsCount { get; set; }
public int TimelineCallsCountMin { get; set; }
public int TimelineCallsCountAvg { get; set; }
public int TimelineCallsCountMax { get; set; }
public int TimelineCallsMax { get; set; }
}
}

64
src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs
View file

@ -1,4 +1,7 @@
using System.Threading;
using System;
using System.Collections.Concurrent;
using System.Linq;
using System.Threading;
using System.Timers;
using BirdsiteLive.Twitter.Models;
@ -15,14 +18,12 @@ namespace BirdsiteLive.Statistics.Domain
//Rate limits: https://developer.twitter.com/en/docs/twitter-api/v1/rate-limits
public class TwitterStatisticsHandler : ITwitterStatisticsHandler
{
private static int _previousUserCalls;
private static int _previousTweetCalls;
private static int _previousTimelineCalls;
private static int _userCalls;
private static int _tweetCalls;
private static int _timelineCalls;
private static ConcurrentDictionary<DateTime, ApiStatisticsSnapshot> _snapshots = new ConcurrentDictionary<DateTime, ApiStatisticsSnapshot>();
private static System.Timers.Timer _resetTimer;
#region Ctor
@ -40,13 +41,23 @@ namespace BirdsiteLive.Statistics.Domain
private void OnTimeResetEvent(object sender, ElapsedEventArgs e)
{
_previousUserCalls = _userCalls;
_previousTweetCalls = _tweetCalls;
_previousTimelineCalls = _timelineCalls;
// Add snapshot
var snapshot = new ApiStatisticsSnapshot(_userCalls, _tweetCalls, _timelineCalls);
bool success;
do
{
success = _snapshots.TryAdd(snapshot.SnapshotDate, snapshot);
} while (!success);
// Reset
Interlocked.Exchange(ref _userCalls, 0);
Interlocked.Exchange(ref _tweetCalls, 0);
Interlocked.Exchange(ref _timelineCalls, 0);
// Clean up
var now = DateTime.UtcNow;
var oldSnapshots = _snapshots.Keys.Where(x => (now - x).TotalHours > 24).ToList();
foreach (var old in oldSnapshots) _snapshots.TryRemove(old, out var data);
}
public void CalledUserApi() //GET users/show - 900/15mins
@ -66,15 +77,44 @@ namespace BirdsiteLive.Statistics.Domain
public ApiStatistics GetStatistics()
{
var snapshots = _snapshots.Values.ToList();
var userCalls = snapshots.Select(x => x.UserCalls).ToList();
var tweetCalls = snapshots.Select(x => x.TweetCalls).ToList();
var timelineCalls = snapshots.Select(x => x.TimelineCalls).ToList();
return new ApiStatistics
{
UserCallsCount = _previousUserCalls,
UserCallsCountMin = userCalls.Any() ? userCalls.Min() : 0,
UserCallsCountAvg = userCalls.Any() ? (int)userCalls.Average() : 0,
UserCallsCountMax = userCalls.Any() ? userCalls.Max() : 0,
UserCallsMax = 900,
TweetCallsCount = _previousTweetCalls,
TweetCallsCountMin = tweetCalls.Any() ? tweetCalls.Min() : 0,
TweetCallsCountAvg = tweetCalls.Any() ? (int)tweetCalls.Average() : 0,
TweetCallsCountMax = tweetCalls.Any() ? tweetCalls.Max() : 0,
TweetCallsMax = 300,
TimelineCallsCount = _previousTimelineCalls,
TimelineCallsCountMin = timelineCalls.Any() ? timelineCalls.Min() : 0,
TimelineCallsCountAvg = timelineCalls.Any() ? (int)timelineCalls.Average() : 0,
TimelineCallsCountMax = timelineCalls.Any() ? timelineCalls.Max() : 0,
TimelineCallsMax = 1500
};
}
}
internal class ApiStatisticsSnapshot
{
#region Ctor
public ApiStatisticsSnapshot(int userCalls, int tweetCalls, int timelineCalls)
{
UserCalls = userCalls;
TweetCalls = tweetCalls;
TimelineCalls = timelineCalls;
SnapshotDate = DateTime.UtcNow;
}
#endregion
public DateTime SnapshotDate { get; }
public int UserCalls { get; set; }
public int TweetCalls { get; set; }
public int TimelineCalls { get; set; }
}
}

7
src/BirdsiteLive.Twitter/TwitterService.cs
View file

@ -43,11 +43,16 @@ namespace BirdsiteLive.Twitter
_statisticsHandler.CalledUserApi();
if (user == null) return null;
// Expand URLs
var description = user.Description;
foreach (var descriptionUrl in user.Entities?.Description?.Urls?.OrderByDescending(x => x.URL.Length))
description = description.Replace(descriptionUrl.URL, descriptionUrl.ExpandedURL);
return new TwitterUser
{
Acct = username,
Name = user.Name,
Description = user.Description,
Description = description,
Url = $"https://twitter.com/{username}",
ProfileImageUrl = user.ProfileImageUrlFullSize,
ProfileBackgroundImageUrl = user.ProfileBackgroundImageUrlHttps,

9
src/BirdsiteLive/Controllers/StatisticsController.cs
View file

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.Domain.Statistics;
using BirdsiteLive.Statistics.Domain;
using Microsoft.AspNetCore.Mvc;
@ -13,13 +14,16 @@ namespace BirdsiteLive.Controllers
private readonly ITwitterUserDal _twitterUserDal;
private readonly IFollowersDal _followersDal;
private readonly ITwitterStatisticsHandler _twitterStatistics;
private readonly IExtractionStatisticsHandler _extractionStatistics;
#region Ctor
public StatisticsController(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, ITwitterStatisticsHandler twitterStatistics)
public StatisticsController(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, ITwitterStatisticsHandler twitterStatistics, IExtractionStatisticsHandler extractionStatistics)
{
_twitterUserDal = twitterUserDal;
_followersDal = followersDal;
_twitterStatistics = twitterStatistics;
_extractionStatistics = extractionStatistics;
}
#endregion
@ -29,7 +33,8 @@ namespace BirdsiteLive.Controllers
{
FollowersCount = await _followersDal.GetFollowersCountAsync(),
TwitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync(),
TwitterStatistics = _twitterStatistics.GetStatistics()
TwitterStatistics = _twitterStatistics.GetStatistics(),
ExtractionStatistics = _extractionStatistics.GetStatistics(),
};
return View(stats);
}

4
src/BirdsiteLive/Models/StatisticsModels/Statistics.cs
View file

@ -1,4 +1,5 @@
using BirdsiteLive.Twitter.Models;
using BirdsiteLive.Domain.Statistics;
using BirdsiteLive.Twitter.Models;
namespace BirdsiteLive.Models.StatisticsModels
{
@ -7,5 +8,6 @@ namespace BirdsiteLive.Models.StatisticsModels
public int FollowersCount { get; set; }
public int TwitterUserCount { get; set; }
public ApiStatistics TwitterStatistics { get; set; }
public ExtractionStatistics ExtractionStatistics { get; set; }
}
}

16
src/BirdsiteLive/Views/Statistics/Index.cshtml
View file

@ -12,9 +12,17 @@
<li>Followers: @Model.FollowersCount</li>
</ul>
<h4>Twitter API</h4>
<h4>Twitter API (Min, Avg, Max for the last 24h)</h4>
<ul>
<li>Users Calls: @Model.TwitterStatistics.UserCallsCount / @Model.TwitterStatistics.UserCallsMax</li>
<li>Tweets Calls: @Model.TwitterStatistics.TweetCallsCount / @Model.TwitterStatistics.TweetCallsMax</li>
<li>Timeline Calls: @Model.TwitterStatistics.TimelineCallsCount / @Model.TwitterStatistics.TimelineCallsMax</li>
<li>Users Calls: @Model.TwitterStatistics.UserCallsCountMin ; @Model.TwitterStatistics.UserCallsCountAvg ; @Model.TwitterStatistics.UserCallsCountMax / @Model.TwitterStatistics.UserCallsMax</li>
<li>Tweets Calls: @Model.TwitterStatistics.TweetCallsCountMin ; @Model.TwitterStatistics.TweetCallsCountAvg ; @Model.TwitterStatistics.TweetCallsCountMax / @Model.TwitterStatistics.TweetCallsMax</li>
<li>Timeline Calls: @Model.TwitterStatistics.TimelineCallsCountMin ; @Model.TwitterStatistics.TimelineCallsCountAvg ; @Model.TwitterStatistics.TimelineCallsCountMax / @Model.TwitterStatistics.TimelineCallsMax</li>
</ul>
<h4>Mentions Extraction</h4>
<ul>
<li>Current day in descriptions: @Model.ExtractionStatistics.MentionsInDescriptionsExtraction</li>
<li>Current day in statuses: @Model.ExtractionStatistics.MentionsInStatusesExtraction</li>
<li>Last day in descriptions: @Model.ExtractionStatistics.LastMentionsInDescriptionsExtraction</li>
<li>Last day in statuses: @Model.ExtractionStatistics.LastMentionsInStatusesExtraction</li>
</ul>