From fffc9af534873733b75c00c6bf44c552519bde5e Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 22 Jul 2020 20:19:40 -0400 Subject: [PATCH] extract only needed info from tweets --- src/BirdsiteLive.Domain/StatusService.cs | 69 +++-------- .../Models/UserWithTweetsToSync.cs | 3 +- .../Processors/RetrieveTweetsProcessor.cs | 5 +- .../SendTweetsToFollowersProcessor.cs | 3 +- .../Models/ExtractedMedia.cs | 8 ++ .../Models/ExtractedTweet.cs | 14 +++ src/BirdsiteLive.Twitter/TwitterService.cs | 107 ++++++++++++++++-- 7 files changed, 140 insertions(+), 69 deletions(-) create mode 100644 src/BirdsiteLive.Twitter/Models/ExtractedMedia.cs create mode 100644 src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs diff --git a/src/BirdsiteLive.Domain/StatusService.cs b/src/BirdsiteLive.Domain/StatusService.cs index cc2b688..e50dddb 100644 --- a/src/BirdsiteLive.Domain/StatusService.cs +++ b/src/BirdsiteLive.Domain/StatusService.cs @@ -1,8 +1,10 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.IO; using System.Linq; using BirdsiteLive.ActivityPub; using BirdsiteLive.Common.Settings; +using BirdsiteLive.Twitter.Models; using Tweetinvi.Models; using Tweetinvi.Models.Entities; @@ -10,7 +12,7 @@ namespace BirdsiteLive.Domain { public interface IStatusService { - Note GetStatus(string username, ITweet tweet); + Note GetStatus(string username, ExtractedTweet tweet); } public class StatusService : IStatusService @@ -24,7 +26,7 @@ namespace BirdsiteLive.Domain } #endregion - public Note GetStatus(string username, ITweet tweet) + public Note GetStatus(string username, ExtractedTweet tweet) { var actorUrl = $"https://{_instanceSettings.Domain}/users/{username}"; var noteId = $"https://{_instanceSettings.Domain}/users/{username}/statuses/{tweet.Id}"; @@ -49,8 +51,8 @@ namespace BirdsiteLive.Domain //cc = new string[0], sensitive = false, - content = $"

{tweet.Text}

", - attachment = GetAttachments(tweet.Media), + content = $"

{tweet.MessageContent}

", + attachment = Convert(tweet.Media), tag = new string[0] }; @@ -58,62 +60,17 @@ namespace BirdsiteLive.Domain return note; } - private Attachment[] GetAttachments(List media) + private Attachment[] Convert(ExtractedMedia[] media) { - var result = new List(); - - foreach (var m in media) + return media.Select(x => { - var mediaUrl = GetMediaUrl(m); - var mediaType = GetMediaType(m.MediaType, mediaUrl); - if (mediaType == null) continue; - - var att = new Attachment + return new Attachment { type = "Document", - mediaType = mediaType, - url = mediaUrl + url = x.Url, + mediaType = x.MediaType }; - result.Add(att); - } - - return result.ToArray(); - } - - private string GetMediaUrl(IMediaEntity media) - { - switch (media.MediaType) - { - case "photo": return media.MediaURLHttps; - case "animated_gif": return media.VideoDetails.Variants[0].URL; - case "video": return media.VideoDetails.Variants.OrderByDescending(x => x.Bitrate).First().URL; - default: return null; - } - } - - private string GetMediaType(string mediaType, string mediaUrl) - { - switch (mediaType) - { - case "photo": - var ext = Path.GetExtension(mediaUrl); - switch (ext) - { - case ".jpg": - case ".jpeg": - return "image/jpeg"; - case ".png": - return "image/png"; - } - return null; - - case "animated_gif": - return "image/gif"; - - case "video": - return "video/mp4"; - } - return null; + }).ToArray(); } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Models/UserWithTweetsToSync.cs b/src/BirdsiteLive.Pipeline/Models/UserWithTweetsToSync.cs index 133e2a5..57810c7 100644 --- a/src/BirdsiteLive.Pipeline/Models/UserWithTweetsToSync.cs +++ b/src/BirdsiteLive.Pipeline/Models/UserWithTweetsToSync.cs @@ -1,4 +1,5 @@ using BirdsiteLive.DAL.Models; +using BirdsiteLive.Twitter.Models; using Tweetinvi.Models; namespace BirdsiteLive.Pipeline.Models @@ -6,7 +7,7 @@ namespace BirdsiteLive.Pipeline.Models public class UserWithTweetsToSync { public SyncTwitterUser User { get; set; } - public ITweet[] Tweets { get; set; } + public ExtractedTweet[] Tweets { get; set; } public Follower[] Followers { get; set; } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs index 22416be..dc556ba 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs @@ -7,6 +7,7 @@ using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Contracts; using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Twitter; +using BirdsiteLive.Twitter.Models; using Tweetinvi.Models; namespace BirdsiteLive.Pipeline.Processors @@ -51,9 +52,9 @@ namespace BirdsiteLive.Pipeline.Processors return usersWtTweets.ToArray(); } - private ITweet[] RetrieveNewTweets(SyncTwitterUser user) + private ExtractedTweet[] RetrieveNewTweets(SyncTwitterUser user) { - ITweet[] tweets; + ExtractedTweet[] tweets; if (user.LastTweetPostedId == -1) tweets = _twitterService.GetTimeline(user.Acct, 1); else diff --git a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs index a20f813..3dd37cd 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs @@ -10,6 +10,7 @@ using BirdsiteLive.Domain; using BirdsiteLive.Pipeline.Contracts; using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Twitter; +using BirdsiteLive.Twitter.Models; using Tweetinvi.Models; namespace BirdsiteLive.Pipeline.Processors @@ -50,7 +51,7 @@ namespace BirdsiteLive.Pipeline.Processors return userWithTweetsToSync; } - private async Task ProcessFollowerAsync(IEnumerable tweets, Follower follower, int userId, + private async Task ProcessFollowerAsync(IEnumerable tweets, Follower follower, int userId, SyncTwitterUser user) { var fromStatusId = follower.FollowingsSyncStatus[userId]; diff --git a/src/BirdsiteLive.Twitter/Models/ExtractedMedia.cs b/src/BirdsiteLive.Twitter/Models/ExtractedMedia.cs new file mode 100644 index 0000000..cdab034 --- /dev/null +++ b/src/BirdsiteLive.Twitter/Models/ExtractedMedia.cs @@ -0,0 +1,8 @@ +namespace BirdsiteLive.Twitter.Models +{ + public class ExtractedMedia + { + public string MediaType { get; set; } + public string Url { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs b/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs new file mode 100644 index 0000000..82c67bd --- /dev/null +++ b/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs @@ -0,0 +1,14 @@ +using System; +using System.Net.Sockets; + +namespace BirdsiteLive.Twitter.Models +{ + public class ExtractedTweet + { + public long Id { get; set; } + public long? InReplyToStatusId { get; set; } + public string MessageContent { get; set; } + public ExtractedMedia[] Media { get; set; } + public DateTime CreatedAt { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/TwitterService.cs b/src/BirdsiteLive.Twitter/TwitterService.cs index 4c8878d..ddbaaf0 100644 --- a/src/BirdsiteLive.Twitter/TwitterService.cs +++ b/src/BirdsiteLive.Twitter/TwitterService.cs @@ -1,11 +1,13 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading.Tasks; using BirdsiteLive.Common.Settings; using BirdsiteLive.Twitter.Models; using Tweetinvi; using Tweetinvi.Models; +using Tweetinvi.Models.Entities; using Tweetinvi.Parameters; namespace BirdsiteLive.Twitter @@ -13,8 +15,8 @@ namespace BirdsiteLive.Twitter public interface ITwitterService { TwitterUser GetUser(string username); - ITweet GetTweet(long statusId); - ITweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1); + ExtractedTweet GetTweet(long statusId); + ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1); } public class TwitterService : ITwitterService @@ -31,7 +33,6 @@ namespace BirdsiteLive.Twitter public TwitterUser GetUser(string username) { - //Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true); var user = User.GetUserFromScreenName(username); if (user == null) return null; @@ -47,16 +48,104 @@ namespace BirdsiteLive.Twitter }; } - public ITweet GetTweet(long statusId) + public ExtractedTweet GetTweet(long statusId) { - //Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true); + TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended; var tweet = Tweet.GetTweet(statusId); - return tweet; + return Extract(tweet); } - public ITweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1) + private ExtractedTweet Extract(ITweet tweet) + { + var extractedTweet = new ExtractedTweet + { + Id = tweet.Id, + InReplyToStatusId = tweet.InReplyToStatusId, + MessageContent = ExtractMessage(tweet), + Media = ExtractMedia(tweet.Media), + CreatedAt = tweet.CreatedAt + }; + return extractedTweet; + } + + private string ExtractMessage(ITweet tweet) + { + var tweetUrls = tweet.Media.Select(x => x.URL).Distinct(); + var message = tweet.FullText; + foreach (var tweetUrl in tweetUrls) + message = message.Replace(tweetUrl, string.Empty).Trim(); + + if (tweet.QuotedTweet != null) message = $"[Quote RT] {message}"; + if (tweet.IsRetweet) + { + if (tweet.RetweetedTweet != null) + message = $"[RT {tweet.RetweetedTweet.CreatedBy.ScreenName}] {tweet.RetweetedTweet.FullText}"; + else + message = message.Replace("RT", "[RT]"); + } + + return message; + } + + private ExtractedMedia[] ExtractMedia(List media) + { + var result = new List(); + + foreach (var m in media) + { + var mediaUrl = GetMediaUrl(m); + var mediaType = GetMediaType(m.MediaType, mediaUrl); + if (mediaType == null) continue; + + var att = new ExtractedMedia + { + MediaType = mediaType, + Url = mediaUrl + }; + result.Add(att); + } + + return result.ToArray(); + } + + private string GetMediaUrl(IMediaEntity media) + { + switch (media.MediaType) + { + case "photo": return media.MediaURLHttps; + case "animated_gif": return media.VideoDetails.Variants[0].URL; + case "video": return media.VideoDetails.Variants.OrderByDescending(x => x.Bitrate).First().URL; + default: return null; + } + } + + private string GetMediaType(string mediaType, string mediaUrl) + { + switch (mediaType) + { + case "photo": + var ext = Path.GetExtension(mediaUrl); + switch (ext) + { + case ".jpg": + case ".jpeg": + return "image/jpeg"; + case ".png": + return "image/png"; + } + return null; + + case "animated_gif": + return "image/gif"; + + case "video": + return "video/mp4"; + } + return null; + } + + public ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1) { - //Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true); TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended; var user = User.GetUserFromScreenName(username); @@ -77,7 +166,7 @@ namespace BirdsiteLive.Twitter if (timeline != null) tweets.AddRange(timeline); } - return tweets.ToArray(); + return tweets.Select(Extract).ToArray(); //return tweets.Where(x => returnReplies || string.IsNullOrWhiteSpace(x.InReplyToScreenName)).ToArray(); } }