sam/BirdsiteLIVE
sam
/
BirdsiteLIVE
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

extract only needed info from tweets

This commit is contained in:
Nicolas Constant 2020-07-22 20:19:40 -04:00
parent 1724d3a902
commit fffc9af534
No known key found for this signature in database
GPG Key ID: 1E9F677FB01A5688
7 changed files with 140 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
src/BirdsiteLive.Domain/StatusService.cs
View File

@ -1,8 +1,10 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using BirdsiteLive.ActivityPub;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.Twitter.Models;
using Tweetinvi.Models;
using Tweetinvi.Models.Entities;
@ -10,7 +12,7 @@ namespace BirdsiteLive.Domain
{
public interface IStatusService
{
Note GetStatus(string username, ITweet tweet);
Note GetStatus(string username, ExtractedTweet tweet);
}
public class StatusService : IStatusService
@ -24,7 +26,7 @@ namespace BirdsiteLive.Domain
}
#endregion
public Note GetStatus(string username, ITweet tweet)
public Note GetStatus(string username, ExtractedTweet tweet)
{
var actorUrl = $"https://{_instanceSettings.Domain}/users/{username}";
var noteId = $"https://{_instanceSettings.Domain}/users/{username}/statuses/{tweet.Id}";
@ -49,8 +51,8 @@ namespace BirdsiteLive.Domain
//cc = new string[0],
sensitive = false,
content = $"<p>{tweet.Text}</p>",
attachment = GetAttachments(tweet.Media),
content = $"<p>{tweet.MessageContent}</p>",
attachment = Convert(tweet.Media),
tag = new string[0]
};
@ -58,62 +60,17 @@ namespace BirdsiteLive.Domain
return note;
}
private Attachment[] GetAttachments(List<IMediaEntity> media)
private Attachment[] Convert(ExtractedMedia[] media)
{
var result = new List<Attachment>();
foreach (var m in media)
return media.Select(x =>
{
var mediaUrl = GetMediaUrl(m);
var mediaType = GetMediaType(m.MediaType, mediaUrl);
if (mediaType == null) continue;
var att = new Attachment
return new Attachment
{
type = "Document",
mediaType = mediaType,
url = mediaUrl
url = x.Url,
mediaType = x.MediaType
};
result.Add(att);
}
return result.ToArray();
}
private string GetMediaUrl(IMediaEntity media)
{
switch (media.MediaType)
{
case "photo": return media.MediaURLHttps;
case "animated_gif": return media.VideoDetails.Variants[0].URL;
case "video": return media.VideoDetails.Variants.OrderByDescending(x => x.Bitrate).First().URL;
default: return null;
}
}
private string GetMediaType(string mediaType, string mediaUrl)
{
switch (mediaType)
{
case "photo":
var ext = Path.GetExtension(mediaUrl);
switch (ext)
{
case ".jpg":
case ".jpeg":
return "image/jpeg";
case ".png":
return "image/png";
}
return null;
case "animated_gif":
return "image/gif";
case "video":
return "video/mp4";
}
return null;
}).ToArray();
}
}
}

3
src/BirdsiteLive.Pipeline/Models/UserWithTweetsToSync.cs
View File

@ -1,4 +1,5 @@
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Twitter.Models;
using Tweetinvi.Models;
namespace BirdsiteLive.Pipeline.Models
@ -6,7 +7,7 @@ namespace BirdsiteLive.Pipeline.Models
public class UserWithTweetsToSync
{
public SyncTwitterUser User { get; set; }
public ITweet[] Tweets { get; set; }
public ExtractedTweet[] Tweets { get; set; }
public Follower[] Followers { get; set; }
}
}

5
src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs
View File

@ -7,6 +7,7 @@ using BirdsiteLive.DAL.Models;
using BirdsiteLive.Pipeline.Contracts;
using BirdsiteLive.Pipeline.Models;
using BirdsiteLive.Twitter;
using BirdsiteLive.Twitter.Models;
using Tweetinvi.Models;
namespace BirdsiteLive.Pipeline.Processors
@ -51,9 +52,9 @@ namespace BirdsiteLive.Pipeline.Processors
return usersWtTweets.ToArray();
}
private ITweet[] RetrieveNewTweets(SyncTwitterUser user)
private ExtractedTweet[] RetrieveNewTweets(SyncTwitterUser user)
{
ITweet[] tweets;
ExtractedTweet[] tweets;
if (user.LastTweetPostedId == -1)
tweets = _twitterService.GetTimeline(user.Acct, 1);
else

3
src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs
View File

@ -10,6 +10,7 @@ using BirdsiteLive.Domain;
using BirdsiteLive.Pipeline.Contracts;
using BirdsiteLive.Pipeline.Models;
using BirdsiteLive.Twitter;
using BirdsiteLive.Twitter.Models;
using Tweetinvi.Models;
namespace BirdsiteLive.Pipeline.Processors
@ -50,7 +51,7 @@ namespace BirdsiteLive.Pipeline.Processors
return userWithTweetsToSync;
}
private async Task ProcessFollowerAsync(IEnumerable<ITweet> tweets, Follower follower, int userId,
private async Task ProcessFollowerAsync(IEnumerable<ExtractedTweet> tweets, Follower follower, int userId,
SyncTwitterUser user)
{
var fromStatusId = follower.FollowingsSyncStatus[userId];

8
src/BirdsiteLive.Twitter/Models/ExtractedMedia.cs Normal file
View File

@ -0,0 +1,8 @@
namespace BirdsiteLive.Twitter.Models
{
public class ExtractedMedia
{
public string MediaType { get; set; }
public string Url { get; set; }
}
}

14
src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs Normal file
View File

@ -0,0 +1,14 @@
using System;
using System.Net.Sockets;
namespace BirdsiteLive.Twitter.Models
{
public class ExtractedTweet
{
public long Id { get; set; }
public long? InReplyToStatusId { get; set; }
public string MessageContent { get; set; }
public ExtractedMedia[] Media { get; set; }
public DateTime CreatedAt { get; set; }
}
}

107
src/BirdsiteLive.Twitter/TwitterService.cs
View File

@ -1,11 +1,13 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.Twitter.Models;
using Tweetinvi;
using Tweetinvi.Models;
using Tweetinvi.Models.Entities;
using Tweetinvi.Parameters;
namespace BirdsiteLive.Twitter
@ -13,8 +15,8 @@ namespace BirdsiteLive.Twitter
public interface ITwitterService
{
TwitterUser GetUser(string username);
ITweet GetTweet(long statusId);
ITweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1);
ExtractedTweet GetTweet(long statusId);
ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1);
}
public class TwitterService : ITwitterService
@ -31,7 +33,6 @@ namespace BirdsiteLive.Twitter
public TwitterUser GetUser(string username)
{
//Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true);
var user = User.GetUserFromScreenName(username);
if (user == null) return null;
@ -47,16 +48,104 @@ namespace BirdsiteLive.Twitter
};
}
public ITweet GetTweet(long statusId)
public ExtractedTweet GetTweet(long statusId)
{
//Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true);
TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended;
var tweet = Tweet.GetTweet(statusId);
return tweet;
return Extract(tweet);
}
public ITweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1)
private ExtractedTweet Extract(ITweet tweet)
{
var extractedTweet = new ExtractedTweet
{
Id = tweet.Id,
InReplyToStatusId = tweet.InReplyToStatusId,
MessageContent = ExtractMessage(tweet),
Media = ExtractMedia(tweet.Media),
CreatedAt = tweet.CreatedAt
};
return extractedTweet;
}
private string ExtractMessage(ITweet tweet)
{
var tweetUrls = tweet.Media.Select(x => x.URL).Distinct();
var message = tweet.FullText;
foreach (var tweetUrl in tweetUrls)
message = message.Replace(tweetUrl, string.Empty).Trim();
if (tweet.QuotedTweet != null) message = $"[Quote RT] {message}";
if (tweet.IsRetweet)
{
if (tweet.RetweetedTweet != null)
message = $"[RT {tweet.RetweetedTweet.CreatedBy.ScreenName}] {tweet.RetweetedTweet.FullText}";
else
message = message.Replace("RT", "[RT]");
}
return message;
}
private ExtractedMedia[] ExtractMedia(List<IMediaEntity> media)
{
var result = new List<ExtractedMedia>();
foreach (var m in media)
{
var mediaUrl = GetMediaUrl(m);
var mediaType = GetMediaType(m.MediaType, mediaUrl);
if (mediaType == null) continue;
var att = new ExtractedMedia
{
MediaType = mediaType,
Url = mediaUrl
};
result.Add(att);
}
return result.ToArray();
}
private string GetMediaUrl(IMediaEntity media)
{
switch (media.MediaType)
{
case "photo": return media.MediaURLHttps;
case "animated_gif": return media.VideoDetails.Variants[0].URL;
case "video": return media.VideoDetails.Variants.OrderByDescending(x => x.Bitrate).First().URL;
default: return null;
}
}
private string GetMediaType(string mediaType, string mediaUrl)
{
switch (mediaType)
{
case "photo":
var ext = Path.GetExtension(mediaUrl);
switch (ext)
{
case ".jpg":
case ".jpeg":
return "image/jpeg";
case ".png":
return "image/png";
}
return null;
case "animated_gif":
return "image/gif";
case "video":
return "video/mp4";
}
return null;
}
public ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1)
{
//Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true);
TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended;
var user = User.GetUserFromScreenName(username);
@ -77,7 +166,7 @@ namespace BirdsiteLive.Twitter
if (timeline != null) tweets.AddRange(timeline);
}
return tweets.ToArray();
return tweets.Select(Extract).ToArray();
//return tweets.Where(x => returnReplies || string.IsNullOrWhiteSpace(x.InReplyToScreenName)).ToArray();
}
}