From d0e4a09d3d17964a7673e26ca193819ff4d49084 Mon Sep 17 00:00:00 2001 From: Vincent Cloutier Date: Mon, 9 May 2022 20:31:18 -0400 Subject: [PATCH] boosts v1 --- src/BirdsiteLive.Domain/ActivityPubService.cs | 8 +-- src/BirdsiteLive.Domain/StatusService.cs | 5 ++ .../SubTasks/SendTweetsToInboxTask.cs | 2 +- .../SubTasks/SendTweetsToSharedInboxTask.cs | 9 ++- .../CachedTwitterService.cs | 10 +++ .../Models/ExtractedTweet.cs | 2 +- .../TwitterTweetsService.cs | 7 +- .../TwitterUserService.cs | 70 +++++++++++++++++++ 8 files changed, 103 insertions(+), 10 deletions(-) diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs index e4cb011..1b29c39 100644 --- a/src/BirdsiteLive.Domain/ActivityPubService.cs +++ b/src/BirdsiteLive.Domain/ActivityPubService.cs @@ -18,7 +18,7 @@ namespace BirdsiteLive.Domain { Task GetUser(string objectId); Task PostDataAsync(T data, string targetHost, string actorUrl, string inbox = null); - Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, + Task PostNewActivity(Note note, string username, string activityType, string noteId, string targetHost, string targetInbox); } @@ -57,7 +57,7 @@ namespace BirdsiteLive.Domain return actor; } - public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox) + public async Task PostNewActivity(Note note, string username, string activityType, string noteId, string targetHost, string targetInbox) { try { @@ -71,7 +71,7 @@ namespace BirdsiteLive.Domain { context = "https://www.w3.org/ns/activitystreams", id = $"{noteUri}/activity", - type = "Create", + type = activityType, actor = actor, published = nowString, @@ -95,7 +95,7 @@ namespace BirdsiteLive.Domain if (!string.IsNullOrWhiteSpace(inbox)) usedInbox = inbox; - var json = JsonConvert.SerializeObject(data); + var json = JsonConvert.SerializeObject(data, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); var date = DateTime.UtcNow.ToUniversalTime(); var httpDate = date.ToString("r"); diff --git a/src/BirdsiteLive.Domain/StatusService.cs b/src/BirdsiteLive.Domain/StatusService.cs index b6a00dc..4b90cd6 100644 --- a/src/BirdsiteLive.Domain/StatusService.cs +++ b/src/BirdsiteLive.Domain/StatusService.cs @@ -42,6 +42,11 @@ namespace BirdsiteLive.Domain { var actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, username); var noteUrl = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, tweet.Id.ToString()); + if (tweet.IsRetweet) + { + actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, tweet.OriginalAuthor.Name); + noteUrl = UrlFactory.GetNoteUrl(_instanceSettings.Domain, tweet.OriginalAuthor.Name, tweet.Id.ToString()); + } var to = $"{actorUrl}/followers"; diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs index a6f6982..c76428c 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs @@ -60,7 +60,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks _settings.PublishReplies) { var note = _statusService.GetStatus(user.Acct, tweet); - await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox); + await _activityPubService.PostNewActivity(note, user.Acct, "Create", tweet.Id.ToString(), follower.Host, inbox); } } catch (ArgumentException e) diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs index 055b47b..8dfebf9 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs @@ -56,12 +56,17 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks { try { - if (!tweet.IsReply || + if (tweet.IsRetweet) + { + var note = _statusService.GetStatus(user.Acct, tweet); + await _activityPubService.PostNewActivity(note, user.Acct, "Announce", tweet.Id.ToString(), host, inbox); + } + else if (!tweet.IsReply || tweet.IsReply && tweet.IsThread || _settings.PublishReplies) { var note = _statusService.GetStatus(user.Acct, tweet); - await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox); + await _activityPubService.PostNewActivity(note, user.Acct, "Create", tweet.Id.ToString(), host, inbox); } } catch (ArgumentException e) diff --git a/src/BirdsiteLive.Twitter/CachedTwitterService.cs b/src/BirdsiteLive.Twitter/CachedTwitterService.cs index c49104a..79645dd 100644 --- a/src/BirdsiteLive.Twitter/CachedTwitterService.cs +++ b/src/BirdsiteLive.Twitter/CachedTwitterService.cs @@ -36,6 +36,16 @@ namespace BirdsiteLive.Twitter } #endregion + public TwitterUser GetUser(long id) + { + if (!_userCache.TryGetValue(id, out TwitterUser user)) + { + user = _twitterService.GetUser(id); + if(user != null) _userCache.Set(id, user, _cacheEntryOptions); + } + + return user; + } public TwitterUser GetUser(string username) { if (!_userCache.TryGetValue(username, out TwitterUser user)) diff --git a/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs b/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs index 51e12f8..268745c 100644 --- a/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs +++ b/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs @@ -15,6 +15,6 @@ namespace BirdsiteLive.Twitter.Models public bool IsThread { get; set; } public bool IsRetweet { get; set; } public string RetweetUrl { get; set; } - public string OriginalAuthor { get; set; } + public TwitterUser OriginalAuthor { get; set; } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs index d18bbdb..583f5d9 100644 --- a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs +++ b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs @@ -110,7 +110,7 @@ namespace BirdsiteLive.Twitter return tweets.RootElement.GetProperty("data").EnumerateArray().Select(Extract).ToArray(); } - public ExtractedTweet Extract(JsonElement tweet) + private ExtractedTweet Extract(JsonElement tweet) { bool IsRetweet = false; bool IsReply = false; @@ -129,9 +129,11 @@ namespace BirdsiteLive.Twitter if (first.GetProperty("type").GetString() == "retweeted") { IsRetweet = true; + var originalAuthor = _twitterUserService.GetUser(Int64.Parse(tweet.GetProperty("author_id").GetString())); var statusId = Int64.Parse(first.GetProperty("id").GetString()); var extracted = GetTweet(statusId); extracted.IsRetweet = true; + extracted.OriginalAuthor = originalAuthor; return extracted; } @@ -158,7 +160,8 @@ namespace BirdsiteLive.Twitter IsReply = IsReply, IsThread = false, IsRetweet = IsRetweet, - RetweetUrl = "https://t.co/123" + RetweetUrl = "https://t.co/123", + OriginalAuthor = null, }; return extractedTweet; diff --git a/src/BirdsiteLive.Twitter/TwitterUserService.cs b/src/BirdsiteLive.Twitter/TwitterUserService.cs index 6a8ecc7..f711e01 100644 --- a/src/BirdsiteLive.Twitter/TwitterUserService.cs +++ b/src/BirdsiteLive.Twitter/TwitterUserService.cs @@ -14,6 +14,7 @@ namespace BirdsiteLive.Twitter public interface ITwitterUserService { TwitterUser GetUser(string username); + TwitterUser GetUser(long id); bool IsUserApiRateLimited(); } @@ -33,6 +34,10 @@ namespace BirdsiteLive.Twitter } #endregion + public TwitterUser GetUser(long id) + { + return GetUserAsync(id).Result; + } public TwitterUser GetUser(string username) { return GetUserAsync(username).Result; @@ -108,6 +113,71 @@ namespace BirdsiteLive.Twitter }; } + public async Task GetUserAsync(long id) + { + //Check if API is saturated + if (IsUserApiRateLimited()) throw new RateLimitExceededException(); + + //Proceed to account retrieval + await _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized(); + + JsonDocument res; + try + { + using (var request = new HttpRequestMessage(new HttpMethod("GET"), "https://api.twitter.com/2/users/"+ id + "?user.fields=name,username,protected,profile_image_url,url,description")) + { + request.Headers.TryAddWithoutValidation("Authorization", "Bearer " + _twitterAuthenticationInitializer.Token); + + var httpResponse = await _httpClient.SendAsync(request); + httpResponse.EnsureSuccessStatusCode(); + + var c = await httpResponse.Content.ReadAsStringAsync(); + res = JsonDocument.Parse(c); + } + } + catch (HttpRequestException e) + { + throw; + //if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("User has been suspended".ToLowerInvariant()))) + //{ + // throw new UserHasBeenSuspendedException(); + //} + //else if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("User not found".ToLowerInvariant()))) + //{ + // throw new UserNotFoundException(); + //} + //else if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("Rate limit exceeded".ToLowerInvariant()))) + //{ + // throw new RateLimitExceededException(); + //} + //else + //{ + // throw; + //} + } + catch (Exception e) + { + _logger.LogError(e, "Error retrieving user id {id}", id); + throw; + } + finally + { + _statisticsHandler.CalledUserApi(); + } + return new TwitterUser + { + Id = id, + Acct = res.RootElement.GetProperty("data").GetProperty("username").GetString(), + Name = res.RootElement.GetProperty("data").GetProperty("name").GetString(), + Description = res.RootElement.GetProperty("data").GetProperty("description").GetString(), + Url = res.RootElement.GetProperty("data").GetProperty("url").GetString(), + ProfileImageUrl = res.RootElement.GetProperty("data").GetProperty("profile_image_url").GetString(), + ProfileBackgroundImageUrl = res.RootElement.GetProperty("data").GetProperty("profile_image_url").GetString(), //for now + ProfileBannerURL = res.RootElement.GetProperty("data").GetProperty("profile_image_url").GetString(), //for now + Protected = res.RootElement.GetProperty("data").GetProperty("protected").GetBoolean(), + }; + } + public bool IsUserApiRateLimited() { // Retrieve limit from tooling