From 59ea905e43273d5f58b7fb1ad39dc950d8189381 Mon Sep 17 00:00:00 2001 From: Vincent Cloutier Date: Thu, 29 Dec 2022 09:58:08 -0500 Subject: [PATCH] cache twitter user id in db --- .../BirdsiteLive.Twitter.csproj | 1 + .../CachedTwitterService.cs | 2 +- .../TwitterTweetsService.cs | 43 +++++++++++++------ .../DbInitializerPostgresDal.cs | 11 ++++- .../TwitterUserPostgresDal.cs | 14 ++++++ .../Contracts/ITwitterUserDal.cs | 1 + .../Models/SyncTwitterUser.cs | 1 + .../BirdsiteLive.Twitter.Tests.csproj | 1 + .../TimelineTests.cs | 12 +++++- .../BirdsiteLive.Twitter.Tests/TweetTests.cs | 5 ++- 10 files changed, 72 insertions(+), 19 deletions(-) diff --git a/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj b/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj index 5c48a42..a8cdea2 100644 --- a/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj +++ b/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj @@ -10,6 +10,7 @@ + diff --git a/src/BirdsiteLive.Twitter/CachedTwitterService.cs b/src/BirdsiteLive.Twitter/CachedTwitterService.cs index 877d3ae..c549883 100644 --- a/src/BirdsiteLive.Twitter/CachedTwitterService.cs +++ b/src/BirdsiteLive.Twitter/CachedTwitterService.cs @@ -34,7 +34,7 @@ namespace BirdsiteLive.Twitter _userCache = new MemoryCache(new MemoryCacheOptions() { - SizeLimit = 5000 //TODO make this use number of entries in db + SizeLimit = 3000 //TODO make this use number of entries in db }); } #endregion diff --git a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs index d252a72..f019ab1 100644 --- a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs +++ b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs @@ -12,6 +12,8 @@ using BirdsiteLive.Twitter.Models; using BirdsiteLive.Twitter.Tools; using Microsoft.Extensions.Logging; using System.Text.RegularExpressions; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; namespace BirdsiteLive.Twitter { @@ -26,15 +28,17 @@ namespace BirdsiteLive.Twitter private readonly ITwitterAuthenticationInitializer _twitterAuthenticationInitializer; private readonly ITwitterStatisticsHandler _statisticsHandler; private readonly ICachedTwitterUserService _twitterUserService; + private readonly ITwitterUserDal _twitterUserDal; private readonly ILogger _logger; private HttpClient _httpClient = new HttpClient(); #region Ctor - public TwitterTweetsService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITwitterStatisticsHandler statisticsHandler, ICachedTwitterUserService twitterUserService, ILogger logger) + public TwitterTweetsService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITwitterStatisticsHandler statisticsHandler, ICachedTwitterUserService twitterUserService, ITwitterUserDal twitterUserDal, ILogger logger) { _twitterAuthenticationInitializer = twitterAuthenticationInitializer; _statisticsHandler = statisticsHandler; _twitterUserService = twitterUserService; + _twitterUserDal = twitterUserDal; _logger = logger; } #endregion @@ -79,12 +83,22 @@ namespace BirdsiteLive.Twitter var client = await _twitterAuthenticationInitializer.MakeHttpClient(); - var user = await _twitterUserService.GetUserAsync(username); - if (user == null || user.Protected) return new ExtractedTweet[0]; + long userId; + SyncTwitterUser user = await _twitterUserDal.GetTwitterUserAsync(username); + if (user.TwitterUserId == default) + { + var user2 = await _twitterUserService.GetUserAsync(username); + userId = user2.Id; + await _twitterUserDal.UpdateTwitterUserIdAsync(username, user2.Id); + } + else + { + userId = user.TwitterUserId; + } var reqURL = "https://twitter.com/i/api/graphql/s0hG9oAmWEYVBqOLJP-TBQ/UserTweetsAndReplies?variables=%7B%22userId%22%3A%22" - + user.Id + + + userId + "%22%2C%22count%22%3A40%2C%22includePromotedContent%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22unified_cards_ad_metadata_container_dynamic_card_content_query_enabled%22%3Atrue%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_uc_gql_enabled%22%3Atrue%2C%22vibe_api_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Afalse%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse%2C%22interactive_text_enabled%22%3Atrue%2C%22responsive_web_text_conversations_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Atrue%7D"; JsonDocument results; List extractedTweets = new List(); @@ -122,6 +136,17 @@ namespace BirdsiteLive.Twitter if (tweet.GetProperty("content").GetProperty("entryType").GetString() != "TimelineTimelineItem") continue; + try + { + JsonElement userDoc = tweet.GetProperty("content").GetProperty("itemContent") + .GetProperty("tweet_results").GetProperty("core").GetProperty("user_results"); + + TwitterUser tweetUser = _twitterUserService.Extract(userDoc); + _twitterUserService.AddUser(tweetUser); + } + catch (Exception _) + {} + try { var extractedTweet = await Extract(tweet); @@ -138,16 +163,6 @@ namespace BirdsiteLive.Twitter } - try - { - JsonElement userDoc = tweet.GetProperty("content").GetProperty("itemContent") - .GetProperty("tweet_results").GetProperty("core").GetProperty("user_results"); - - TwitterUser tweetUser = _twitterUserService.Extract(userDoc); - _twitterUserService.AddUser(tweetUser); - } - catch (Exception e) - {} } } diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs index 2f9cb54..d088545 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs @@ -23,7 +23,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public class DbInitializerPostgresDal : PostgresBase, IDbInitializerDal { private readonly PostgresTools _tools; - private readonly Version _currentVersion = new Version(2, 4); + private readonly Version _currentVersion = new Version(2, 5); private const string DbVersionType = "db-version"; #region Ctor @@ -135,7 +135,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers new Tuple(new Version(2,0), new Version(2,1)), new Tuple(new Version(2,1), new Version(2,2)), new Tuple(new Version(2,2), new Version(2,3)), - new Tuple(new Version(2,3), new Version(2,4)) + new Tuple(new Version(2,3), new Version(2,4)), + new Tuple(new Version(2,4), new Version(2,5)) }; } @@ -172,6 +173,12 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers var alterPostingError = $@"ALTER TABLE {_settings.FollowersTableName} ALTER COLUMN postingErrorCount TYPE INTEGER"; await _tools.ExecuteRequestAsync(alterPostingError); } + else if (from == new Version(2, 4) && to == new Version(2, 5)) + { + var alterTwitterUserId = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD twitterUserId BIGINT"; + await _tools.ExecuteRequestAsync(alterTwitterUserId); + + } else { throw new NotImplementedException(); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs index 3ed1610..db0b9a4 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs @@ -125,6 +125,20 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } } + public async Task UpdateTwitterUserIdAsync(string username, long twitterUserId) + { + if(username == default) throw new ArgumentException("id"); + if(twitterUserId == default) throw new ArgumentException("twtterUserId"); + + var query = $"UPDATE {_settings.TwitterUserTableName} SET twitterUserId = @twitterUserId WHERE acct = @username"; + + using (var dbConnection = Connection) + { + dbConnection.Open(); + + await dbConnection.QueryAsync(query, new { username, twitterUserId }); + } + } public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync) { if(id == default) throw new ArgumentException("id"); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs index f6b592f..a5a5df4 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs @@ -13,6 +13,7 @@ namespace BirdsiteLive.DAL.Contracts Task GetAllTwitterUsersAsync(int maxNumber); Task GetAllTwitterUsersAsync(); Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync); + Task UpdateTwitterUserIdAsync(string username, long twitterUserId); Task UpdateTwitterUserAsync(SyncTwitterUser user); Task DeleteTwitterUserAsync(string acct); Task DeleteTwitterUserAsync(int id); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs index 8b18ba1..06b7a0b 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs @@ -5,6 +5,7 @@ namespace BirdsiteLive.DAL.Models public class SyncTwitterUser { public int Id { get; set; } + public long TwitterUserId { get; set; } public string Acct { get; set; } public long LastTweetPostedId { get; set; } diff --git a/src/Tests/BirdsiteLive.Twitter.Tests/BirdsiteLive.Twitter.Tests.csproj b/src/Tests/BirdsiteLive.Twitter.Tests/BirdsiteLive.Twitter.Tests.csproj index 45319de..1463362 100644 --- a/src/Tests/BirdsiteLive.Twitter.Tests/BirdsiteLive.Twitter.Tests.csproj +++ b/src/Tests/BirdsiteLive.Twitter.Tests/BirdsiteLive.Twitter.Tests.csproj @@ -17,6 +17,7 @@ + diff --git a/src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs b/src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs index 4bb6bd2..34a9021 100644 --- a/src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs +++ b/src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs @@ -5,6 +5,8 @@ using BirdsiteLive.Twitter; using BirdsiteLive.Twitter.Tools; using BirdsiteLive.Statistics.Domain; using Moq; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; namespace BirdsiteLive.ActivityPub.Tests { @@ -20,10 +22,18 @@ namespace BirdsiteLive.ActivityPub.Tests var logger3 = new Mock>(); var stats = new Mock(); var settings = new Mock(); + var twitterDal = new Mock(); + + twitterDal + .Setup(x => x.GetTwitterUserAsync( + It.Is(y => true) + )) + .ReturnsAsync(new SyncTwitterUser { TwitterUserId = default }); + ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(logger1.Object); ITwitterUserService user = new TwitterUserService(auth, stats.Object, logger2.Object); ICachedTwitterUserService user2 = new CachedTwitterUserService(user, settings.Object); - _tweetService = new TwitterTweetsService(auth, stats.Object, user2, logger3.Object); + _tweetService = new TwitterTweetsService(auth, stats.Object, user2, twitterDal.Object, logger3.Object); } [TestMethod] diff --git a/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs b/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs index 089dc09..8fad641 100644 --- a/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs +++ b/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs @@ -5,6 +5,8 @@ using BirdsiteLive.Twitter; using BirdsiteLive.Twitter.Tools; using BirdsiteLive.Statistics.Domain; using Moq; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; namespace BirdsiteLive.ActivityPub.Tests { @@ -20,10 +22,11 @@ namespace BirdsiteLive.ActivityPub.Tests var logger3 = new Mock>(); var settings = new Mock(); var stats = new Mock(); + var twitterDal = new Mock(); ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(logger1.Object); ITwitterUserService user = new TwitterUserService(auth, stats.Object, logger2.Object); ICachedTwitterUserService user2 = new CachedTwitterUserService(user, settings.Object); - _tweetService = new TwitterTweetsService(auth, stats.Object, user2, logger3.Object); + _tweetService = new TwitterTweetsService(auth, stats.Object, user2, twitterDal.Object, logger3.Object); } [TestMethod]