sam/BirdsiteLIVE
sam/BirdsiteLIVE
Archived
1
0
Fork 0
Code Issues 1 Pull requests 1 Packages Projects Releases Wiki Activity

cache twitter user id in db

Browse source
This commit is contained in:
Vincent Cloutier 2022-12-29 09:58:08 -05:00
parent bdb4b86ae8
commit 59ea905e43
10 changed files with 72 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj
View file

@ -10,6 +10,7 @@
<ItemGroup>
<ProjectReference Include="..\BirdsiteLive.Common\BirdsiteLive.Common.csproj" />
<ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
</ItemGroup>
<ItemGroup>

2
src/BirdsiteLive.Twitter/CachedTwitterService.cs
View file

@ -34,7 +34,7 @@ namespace BirdsiteLive.Twitter
_userCache = new MemoryCache(new MemoryCacheOptions()
{
SizeLimit = 5000 //TODO make this use number of entries in db
SizeLimit = 3000 //TODO make this use number of entries in db
});
}
#endregion

43
src/BirdsiteLive.Twitter/TwitterTweetsService.cs
View file

@ -12,6 +12,8 @@ using BirdsiteLive.Twitter.Models;
using BirdsiteLive.Twitter.Tools;
using Microsoft.Extensions.Logging;
using System.Text.RegularExpressions;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
namespace BirdsiteLive.Twitter
{
@ -26,15 +28,17 @@ namespace BirdsiteLive.Twitter
private readonly ITwitterAuthenticationInitializer _twitterAuthenticationInitializer;
private readonly ITwitterStatisticsHandler _statisticsHandler;
private readonly ICachedTwitterUserService _twitterUserService;
private readonly ITwitterUserDal _twitterUserDal;
private readonly ILogger<TwitterTweetsService> _logger;
private HttpClient _httpClient = new HttpClient();
#region Ctor
public TwitterTweetsService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITwitterStatisticsHandler statisticsHandler, ICachedTwitterUserService twitterUserService, ILogger<TwitterTweetsService> logger)
public TwitterTweetsService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITwitterStatisticsHandler statisticsHandler, ICachedTwitterUserService twitterUserService, ITwitterUserDal twitterUserDal, ILogger<TwitterTweetsService> logger)
{
_twitterAuthenticationInitializer = twitterAuthenticationInitializer;
_statisticsHandler = statisticsHandler;
_twitterUserService = twitterUserService;
_twitterUserDal = twitterUserDal;
_logger = logger;
}
#endregion
@ -79,12 +83,22 @@ namespace BirdsiteLive.Twitter
var client = await _twitterAuthenticationInitializer.MakeHttpClient();
var user = await _twitterUserService.GetUserAsync(username);
if (user == null || user.Protected) return new ExtractedTweet[0];
long userId;
SyncTwitterUser user = await _twitterUserDal.GetTwitterUserAsync(username);
if (user.TwitterUserId == default)
{
var user2 = await _twitterUserService.GetUserAsync(username);
userId = user2.Id;
await _twitterUserDal.UpdateTwitterUserIdAsync(username, user2.Id);
}
else
{
userId = user.TwitterUserId;
}
var reqURL = "https://twitter.com/i/api/graphql/s0hG9oAmWEYVBqOLJP-TBQ/UserTweetsAndReplies?variables=%7B%22userId%22%3A%22"
+ user.Id +
+ userId +
"%22%2C%22count%22%3A40%2C%22includePromotedContent%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22unified_cards_ad_metadata_container_dynamic_card_content_query_enabled%22%3Atrue%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_uc_gql_enabled%22%3Atrue%2C%22vibe_api_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Afalse%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse%2C%22interactive_text_enabled%22%3Atrue%2C%22responsive_web_text_conversations_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Atrue%7D";
JsonDocument results;
List<ExtractedTweet> extractedTweets = new List<ExtractedTweet>();
@ -122,6 +136,17 @@ namespace BirdsiteLive.Twitter
if (tweet.GetProperty("content").GetProperty("entryType").GetString() != "TimelineTimelineItem")
continue;
try
{
JsonElement userDoc = tweet.GetProperty("content").GetProperty("itemContent")
.GetProperty("tweet_results").GetProperty("core").GetProperty("user_results");
TwitterUser tweetUser = _twitterUserService.Extract(userDoc);
_twitterUserService.AddUser(tweetUser);
}
catch (Exception _)
{}
try
{
var extractedTweet = await Extract(tweet);
@ -138,16 +163,6 @@ namespace BirdsiteLive.Twitter
}
try
{
JsonElement userDoc = tweet.GetProperty("content").GetProperty("itemContent")
.GetProperty("tweet_results").GetProperty("core").GetProperty("user_results");
TwitterUser tweetUser = _twitterUserService.Extract(userDoc);
_twitterUserService.AddUser(tweetUser);
}
catch (Exception e)
{}
}
}

11
src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs
View file

@ -23,7 +23,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
public class DbInitializerPostgresDal : PostgresBase, IDbInitializerDal
{
private readonly PostgresTools _tools;
private readonly Version _currentVersion = new Version(2, 4);
private readonly Version _currentVersion = new Version(2, 5);
private const string DbVersionType = "db-version";
#region Ctor
@ -135,7 +135,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
new Tuple<Version, Version>(new Version(2,0), new Version(2,1)),
new Tuple<Version, Version>(new Version(2,1), new Version(2,2)),
new Tuple<Version, Version>(new Version(2,2), new Version(2,3)),
new Tuple<Version, Version>(new Version(2,3), new Version(2,4))
new Tuple<Version, Version>(new Version(2,3), new Version(2,4)),
new Tuple<Version, Version>(new Version(2,4), new Version(2,5))
};
}
@ -172,6 +173,12 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
var alterPostingError = $@"ALTER TABLE {_settings.FollowersTableName} ALTER COLUMN postingErrorCount TYPE INTEGER";
await _tools.ExecuteRequestAsync(alterPostingError);
}
else if (from == new Version(2, 4) && to == new Version(2, 5))
{
var alterTwitterUserId = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD twitterUserId BIGINT";
await _tools.ExecuteRequestAsync(alterTwitterUserId);
}
else
{
throw new NotImplementedException();

14
src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs
View file

@ -125,6 +125,20 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
}
}
public async Task UpdateTwitterUserIdAsync(string username, long twitterUserId)
{
if(username == default) throw new ArgumentException("id");
if(twitterUserId == default) throw new ArgumentException("twtterUserId");
var query = $"UPDATE {_settings.TwitterUserTableName} SET twitterUserId = @twitterUserId WHERE acct = @username";
using (var dbConnection = Connection)
{
dbConnection.Open();
await dbConnection.QueryAsync(query, new { username, twitterUserId });
}
}
public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync)
{
if(id == default) throw new ArgumentException("id");

1
src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs
View file

@ -13,6 +13,7 @@ namespace BirdsiteLive.DAL.Contracts
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber);
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync();
Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync);
Task UpdateTwitterUserIdAsync(string username, long twitterUserId);
Task UpdateTwitterUserAsync(SyncTwitterUser user);
Task DeleteTwitterUserAsync(string acct);
Task DeleteTwitterUserAsync(int id);

1
src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs
View file

@ -5,6 +5,7 @@ namespace BirdsiteLive.DAL.Models
public class SyncTwitterUser
{
public int Id { get; set; }
public long TwitterUserId { get; set; }
public string Acct { get; set; }
public long LastTweetPostedId { get; set; }

1
src/Tests/BirdsiteLive.Twitter.Tests/BirdsiteLive.Twitter.Tests.csproj
View file

@ -17,6 +17,7 @@
<ItemGroup>
<ProjectReference Include="..\..\BirdsiteLive.Twitter\BirdsiteLive.Twitter.csproj" />
<ProjectReference Include="..\..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
</ItemGroup>
</Project>

12
src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs
View file

@ -5,6 +5,8 @@ using BirdsiteLive.Twitter;
using BirdsiteLive.Twitter.Tools;
using BirdsiteLive.Statistics.Domain;
using Moq;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
namespace BirdsiteLive.ActivityPub.Tests
{
@ -20,10 +22,18 @@ namespace BirdsiteLive.ActivityPub.Tests
var logger3 = new Mock<ILogger<TwitterTweetsService>>();
var stats = new Mock<ITwitterStatisticsHandler>();
var settings = new Mock<Common.Settings.InstanceSettings>();
var twitterDal = new Mock<ITwitterUserDal>();
twitterDal
.Setup(x => x.GetTwitterUserAsync(
It.Is<string>(y => true)
))
.ReturnsAsync(new SyncTwitterUser { TwitterUserId = default });
ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(logger1.Object);
ITwitterUserService user = new TwitterUserService(auth, stats.Object, logger2.Object);
ICachedTwitterUserService user2 = new CachedTwitterUserService(user, settings.Object);
_tweetService = new TwitterTweetsService(auth, stats.Object, user2, logger3.Object);
_tweetService = new TwitterTweetsService(auth, stats.Object, user2, twitterDal.Object, logger3.Object);
}
[TestMethod]

5
src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs
View file

@ -5,6 +5,8 @@ using BirdsiteLive.Twitter;
using BirdsiteLive.Twitter.Tools;
using BirdsiteLive.Statistics.Domain;
using Moq;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
namespace BirdsiteLive.ActivityPub.Tests
{
@ -20,10 +22,11 @@ namespace BirdsiteLive.ActivityPub.Tests
var logger3 = new Mock<ILogger<TwitterTweetsService>>();
var settings = new Mock<Common.Settings.InstanceSettings>();
var stats = new Mock<ITwitterStatisticsHandler>();
var twitterDal = new Mock<ITwitterUserDal>();
ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(logger1.Object);
ITwitterUserService user = new TwitterUserService(auth, stats.Object, logger2.Object);
ICachedTwitterUserService user2 = new CachedTwitterUserService(user, settings.Object);
_tweetService = new TwitterTweetsService(auth, stats.Object, user2, logger3.Object);
_tweetService = new TwitterTweetsService(auth, stats.Object, user2, twitterDal.Object, logger3.Object);
}
[TestMethod]