diff --git a/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj b/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj
index 5c48a42..a8cdea2 100644
--- a/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj
+++ b/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj
@@ -10,6 +10,7 @@
+
diff --git a/src/BirdsiteLive.Twitter/CachedTwitterService.cs b/src/BirdsiteLive.Twitter/CachedTwitterService.cs
index 877d3ae..c549883 100644
--- a/src/BirdsiteLive.Twitter/CachedTwitterService.cs
+++ b/src/BirdsiteLive.Twitter/CachedTwitterService.cs
@@ -34,7 +34,7 @@ namespace BirdsiteLive.Twitter
_userCache = new MemoryCache(new MemoryCacheOptions()
{
- SizeLimit = 5000 //TODO make this use number of entries in db
+ SizeLimit = 3000 //TODO make this use number of entries in db
});
}
#endregion
diff --git a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs
index d252a72..f019ab1 100644
--- a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs
+++ b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs
@@ -12,6 +12,8 @@ using BirdsiteLive.Twitter.Models;
using BirdsiteLive.Twitter.Tools;
using Microsoft.Extensions.Logging;
using System.Text.RegularExpressions;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Models;
namespace BirdsiteLive.Twitter
{
@@ -26,15 +28,17 @@ namespace BirdsiteLive.Twitter
private readonly ITwitterAuthenticationInitializer _twitterAuthenticationInitializer;
private readonly ITwitterStatisticsHandler _statisticsHandler;
private readonly ICachedTwitterUserService _twitterUserService;
+ private readonly ITwitterUserDal _twitterUserDal;
private readonly ILogger _logger;
private HttpClient _httpClient = new HttpClient();
#region Ctor
- public TwitterTweetsService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITwitterStatisticsHandler statisticsHandler, ICachedTwitterUserService twitterUserService, ILogger logger)
+ public TwitterTweetsService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITwitterStatisticsHandler statisticsHandler, ICachedTwitterUserService twitterUserService, ITwitterUserDal twitterUserDal, ILogger logger)
{
_twitterAuthenticationInitializer = twitterAuthenticationInitializer;
_statisticsHandler = statisticsHandler;
_twitterUserService = twitterUserService;
+ _twitterUserDal = twitterUserDal;
_logger = logger;
}
#endregion
@@ -79,12 +83,22 @@ namespace BirdsiteLive.Twitter
var client = await _twitterAuthenticationInitializer.MakeHttpClient();
- var user = await _twitterUserService.GetUserAsync(username);
- if (user == null || user.Protected) return new ExtractedTweet[0];
+ long userId;
+ SyncTwitterUser user = await _twitterUserDal.GetTwitterUserAsync(username);
+ if (user.TwitterUserId == default)
+ {
+ var user2 = await _twitterUserService.GetUserAsync(username);
+ userId = user2.Id;
+ await _twitterUserDal.UpdateTwitterUserIdAsync(username, user2.Id);
+ }
+ else
+ {
+ userId = user.TwitterUserId;
+ }
var reqURL = "https://twitter.com/i/api/graphql/s0hG9oAmWEYVBqOLJP-TBQ/UserTweetsAndReplies?variables=%7B%22userId%22%3A%22"
- + user.Id +
+ + userId +
"%22%2C%22count%22%3A40%2C%22includePromotedContent%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22unified_cards_ad_metadata_container_dynamic_card_content_query_enabled%22%3Atrue%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_uc_gql_enabled%22%3Atrue%2C%22vibe_api_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Afalse%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse%2C%22interactive_text_enabled%22%3Atrue%2C%22responsive_web_text_conversations_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Atrue%7D";
JsonDocument results;
List extractedTweets = new List();
@@ -122,6 +136,17 @@ namespace BirdsiteLive.Twitter
if (tweet.GetProperty("content").GetProperty("entryType").GetString() != "TimelineTimelineItem")
continue;
+ try
+ {
+ JsonElement userDoc = tweet.GetProperty("content").GetProperty("itemContent")
+ .GetProperty("tweet_results").GetProperty("core").GetProperty("user_results");
+
+ TwitterUser tweetUser = _twitterUserService.Extract(userDoc);
+ _twitterUserService.AddUser(tweetUser);
+ }
+ catch (Exception _)
+ {}
+
try
{
var extractedTweet = await Extract(tweet);
@@ -138,16 +163,6 @@ namespace BirdsiteLive.Twitter
}
- try
- {
- JsonElement userDoc = tweet.GetProperty("content").GetProperty("itemContent")
- .GetProperty("tweet_results").GetProperty("core").GetProperty("user_results");
-
- TwitterUser tweetUser = _twitterUserService.Extract(userDoc);
- _twitterUserService.AddUser(tweetUser);
- }
- catch (Exception e)
- {}
}
}
diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs
index 2f9cb54..d088545 100644
--- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs
+++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs
@@ -23,7 +23,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
public class DbInitializerPostgresDal : PostgresBase, IDbInitializerDal
{
private readonly PostgresTools _tools;
- private readonly Version _currentVersion = new Version(2, 4);
+ private readonly Version _currentVersion = new Version(2, 5);
private const string DbVersionType = "db-version";
#region Ctor
@@ -135,7 +135,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
new Tuple(new Version(2,0), new Version(2,1)),
new Tuple(new Version(2,1), new Version(2,2)),
new Tuple(new Version(2,2), new Version(2,3)),
- new Tuple(new Version(2,3), new Version(2,4))
+ new Tuple(new Version(2,3), new Version(2,4)),
+ new Tuple(new Version(2,4), new Version(2,5))
};
}
@@ -172,6 +173,12 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
var alterPostingError = $@"ALTER TABLE {_settings.FollowersTableName} ALTER COLUMN postingErrorCount TYPE INTEGER";
await _tools.ExecuteRequestAsync(alterPostingError);
}
+ else if (from == new Version(2, 4) && to == new Version(2, 5))
+ {
+ var alterTwitterUserId = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD twitterUserId BIGINT";
+ await _tools.ExecuteRequestAsync(alterTwitterUserId);
+
+ }
else
{
throw new NotImplementedException();
diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs
index 3ed1610..db0b9a4 100644
--- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs
+++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs
@@ -125,6 +125,20 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
}
}
+ public async Task UpdateTwitterUserIdAsync(string username, long twitterUserId)
+ {
+ if(username == default) throw new ArgumentException("id");
+ if(twitterUserId == default) throw new ArgumentException("twtterUserId");
+
+ var query = $"UPDATE {_settings.TwitterUserTableName} SET twitterUserId = @twitterUserId WHERE acct = @username";
+
+ using (var dbConnection = Connection)
+ {
+ dbConnection.Open();
+
+ await dbConnection.QueryAsync(query, new { username, twitterUserId });
+ }
+ }
public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync)
{
if(id == default) throw new ArgumentException("id");
diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs
index f6b592f..a5a5df4 100644
--- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs
+++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs
@@ -13,6 +13,7 @@ namespace BirdsiteLive.DAL.Contracts
Task GetAllTwitterUsersAsync(int maxNumber);
Task GetAllTwitterUsersAsync();
Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync);
+ Task UpdateTwitterUserIdAsync(string username, long twitterUserId);
Task UpdateTwitterUserAsync(SyncTwitterUser user);
Task DeleteTwitterUserAsync(string acct);
Task DeleteTwitterUserAsync(int id);
diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs
index 8b18ba1..06b7a0b 100644
--- a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs
+++ b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs
@@ -5,6 +5,7 @@ namespace BirdsiteLive.DAL.Models
public class SyncTwitterUser
{
public int Id { get; set; }
+ public long TwitterUserId { get; set; }
public string Acct { get; set; }
public long LastTweetPostedId { get; set; }
diff --git a/src/Tests/BirdsiteLive.Twitter.Tests/BirdsiteLive.Twitter.Tests.csproj b/src/Tests/BirdsiteLive.Twitter.Tests/BirdsiteLive.Twitter.Tests.csproj
index 45319de..1463362 100644
--- a/src/Tests/BirdsiteLive.Twitter.Tests/BirdsiteLive.Twitter.Tests.csproj
+++ b/src/Tests/BirdsiteLive.Twitter.Tests/BirdsiteLive.Twitter.Tests.csproj
@@ -17,6 +17,7 @@
+
diff --git a/src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs b/src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs
index 4bb6bd2..34a9021 100644
--- a/src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs
+++ b/src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs
@@ -5,6 +5,8 @@ using BirdsiteLive.Twitter;
using BirdsiteLive.Twitter.Tools;
using BirdsiteLive.Statistics.Domain;
using Moq;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Models;
namespace BirdsiteLive.ActivityPub.Tests
{
@@ -20,10 +22,18 @@ namespace BirdsiteLive.ActivityPub.Tests
var logger3 = new Mock>();
var stats = new Mock();
var settings = new Mock();
+ var twitterDal = new Mock();
+
+ twitterDal
+ .Setup(x => x.GetTwitterUserAsync(
+ It.Is(y => true)
+ ))
+ .ReturnsAsync(new SyncTwitterUser { TwitterUserId = default });
+
ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(logger1.Object);
ITwitterUserService user = new TwitterUserService(auth, stats.Object, logger2.Object);
ICachedTwitterUserService user2 = new CachedTwitterUserService(user, settings.Object);
- _tweetService = new TwitterTweetsService(auth, stats.Object, user2, logger3.Object);
+ _tweetService = new TwitterTweetsService(auth, stats.Object, user2, twitterDal.Object, logger3.Object);
}
[TestMethod]
diff --git a/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs b/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs
index 089dc09..8fad641 100644
--- a/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs
+++ b/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs
@@ -5,6 +5,8 @@ using BirdsiteLive.Twitter;
using BirdsiteLive.Twitter.Tools;
using BirdsiteLive.Statistics.Domain;
using Moq;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Models;
namespace BirdsiteLive.ActivityPub.Tests
{
@@ -20,10 +22,11 @@ namespace BirdsiteLive.ActivityPub.Tests
var logger3 = new Mock>();
var settings = new Mock();
var stats = new Mock();
+ var twitterDal = new Mock();
ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(logger1.Object);
ITwitterUserService user = new TwitterUserService(auth, stats.Object, logger2.Object);
ICachedTwitterUserService user2 = new CachedTwitterUserService(user, settings.Object);
- _tweetService = new TwitterTweetsService(auth, stats.Object, user2, logger3.Object);
+ _tweetService = new TwitterTweetsService(auth, stats.Object, user2, twitterDal.Object, logger3.Object);
}
[TestMethod]