sam/BirdsiteLIVE
sam
/
BirdsiteLIVE
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

don't retrieve deleted users

This commit is contained in:
Nicolas Constant 2022-12-24 18:44:41 -05:00
parent d219c59cfe
commit 1c3da007fd
No known key found for this signature in database
GPG Key ID: 1E9F677FB01A5688
8 changed files with 93 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/BirdsiteLive.Domain/MigrationService.cs
View File

@ -117,6 +117,7 @@ namespace BirdsiteLive.Domain
twitterAccount.MovedTo = validatedUser.ObjectId;
twitterAccount.MovedToAcct = validatedUser.FediverseAcct;
twitterAccount.LastSync = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount);
// Notify Followers
@ -180,6 +181,7 @@ namespace BirdsiteLive.Domain
}
twitterAccount.Deleted = true;
twitterAccount.LastSync = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount);
// Notify Followers

2
src/BirdsiteLive.Moderation/Processors/TwitterAccountModerationProcessor.cs
View File

@ -29,7 +29,7 @@ namespace BirdsiteLive.Moderation.Processors
{
if (type == ModerationTypeEnum.None) return;
var twitterUsers = await _twitterUserDal.GetAllTwitterUsersAsync();
var twitterUsers = await _twitterUserDal.GetAllTwitterUsersAsync(false);
foreach (var user in twitterUsers)
{

2
src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
View File

@ -39,7 +39,7 @@ namespace BirdsiteLive.Pipeline.Processors
try
{
var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync();
var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber);
var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber, false);
var userCount = users.Any() ? users.Length : 1;
var splitNumber = (int) Math.Ceiling(userCount / 15d);

14
src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs
View File

@ -69,7 +69,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
public async Task<int> GetTwitterUsersCountAsync()
{
var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE";
var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE";
using (var dbConnection = Connection)
{
@ -82,7 +82,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
public async Task<int> GetFailingTwitterUsersCountAsync()
{
var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE fetchingErrorCount > 0 AND (movedTo = '') IS NOT FALSE";
var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE fetchingErrorCount > 0 AND (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE";
using (var dbConnection = Connection)
{
@ -93,9 +93,10 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
}
}
public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber)
public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber, bool retrieveDisabledUser)
{
var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber";
var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber";
if (retrieveDisabledUser) query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber";
using (var dbConnection = Connection)
{
@ -106,9 +107,10 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
}
}
public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync()
public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(bool retrieveDisabledUser)
{
var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE";
var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE";
if(retrieveDisabledUser) query = $"SELECT * FROM {_settings.TwitterUserTableName}";
using (var dbConnection = Connection)
{

4
src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs
View File

@ -10,8 +10,8 @@ namespace BirdsiteLive.DAL.Contracts
string movedToAcct = null);
Task<SyncTwitterUser> GetTwitterUserAsync(string acct);
Task<SyncTwitterUser> GetTwitterUserAsync(int id);
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber);
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync();
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber, bool retrieveDisabledUser);
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(bool retrieveDisabledUser);
Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync, string movedTo, string movedToAcct, bool deleted);
Task UpdateTwitterUserAsync(SyncTwitterUser user);
Task DeleteTwitterUserAsync(string acct);

71
src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs
View File

@ -348,12 +348,71 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain");
}
var result = await dal.GetAllTwitterUsersAsync(1100);
for (int i = 0; i < 10; i++)
{
var acct = $"deleted-myid{i}";
var lastTweetId = 148L;
await dal.CreateTwitterUserAsync(acct, lastTweetId);
var user = await dal.GetTwitterUserAsync(acct);
user.Deleted = true;
user.LastSync = DateTime.UtcNow;
await dal.UpdateTwitterUserAsync(user);
}
var result = await dal.GetAllTwitterUsersAsync(1100, false);
Assert.AreEqual(1000, result.Length);
Assert.IsFalse(result[0].Id == default);
Assert.IsFalse(result[0].Acct == default);
Assert.IsFalse(result[0].LastTweetPostedId == default);
Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
foreach (var user in result)
{
Assert.IsTrue(string.IsNullOrWhiteSpace(user.MovedTo));
Assert.IsTrue(string.IsNullOrWhiteSpace(user.MovedToAcct));
Assert.IsFalse(user.Deleted);
}
}
[TestMethod]
public async Task GetAllTwitterUsers_Top_RetrieveDeleted()
{
var dal = new TwitterUserPostgresDal(_settings);
for (var i = 0; i < 1000; i++)
{
var acct = $"myid{i}";
var lastTweetId = 1548L;
await dal.CreateTwitterUserAsync(acct, lastTweetId);
}
for (int i = 0; i < 10; i++)
{
var acct = $"migrated-myid{i}";
var lastTweetId = 1548L;
await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain");
}
for (int i = 0; i < 10; i++)
{
var acct = $"deleted-myid{i}";
var lastTweetId = 148L;
await dal.CreateTwitterUserAsync(acct, lastTweetId);
var user = await dal.GetTwitterUserAsync(acct);
user.Deleted = true;
user.LastSync = DateTime.UtcNow;
await dal.UpdateTwitterUserAsync(user);
}
var result = await dal.GetAllTwitterUsersAsync(1100, true);
Assert.AreEqual(1020, result.Length);
Assert.IsFalse(result[0].Id == default);
Assert.IsFalse(result[0].Acct == default);
Assert.IsFalse(result[0].LastTweetPostedId == default);
Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
}
[TestMethod]
@ -371,7 +430,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
// Update accounts
var now = DateTime.UtcNow;
var allUsers = await dal.GetAllTwitterUsersAsync();
var allUsers = await dal.GetAllTwitterUsersAsync(false);
foreach (var acc in allUsers)
{
var lastSync = now.AddDays(acc.LastTweetPostedId);
@ -382,7 +441,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
// Create a not init account
await dal.CreateTwitterUserAsync("not_init", -1);
var result = await dal.GetAllTwitterUsersAsync(10);
var result = await dal.GetAllTwitterUsersAsync(10, false);
Assert.IsTrue(result.Any(x => x.Acct == "myid0"));
Assert.IsTrue(result.Any(x => x.Acct == "myid8"));
@ -405,7 +464,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
await dal.CreateTwitterUserAsync(acct, lastTweetId);
}
var allUsers = await dal.GetAllTwitterUsersAsync(100);
var allUsers = await dal.GetAllTwitterUsersAsync(100, false);
for (var i = 0; i < 20; i++)
{
var user = allUsers[i];
@ -413,7 +472,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date, null, null, false);
}
var result = await dal.GetAllTwitterUsersAsync(10);
var result = await dal.GetAllTwitterUsersAsync(10, false);
Assert.AreEqual(10, result.Length);
Assert.IsFalse(result[0].Id == default);
Assert.IsFalse(result[0].Acct == default);
@ -444,7 +503,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain");
}
var result = await dal.GetAllTwitterUsersAsync();
var result = await dal.GetAllTwitterUsersAsync(false);
Assert.AreEqual(1000, result.Length);
Assert.IsFalse(result[0].Id == default);
Assert.IsFalse(result[0].Acct == default);

8
src/Tests/BirdsiteLive.Moderation.Tests/Processors/TwitterAccountModerationProcessorTests.cs
View File

@ -48,7 +48,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors
#region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync())
.Setup(x => x.GetAllTwitterUsersAsync(It.Is<bool>(y => y == false)))
.ReturnsAsync(allUsers.ToArray());
var moderationRepositoryMock = new Mock<IModerationRepository>(MockBehavior.Strict);
@ -87,7 +87,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors
#region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync())
.Setup(x => x.GetAllTwitterUsersAsync(It.Is<bool>(y => y == false)))
.ReturnsAsync(allUsers.ToArray());
var moderationRepositoryMock = new Mock<IModerationRepository>(MockBehavior.Strict);
@ -130,7 +130,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors
#region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync())
.Setup(x => x.GetAllTwitterUsersAsync(It.Is<bool>(y => y == false)))
.ReturnsAsync(allUsers.ToArray());
var moderationRepositoryMock = new Mock<IModerationRepository>(MockBehavior.Strict);
@ -173,7 +173,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors
#region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync())
.Setup(x => x.GetAllTwitterUsersAsync(It.Is<bool>(y => y == false)))
.ReturnsAsync(allUsers.ToArray());
var moderationRepositoryMock = new Mock<IModerationRepository>(MockBehavior.Strict);

15
src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs
View File

@ -40,7 +40,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == maxUsers)))
It.Is<int>(y => y == maxUsers),
It.Is<bool>(y => y == false)))
.ReturnsAsync(users);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
@ -83,7 +84,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.SetupSequence(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == maxUsers)))
It.Is<int>(y => y == maxUsers),
It.Is<bool>(y => y == false)))
.ReturnsAsync(users.ToArray())
.ReturnsAsync(new SyncTwitterUser[0])
.ReturnsAsync(new SyncTwitterUser[0])
@ -130,7 +132,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.SetupSequence(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == maxUsers)))
It.Is<int>(y => y == maxUsers),
It.Is<bool>(y => y == false)))
.ReturnsAsync(users.ToArray())
.ReturnsAsync(new SyncTwitterUser[0])
.ReturnsAsync(new SyncTwitterUser[0])
@ -178,7 +181,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == maxUsers)))
It.Is<int>(y => y == maxUsers),
It.Is<bool>(y => y == false)))
.ReturnsAsync(new SyncTwitterUser[0]);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
@ -215,7 +219,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == maxUsers)))
It.Is<int>(y => y == maxUsers),
It.Is<bool>(y => y == false)))
.Returns(async () => await DelayFaultedTask<SyncTwitterUser[]>(new Exception()));
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();