sam/BirdsiteLIVE
sam
/
BirdsiteLIVE
Archived
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

RetrieveTwitterUsersProcessor tweaks

This commit is contained in:
Vincent Cloutier 2023-04-02 10:23:45 -04:00
parent 3346b7b5e8
commit 46f7594e43
1 changed files with 21 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
View File

@ -38,31 +38,31 @@ namespace BirdsiteLive.Pipeline.Processors
{
ct.ThrowIfCancellationRequested();
try
if (_instanceSettings.ParallelTwitterRequests == 0)
{
var users = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000, _instanceSettings.n_start, _instanceSettings.n_end, _instanceSettings.m);
var userCount = users.Any() ? Math.Min(users.Length, 200) : 1;
var splitUsers = users.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList();
foreach (var u in splitUsers)
{
ct.ThrowIfCancellationRequested();
UserWithDataToSync[] toSync = await Task.WhenAll(
u.Select(async x => new UserWithDataToSync
{ User = x, Followers = await _followersDal.GetFollowersAsync(x.Id) }
)
);
await twitterUsersBufferBlock.SendAsync(toSync, ct);
}
await Task.Delay(10, ct); // this is somehow necessary
while (true)
await Task.Delay(10000);
}
catch (Exception e)
var users = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000, _instanceSettings.n_start, _instanceSettings.n_end, _instanceSettings.m);
var userCount = users.Any() ? Math.Min(users.Length, 200) : 1;
var splitUsers = users.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList();
foreach (var u in splitUsers)
{
_logger.LogError(e, "Failing retrieving Twitter Users.");
ct.ThrowIfCancellationRequested();
UserWithDataToSync[] toSync = await Task.WhenAll(
u.Select(async x => new UserWithDataToSync
{ User = x, Followers = await _followersDal.GetFollowersAsync(x.Id) }
)
);
await twitterUsersBufferBlock.SendAsync(toSync, ct);
}
await Task.Delay(10, ct); // this is somehow necessary
}
}
}