sam/BirdsiteLIVE
sam/BirdsiteLIVE
Archived
1
0
Fork 0
Code Issues 1 Pull requests 1 Packages Projects Releases Wiki Activity

reduced batch size

Browse source
This commit is contained in:
Vincent Cloutier 2022-12-25 14:41:28 -05:00
parent 97f982903e
commit e7438057d1
2 changed files with 4 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs
View file

@ -58,7 +58,7 @@ namespace BirdsiteLive.Pipeline.Processors
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now);
}
await Task.Delay(10 * 1000);
await Task.Delay(4 * 1000);
}
return usersWtTweets.ToArray();

17
src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
View file

@ -38,29 +38,18 @@ namespace BirdsiteLive.Pipeline.Processors
try
{
var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync();
var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber);
var users = await _twitterUserDal.GetAllTwitterUsersAsync(50);
var userCount = users.Any() ? users.Length : 1;
var splitNumber = (int) Math.Ceiling(userCount / 15d);
var splitUsers = users.Split(splitNumber).ToList();
var splitUsers = users.Split(50).ToList();
foreach (var u in splitUsers)
{
ct.ThrowIfCancellationRequested();
await twitterUsersBufferBlock.SendAsync(u.ToArray(), ct);
await Task.Delay(WaitFactor, ct);
}
var splitCount = splitUsers.Count();
if (splitCount < 15) await Task.Delay((15 - splitCount) * WaitFactor, ct); //Always wait 15min
//// Extra wait time to fit 100.000/day limit
//var extraWaitTime = (int)Math.Ceiling((60 / ((100000d / 24) / userCount)) - 15);
//if (extraWaitTime < 0) extraWaitTime = 0;
//await Task.Delay(extraWaitTime * 1000, ct);
await Task.Delay(1000, ct);
}
catch (Exception e)
{