Merge branch 'master' of bird.makeup into makeup
Some checks failed
continuous-integration/drone/push Build is failing

This commit is contained in:
Sam Therapy 2023-04-03 16:23:10 +02:00
commit fa5b50f92b
Signed by: sam
GPG key ID: 4D8B07C18F31ACBD
51 changed files with 642 additions and 382 deletions

View file

@ -13,8 +13,12 @@ tasks:
sudo docker run -d -p 5432:5432 -e POSTGRES_PASSWORD=birdsitelive -e POSTGRES_USER=birdsitelive -e POSTGRES_DB=birdsitelive postgres:15
cd bird.makeup/src
dotnet test
- publish: |
- publish-arm: |
cd bird.makeup/src/BirdsiteLive
dotnet publish --os linux --arch arm64 /t:PublishContainer -c Release
docker tag cloutier/bird.makeup:1.0 cloutier/bird.makeup:latest-arm
docker push cloutier/bird.makeup:latest-arm
- publish-x64: |
cd bird.makeup/src/BirdsiteLive
dotnet publish --os linux --arch x64 /t:PublishContainer -c Release
docker tag cloutier/bird.makeup:1.0 cloutier/bird.makeup:latest

View file

@ -1,8 +1,8 @@
using System;
using System.IO;
using System.Text.Json;
using System.Runtime.CompilerServices;
using BirdsiteLive.Common.Settings;
using Newtonsoft.Json;
namespace BSLManager.Tools
{
@ -94,7 +94,7 @@ namespace BSLManager.Tools
if (!File.Exists(LocalFileName)) return null;
var jsonContent = File.ReadAllText(LocalFileName);
var content = JsonConvert.DeserializeObject<LocalSettingsData>(jsonContent);
var content = JsonSerializer.Deserialize<LocalSettingsData>(jsonContent);
return content;
}
catch (Exception)
@ -105,7 +105,7 @@ namespace BSLManager.Tools
private void SaveLocalSettings(LocalSettingsData data)
{
var jsonContent = JsonConvert.SerializeObject(data);
var jsonContent = JsonSerializer.Serialize(data);
File.WriteAllText(LocalFileName, jsonContent);
}
}

View file

@ -1,6 +1,7 @@
using System;
using BirdsiteLive.ActivityPub.Models;
using Newtonsoft.Json;
using System.Text.Json.Serialization;
using System.Text.Json;
namespace BirdsiteLive.ActivityPub
{
@ -10,22 +11,21 @@ namespace BirdsiteLive.ActivityPub
{
try
{
var activity = JsonConvert.DeserializeObject<Activity>(json);
var activity = JsonSerializer.Deserialize<Activity>(json);
switch (activity.type)
{
case "Follow":
return JsonConvert.DeserializeObject<ActivityFollow>(json);
return JsonSerializer.Deserialize<ActivityFollow>(json);
case "Undo":
var a = JsonConvert.DeserializeObject<ActivityUndo>(json);
var a = JsonSerializer.Deserialize<ActivityUndo>(json);
if(a.apObject.type == "Follow")
return JsonConvert.DeserializeObject<ActivityUndoFollow>(json);
return JsonSerializer.Deserialize<ActivityUndoFollow>(json);
break;
case "Delete":
return JsonConvert.DeserializeObject<ActivityDelete>(json);
return JsonSerializer.Deserialize<ActivityDelete>(json);
case "Accept":
var accept = JsonConvert.DeserializeObject<ActivityAccept>(json);
//var acceptType = JsonConvert.DeserializeObject<Activity>(accept.apObject);
switch ((accept.apObject as dynamic).type.ToString())
var accept = JsonSerializer.Deserialize<ActivityAccept>(json);
switch (accept.apObject.type)
{
case "Follow":
var acceptFollow = new ActivityAcceptFollow()
@ -36,11 +36,12 @@ namespace BirdsiteLive.ActivityPub
context = accept.context,
apObject = new ActivityFollow()
{
id = (accept.apObject as dynamic).id?.ToString(),
type = (accept.apObject as dynamic).type?.ToString(),
actor = (accept.apObject as dynamic).actor?.ToString(),
context = (accept.apObject as dynamic).context?.ToString(),
apObject = (accept.apObject as dynamic).@object?.ToString()
id = accept.apObject.id,
type = accept.apObject.type,
actor = accept.apObject.actor,
context = accept.apObject.context?.ToString(),
apObject = accept.apObject.apObject,
}
};
return acceptFollow;
@ -56,10 +57,5 @@ namespace BirdsiteLive.ActivityPub
return null;
}
private class Ac : Activity
{
[JsonProperty("object")]
public Activity apObject { get; set; }
}
}
}

View file

@ -6,7 +6,6 @@
<ItemGroup>
<PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
</ItemGroup>
</Project>

View file

@ -1,39 +0,0 @@
using System;
using System.Collections.Generic;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub.Converters
{
public class ContextArrayConverter : JsonConverter
{
public override bool CanWrite { get { return false; } }
public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
{
throw new NotImplementedException();
}
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
{
var result = new List<string>();
var list = serializer.Deserialize<List<object>>(reader);
foreach (var l in list)
{
if (l is string s)
result.Add(s);
else
{
var str = JsonConvert.SerializeObject(l);
result.Add(str);
}
}
return result.ToArray();
}
public override bool CanConvert(Type objectType)
{
throw new NotImplementedException();
}
}
}

View file

@ -1,36 +1,17 @@
using System.Collections.Generic;
using System.Collections.Generic;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub
{
public class Activity
{
[Newtonsoft.Json.JsonIgnore]
public static readonly object[] DefaultContext = new object[] {
"https://www.w3.org/ns/activitystreams",
"https://w3id.org/security/v1",
new Dictionary<string, string>
{
{ "Emoji", "toot:Emoji" },
{ "Hashtag", "as:Hashtag" },
{ "PropertyValue", "schema:PropertyValue" },
{ "value", "schema:value" },
{ "sensitive", "as:sensitive" },
{ "quoteUrl", "as:quoteUrl" },
{ "schema", "http://schema.org#" },
{ "toot", "https://joinmastodon.org/ns#" }
}
};
[JsonProperty("@context")]
public object context { get; set; } = DefaultContext;
[JsonPropertyName("@context")]
public string context { get; set; }
public string id { get; set; }
public string type { get; set; }
public string actor { get; set; }
//[JsonProperty("object")]
//[JsonPropertyName("object")]
//public string apObject { get; set; }
}
}

View file

@ -1,10 +1,10 @@
using Newtonsoft.Json;
using System.Text.Json.Serialization;
namespace BirdsiteLive.ActivityPub
{
public class ActivityAccept : Activity
{
[JsonProperty("object")]
public object apObject { get; set; }
[JsonPropertyName("object")]
public NestedActivity apObject { get; set; }
}
}

View file

@ -1,10 +1,10 @@
using Newtonsoft.Json;
using System.Text.Json.Serialization;
namespace BirdsiteLive.ActivityPub
{
public class ActivityAcceptFollow : Activity
{
[JsonProperty("object")]
[JsonPropertyName("object")]
public ActivityFollow apObject { get; set; }
}
}

View file

@ -1,10 +1,10 @@
using Newtonsoft.Json;
using System.Text.Json.Serialization;
namespace BirdsiteLive.ActivityPub
{
public class ActivityAcceptUndoFollow : Activity
{
[JsonProperty("object")]
[JsonPropertyName("object")]
public ActivityUndoFollow apObject { get; set; }
}
}

View file

@ -1,6 +1,5 @@
using System;
using BirdsiteLive.ActivityPub.Models;
using Newtonsoft.Json;
using BirdsiteLive.ActivityPub.Models;
using System.Text.Json.Serialization;
namespace BirdsiteLive.ActivityPub
{
@ -10,7 +9,7 @@ namespace BirdsiteLive.ActivityPub
public string[] to { get; set; }
public string[] cc { get; set; }
[JsonProperty("object")]
[JsonPropertyName("object")]
public Note apObject { get; set; }
}
}

View file

@ -1,10 +1,10 @@
using Newtonsoft.Json;
using System.Text.Json.Serialization;
namespace BirdsiteLive.ActivityPub.Models
{
public class ActivityDelete : Activity
{
[JsonProperty("object")]
public object apObject { get; set; }
[JsonPropertyName("object")]
public string apObject { get; set; }
}
}

View file

@ -1,10 +1,10 @@
using Newtonsoft.Json;
using System.Text.Json.Serialization;
namespace BirdsiteLive.ActivityPub
{
public class ActivityFollow : Activity
{
[JsonProperty("object")]
[JsonPropertyName("object")]
public string apObject { get; set; }
}
}

View file

@ -1,10 +1,10 @@
using Newtonsoft.Json;
using System.Text.Json.Serialization;
namespace BirdsiteLive.ActivityPub
{
public class ActivityRejectFollow : Activity
{
[JsonProperty("object")]
[JsonPropertyName("object")]
public ActivityFollow apObject { get; set; }
}
}

View file

@ -1,10 +1,10 @@
using Newtonsoft.Json;
using System.Text.Json.Serialization;
namespace BirdsiteLive.ActivityPub
{
public class ActivityUndo : Activity
{
[JsonProperty("object")]
[JsonPropertyName("object")]
public Activity apObject { get; set; }
}
}

View file

@ -1,10 +1,10 @@
using Newtonsoft.Json;
using System.Text.Json.Serialization;
namespace BirdsiteLive.ActivityPub
{
public class ActivityUndoFollow : Activity
{
[JsonProperty("object")]
[JsonPropertyName("object")]
public ActivityFollow apObject { get; set; }
}
}

View file

@ -1,15 +1,13 @@
using System.Net;
using BirdsiteLive.ActivityPub.Converters;
using Newtonsoft.Json;
using System.Text.Json.Serialization;
namespace BirdsiteLive.ActivityPub
{
public class Actor
{
//[JsonPropertyName("@context")]
[JsonProperty("@context")]
[JsonConverter(typeof(ContextArrayConverter))]
public object[] context { get; set; } = Activity.DefaultContext;
[JsonPropertyName("@context")]
public object[] context { get; set; } = new string[] { "https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1" };
public string id { get; set; }
public string type { get; set; }
public string followers { get; set; }

View file

@ -1,13 +1,12 @@
using BirdsiteLive.ActivityPub.Converters;
using Newtonsoft.Json;
using System.Text.Json.Serialization;
namespace BirdsiteLive.ActivityPub.Models
{
public class Followers
{
[JsonProperty("@context")]
[JsonConverter(typeof(ContextArrayConverter))]
public object[] context { get; set; } = Activity.DefaultContext;
[JsonPropertyName("@context")]
public string context { get; set; } = "https://www.w3.org/ns/activitystreams";
public string id { get; set; }
public string type { get; set; } = "OrderedCollection";

View file

@ -0,0 +1,17 @@
using System.Text.Json.Serialization;
namespace BirdsiteLive.ActivityPub
{
public class NestedActivity
{
[JsonPropertyName("@context")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public object context { get; set; }
public string id { get; set; }
public string type { get; set; }
public string actor { get; set; }
[JsonPropertyName("object")]
public string apObject { get; set; }
}
}

View file

@ -1,13 +1,11 @@
using BirdsiteLive.ActivityPub.Converters;
using Newtonsoft.Json;
using System.Text.Json.Serialization;
namespace BirdsiteLive.ActivityPub.Models
{
public class Note
{
[JsonProperty("@context")]
[JsonConverter(typeof(ContextArrayConverter))]
public object[] context { get; set; } = Activity.DefaultContext;
[JsonPropertyName("@context")]
public string[] context { get; set; } = new[] { "https://www.w3.org/ns/activitystreams" };
public string id { get; set; }
public string announceId { get; set; }

View file

@ -14,8 +14,13 @@
public int FailingTwitterUserCleanUpThreshold { get; set; }
public int FailingFollowerCleanUpThreshold { get; set; } = -1;
public int UserCacheCapacity { get; set; } = 20_000;
public int UserCacheCapacity { get; set; } = 40_000;
public int TweetCacheCapacity { get; set; } = 20_000;
// "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
public string TwitterBearerToken { get; set; } = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA";
public int m { get; set; } = 1;
public int n_start { get; set; } = 0;
public int n_end { get; set; } = 1;
public int ParallelTwitterRequests { get; set; } = 10;
public int ParallelFediversePosts { get; set; } = 10;
}

View file

@ -5,13 +5,14 @@ using System.Net.Http;
using System.Runtime.InteropServices;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
using BirdsiteLive.ActivityPub;
using BirdsiteLive.ActivityPub.Converters;
using BirdsiteLive.ActivityPub.Models;
using BirdsiteLive.Common.Settings;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
namespace BirdsiteLive.Domain
{
@ -21,6 +22,8 @@ namespace BirdsiteLive.Domain
Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null);
Task PostNewActivity(ActivityCreateNote note, string username, string noteId, string targetHost,
string targetInbox);
ActivityAcceptFollow BuildAcceptFollow(ActivityFollow activity);
}
public class ActivityPubService : IActivityPubService
@ -53,7 +56,7 @@ namespace BirdsiteLive.Domain
var content = await result.Content.ReadAsStringAsync();
var actor = JsonConvert.DeserializeObject<Actor>(content);
var actor = JsonSerializer.Deserialize<Actor>(content);
if (string.IsNullOrWhiteSpace(actor.url)) actor.url = objectId;
return actor;
}
@ -73,13 +76,32 @@ namespace BirdsiteLive.Domain
}
}
public async Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null)
public ActivityAcceptFollow BuildAcceptFollow(ActivityFollow activity)
{
var acceptFollow = new ActivityAcceptFollow()
{
context = "https://www.w3.org/ns/activitystreams",
id = $"{activity.apObject}#accepts/follows/{Guid.NewGuid()}",
type = "Accept",
actor = activity.apObject,
apObject = new ActivityFollow()
{
id = activity.id,
type = activity.type,
actor = activity.actor,
apObject = activity.apObject
}
};
return acceptFollow;
}
public HttpRequestMessage BuildRequest<T>(T data, string targetHost, string actorUrl,
string inbox = null)
{
var usedInbox = $"/inbox";
if (!string.IsNullOrWhiteSpace(inbox))
usedInbox = inbox;
var json = JsonConvert.SerializeObject(data, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore });
var json = JsonSerializer.Serialize(data);
var date = DateTime.UtcNow.ToUniversalTime();
var httpDate = date.ToString("r");
@ -88,22 +110,30 @@ namespace BirdsiteLive.Domain
var signature = _cryptoService.SignAndGetSignatureHeader(date, actorUrl, targetHost, digest, usedInbox);
var client = _httpClientFactory.CreateClient("BirdsiteLIVE");
client.Timeout = TimeSpan.FromSeconds(2);
var httpRequestMessage = new HttpRequestMessage
{
Method = HttpMethod.Post,
RequestUri = new Uri($"https://{targetHost}{usedInbox}"),
Headers =
{
{"Host", targetHost},
{"Date", httpDate},
{"Signature", signature},
{"Digest", $"SHA-256={digest}"}
{ "Host", targetHost },
{ "Date", httpDate },
{ "Signature", signature },
{ "Digest", $"SHA-256={digest}" }
},
Content = new StringContent(json, Encoding.UTF8, "application/ld+json")
};
return httpRequestMessage;
}
public async Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null)
{
var httpRequestMessage = BuildRequest(data, targetHost, actorUrl, inbox);
var client = _httpClientFactory.CreateClient("BirdsiteLIVE");
client.Timeout = TimeSpan.FromSeconds(2);
var response = await client.SendAsync(httpRequestMessage);
response.EnsureSuccessStatusCode();

View file

@ -183,30 +183,11 @@ namespace BirdsiteLive.Domain
private async Task<bool> SendAcceptFollowAsync(ActivityFollow activity, string followerHost)
{
var acceptFollow = new ActivityAcceptFollow()
{
context = "https://www.w3.org/ns/activitystreams",
id = $"{activity.apObject}#accepts/follows/{Guid.NewGuid()}",
type = "Accept",
actor = activity.apObject,
apObject = new ActivityFollow()
{
id = activity.id,
type = activity.type,
actor = activity.actor,
apObject = activity.apObject
}
};
try
{
var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject);
return result == HttpStatusCode.Accepted ||
result == HttpStatusCode.OK;
}
catch (Exception e)
{
return false;
}
var acceptFollow = _activityPubService.BuildAcceptFollow(activity);
var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject);
return result == HttpStatusCode.Accepted ||
result == HttpStatusCode.OK; //TODO: revamp this for better error handling
}
public async Task<bool> SendRejectFollowAsync(ActivityFollow activity, string followerHost)
@ -264,10 +245,11 @@ namespace BirdsiteLive.Domain
actor = activity.apObject.apObject,
apObject = new ActivityUndoFollow()
{
id = activity.id,
type = activity.type,
actor = activity.actor,
apObject = activity.apObject
id = (activity.apObject as dynamic).id?.ToString(),
type = (activity.apObject as dynamic).type?.ToString(),
actor = (activity.apObject as dynamic).actor?.ToString(),
context = (activity.apObject as dynamic).context?.ToString(),
apObject = (activity.apObject as dynamic).@object?.ToString()
}
};
var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject.apObject);

View file

@ -6,6 +6,6 @@ namespace BirdsiteLive.Pipeline.Contracts
{
public interface ISendTweetsToFollowersProcessor
{
Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct);
Task ProcessAsync(UserWithDataToSync[] usersWithTweetsToSync, CancellationToken ct);
}
}

View file

@ -21,18 +21,18 @@ namespace BirdsiteLive.Pipeline.Processors
public async Task<IEnumerable<UserWithDataToSync>> ProcessAsync(UserWithDataToSync[] userWithTweetsToSyncs, CancellationToken ct)
{
List<Task> todo = new List<Task>();
foreach (var user in userWithTweetsToSyncs)
{
var t = Task.Run(
async() => {
var followers = await _followersDal.GetFollowersAsync(user.User.Id);
user.Followers = followers;
});
todo.Add(t);
}
await Task.WhenAll(todo);
//List<Task> todo = new List<Task>();
//foreach (var user in userWithTweetsToSyncs)
//{
// var t = Task.Run(
// async() => {
// var followers = await _followersDal.GetFollowersAsync(user.User.Id);
// user.Followers = followers;
// });
// todo.Add(t);
//}
//
//await Task.WhenAll(todo);
return userWithTweetsToSyncs;
}

View file

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
@ -15,15 +16,19 @@ namespace BirdsiteLive.Pipeline.Processors
public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor
{
private readonly ITwitterUserDal _twitterUserDal;
private readonly IFollowersDal _followersDal;
private readonly InstanceSettings _instanceSettings;
private readonly ILogger<RetrieveTwitterUsersProcessor> _logger;
private static Random rng = new Random();
public int WaitFactor = 1000 * 60; //1 min
#region Ctor
public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, ILogger<RetrieveTwitterUsersProcessor> logger)
public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, InstanceSettings instanceSettings, ILogger<RetrieveTwitterUsersProcessor> logger)
{
_twitterUserDal = twitterUserDal;
_followersDal = followersDal;
_instanceSettings = instanceSettings;
_logger = logger;
}
#endregion
@ -34,27 +39,37 @@ namespace BirdsiteLive.Pipeline.Processors
{
ct.ThrowIfCancellationRequested();
try
if (_instanceSettings.ParallelTwitterRequests == 0)
{
var users = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000);
while (true)
await Task.Delay(10000);
}
var usersDal = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000, _instanceSettings.n_start, _instanceSettings.n_end, _instanceSettings.m);
var userCount = users.Any() ? Math.Min(users.Length, 200) : 1;
var splitUsers = users.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList();
var userCount = usersDal.Any() ? Math.Min(usersDal.Length, 200) : 1;
var splitUsers = usersDal.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList();
foreach (var u in splitUsers)
foreach (var users in splitUsers)
{
ct.ThrowIfCancellationRequested();
List<UserWithDataToSync> toSync = new List<UserWithDataToSync>();
foreach (var u in users)
{
ct.ThrowIfCancellationRequested();
UserWithDataToSync[] toSync = u.Select(x => new UserWithDataToSync { User = x }).ToArray();
await twitterUsersBufferBlock.SendAsync(toSync, ct);
var followers = await _followersDal.GetFollowersAsync(u.Id);
toSync.Add( new UserWithDataToSync()
{
User = u,
Followers = followers
});
}
await Task.Delay(10, ct); // this is somehow necessary
}
catch (Exception e)
{
_logger.LogError(e, "Failing retrieving Twitter Users.");
await twitterUsersBufferBlock.SendAsync(toSync.ToArray(), ct);
}
await Task.Delay(10, ct); // this is somehow necessary
}
}
}

View file

@ -41,34 +41,40 @@ namespace BirdsiteLive.Pipeline.Processors
}
#endregion
public async Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
public async Task ProcessAsync(UserWithDataToSync[] usersWithTweetsToSync, CancellationToken ct)
{
var user = userWithTweetsToSync.User;
_todo = _todo.Where(x => !x.IsCompleted).ToList();
var t = Task.Run( async () =>
foreach (var userWithTweetsToSync in usersWithTweetsToSync)
{
// Process Shared Inbox
var followersWtSharedInbox = userWithTweetsToSync.Followers
.Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
.ToList();
await ProcessFollowersWithSharedInboxAsync(userWithTweetsToSync.Tweets, followersWtSharedInbox, user);
var user = userWithTweetsToSync.User;
// Process Inbox
var followerWtInbox = userWithTweetsToSync.Followers
.Where(x => string.IsNullOrWhiteSpace(x.SharedInboxRoute))
.ToList();
await ProcessFollowersWithInboxAsync(userWithTweetsToSync.Tweets, followerWtInbox, user);
});
_todo.Add(t);
_todo = _todo.Where(x => !x.IsCompleted).ToList();
var t = Task.Run( async () =>
{
// Process Shared Inbox
var followersWtSharedInbox = userWithTweetsToSync.Followers
.Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
.ToList();
await ProcessFollowersWithSharedInboxAsync(userWithTweetsToSync.Tweets, followersWtSharedInbox, user);
if (_todo.Count >= _instanceSettings.ParallelFediversePosts)
{
await Task.WhenAny(_todo);
// Process Inbox
var followerWtInbox = userWithTweetsToSync.Followers
.Where(x => string.IsNullOrWhiteSpace(x.SharedInboxRoute))
.ToList();
await ProcessFollowersWithInboxAsync(userWithTweetsToSync.Tweets, followerWtInbox, user);
_logger.LogInformation("Done sending " + userWithTweetsToSync.Tweets.Length + " tweets for "
+ userWithTweetsToSync.Followers.Length + "followers for user " + userWithTweetsToSync.User.Acct);
}, ct);
_todo.Add(t);
if (_todo.Count >= _instanceSettings.ParallelFediversePosts)
{
await Task.WhenAny(_todo);
}
}
_logger.LogInformation("Done sending " + userWithTweetsToSync.Followers.Length + "tweets for user " + userWithTweetsToSync.User.Acct);
}

View file

@ -43,7 +43,6 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
var tweetsToSend = tweets
.OrderBy(x => x.Id)
.ToList();
_logger.LogInformation("After filtering, there were " + tweetsToSend.Count() + " tweets left to send");
foreach (var tweet in tweetsToSend)
{

View file

@ -44,17 +44,15 @@ namespace BirdsiteLive.Pipeline
var twitterUserToRefreshBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions
{ BoundedCapacity = 1, CancellationToken = ct });
var retrieveTweetsBlock = new TransformBlock<UserWithDataToSync[], UserWithDataToSync[]>(async x => await _retrieveTweetsProcessor.ProcessAsync(x, ct), standardBlockOptions );
var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
var retrieveFollowersBlock = new TransformManyBlock<UserWithDataToSync[], UserWithDataToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { BoundedCapacity = 1 } );
var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 500, CancellationToken = ct });
var sendTweetsToFollowersBlock = new ActionBlock<UserWithDataToSync>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), standardBlockOptions);
var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 2, CancellationToken = ct });
// var retrieveFollowersBlock = new TransformManyBlock<UserWithDataToSync[], UserWithDataToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { BoundedCapacity = 1 } );
// var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 500, CancellationToken = ct });
var sendTweetsToFollowersBlock = new ActionBlock<UserWithDataToSync[]>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), standardBlockOptions);
// Link pipeline
twitterUserToRefreshBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions { PropagateCompletion = true });
retrieveTweetsBlock.LinkTo(retrieveTweetsBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
retrieveTweetsBufferBlock.LinkTo(retrieveFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
retrieveFollowersBlock.LinkTo(retrieveFollowersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
retrieveFollowersBufferBlock.LinkTo(sendTweetsToFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
retrieveTweetsBufferBlock.LinkTo(sendTweetsToFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
// Launch twitter user retriever after a little delay
// to give time for the Tweet cache to fill

View file

@ -7,6 +7,7 @@
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="5.0.0" />
<PackageReference Include="Microsoft.Extensions.Http" Version="5.0.0" />
<PackageReference Include="System.Threading.RateLimiting" Version="7.0.0" />
</ItemGroup>
<ItemGroup>

View file

@ -11,6 +11,7 @@ namespace BirdsiteLive.Twitter
{
void PurgeUser(string username);
void AddUser(TwitterUser user);
bool UserIsCached(string username);
}
public class CachedTwitterUserService : ICachedTwitterUserService
@ -39,6 +40,10 @@ namespace BirdsiteLive.Twitter
}
#endregion
public bool UserIsCached(string username)
{
return _userCache.TryGetValue(username, out _);
}
public async Task<TwitterUser> GetUserAsync(string username)
{
if (!_userCache.TryGetValue(username, out Task<TwitterUser> user))

View file

@ -1,20 +1,24 @@
using System;
using System.Threading;
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.Linq;
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using Microsoft.Extensions.Logging;
using System.Net.Http;
using System.Net;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Threading.RateLimiting;
namespace BirdsiteLive.Twitter.Tools
{
public interface ITwitterAuthenticationInitializer
{
Task<HttpClient> MakeHttpClient();
HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint);
HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint, bool addToken);
Task RefreshClient(HttpRequestMessage client);
}
@ -22,79 +26,82 @@ namespace BirdsiteLive.Twitter.Tools
{
private readonly ILogger<TwitterAuthenticationInitializer> _logger;
private static bool _initialized;
private static System.Timers.Timer aTimer;
private readonly IHttpClientFactory _httpClientFactory;
private List<HttpClient> _twitterClients = new List<HttpClient>();
private List<String> _tokens = new List<string>();
private ConcurrentDictionary<String, String> _token2 = new ConcurrentDictionary<string, string>();
static Random rnd = new Random();
private const int _targetClients = 20;
private RateLimiter _rateLimiter;
private const int _targetClients = 3;
private InstanceSettings _instanceSettings;
private readonly (string, string)[] _apiKeys = new[]
{
("IQKbtAYlXLripLGPWd0HUA", "GgDYlkSvaPxGxC4X8liwpUoqKwwr3lCADbz8A7ADU"), // iPhone
("3nVuSoBZnx6U4vzUxf5w", "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"), // Android
("CjulERsDeqhhjSme66ECg", "IQWdVyqFxghAtURHGeGiWAsmCAGmdW3WmbEx6Hck"), // iPad
("3rJOl1ODzm9yZy63FACdg", "5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8"), // Mac
};
public String BearerToken {
get { return "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"; }
get
{
return _instanceSettings.TwitterBearerToken;
}
}
#region Ctor
public TwitterAuthenticationInitializer(IHttpClientFactory httpClientFactory, ILogger<TwitterAuthenticationInitializer> logger)
public TwitterAuthenticationInitializer(IHttpClientFactory httpClientFactory, InstanceSettings settings, ILogger<TwitterAuthenticationInitializer> logger)
{
_logger = logger;
_instanceSettings = settings;
_httpClientFactory = httpClientFactory;
aTimer = new System.Timers.Timer();
aTimer.Interval = 20 * 1000;
aTimer.Elapsed += async (sender, e) => await RefreshCred();
aTimer.Start();
}
#endregion
private async Task<string> GenerateBearerToken()
{
var httpClient = _httpClientFactory.CreateClient();
using (var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.twitter.com/oauth2/token?grant_type=client_credentials"))
{
int r = rnd.Next(_apiKeys.Length);
var (login, password) = _apiKeys[r];
var authValue = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{login}:{password}")));
request.Headers.Authorization = authValue;
var httpResponse = await httpClient.SendAsync(request);
var c = await httpResponse.Content.ReadAsStringAsync();
httpResponse.EnsureSuccessStatusCode();
var doc = JsonDocument.Parse(c);
var token = doc.RootElement.GetProperty("access_token").GetString();
return token;
}
}
public async Task RefreshClient(HttpRequestMessage req)
{
string token = req.Headers.GetValues("x-guest-token").First();
var i = _tokens.IndexOf(token);
// this is prabably not thread save but yolo
try
{
_twitterClients.RemoveAt(i);
_tokens.RemoveAt(i);
}
catch (IndexOutOfRangeException _)
{
_logger.LogError("Error refreshing twitter token");
}
_token2.TryRemove(token, out _);
await RefreshCred();
await Task.Delay(1000);
await RefreshCred();
}
private async Task RefreshCred()
{
(string bearer, string guest) = await GetCred();
HttpClient client = _httpClientFactory.CreateClient();
//HttpClient client = new HttpClient();
client.DefaultRequestHeaders.TryAddWithoutValidation("Authorization", $"Bearer " + bearer);
client.DefaultRequestHeaders.TryAddWithoutValidation("x-guest-token", guest);
client.DefaultRequestHeaders.TryAddWithoutValidation("Referer", "https://twitter.com/");
client.DefaultRequestHeaders.TryAddWithoutValidation("x-twitter-active-user", "yes");
_twitterClients.Add(client);
_tokens.Add(guest);
if (_twitterClients.Count > _targetClients)
{
_twitterClients.RemoveAt(0);
_tokens.RemoveAt(0);
}
_token2.TryAdd(guest, bearer);
}
private async Task<(string, string)> GetCred()
{
string token;
var httpClient = _httpClientFactory.CreateClient();
string bearer = await GenerateBearerToken();
using (var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.twitter.com/1.1/guest/activate.json"))
{
request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + BearerToken);
request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer);
var httpResponse = await httpClient.SendAsync(request);
@ -104,40 +111,26 @@ namespace BirdsiteLive.Twitter.Tools
token = doc.RootElement.GetProperty("guest_token").GetString();
}
return (BearerToken, token);
return (bearer, token);
}
private async Task InitTwitterCredentials()
{
for (;;)
{
try
{
await RefreshCred();
_initialized = true;
return;
}
catch (Exception e)
{
_logger.LogError(e, "Twitter Authentication Failed");
await Task.Delay(3600*1000);
}
}
}
public async Task<HttpClient> MakeHttpClient()
{
if (_twitterClients.Count < _targetClients)
if (_token2.Count < _targetClients)
await RefreshCred();
int r = rnd.Next(_twitterClients.Count);
return _twitterClients[r];
return _httpClientFactory.CreateClient();
}
public HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint)
public HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint, bool addToken)
{
var request = new HttpRequestMessage(m, endpoint);
int r = rnd.Next(_twitterClients.Count);
request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + BearerToken);
request.Headers.TryAddWithoutValidation("x-guest-token", _tokens[r]);
//(string bearer, string token) = _tokens[r];
(string token, string bearer) = _token2.MaxBy(x => rnd.Next());
request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer);
request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");
request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes");
if (addToken)
request.Headers.TryAddWithoutValidation("x-guest-token", token);
//request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");
//request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes");
return request;

View file

@ -55,16 +55,14 @@ namespace BirdsiteLive.Twitter
"https://api.twitter.com/graphql/XjlydVWHFIDaAUny86oh2g/TweetDetail?variables=%7B%22focalTweetId%22%3A%22"
+ statusId +
"%22,%22with_rux_injections%22%3Atrue,%22includePromotedContent%22%3Afalse,%22withCommunity%22%3Afalse,%22withQuickPromoteEligibilityTweetFields%22%3Afalse,%22withBirdwatchNotes%22%3Afalse,%22withSuperFollowsUserFields%22%3Afalse,%22withDownvotePerspective%22%3Afalse,%22withReactionsMetadata%22%3Afalse,%22withReactionsPerspective%22%3Afalse,%22withSuperFollowsTweetFields%22%3Afalse,%22withVoice%22%3Atrue,%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue,%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue,%22verified_phone_label_enabled%22%3Afalse,%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue,%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse,%22tweetypie_unmention_optimization_enabled%22%3Atrue,%22vibe_api_enabled%22%3Atrue,%22responsive_web_edit_tweet_api_enabled%22%3Atrue,%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Afalse,%22view_counts_everywhere_api_enabled%22%3Atrue,%22longform_notetweets_consumption_enabled%22%3Atrue,%22tweet_awards_web_tipping_enabled%22%3Afalse,%22freedom_of_speech_not_reach_fetch_enabled%22%3Afalse,%22standardized_nudges_misinfo%22%3Atrue,%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse,%22interactive_text_enabled%22%3Atrue,%22responsive_web_text_conversations_enabled%22%3Afalse,%22longform_notetweets_richtext_consumption_enabled%22%3Afalse,%22responsive_web_enhance_cards_enabled%22%3Atrue%7D";
using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL, true);
try
{
JsonDocument tweet;
using (var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL))
{
var httpResponse = await client.SendAsync(request);
httpResponse.EnsureSuccessStatusCode();
var c = await httpResponse.Content.ReadAsStringAsync();
tweet = JsonDocument.Parse(c);
}
var httpResponse = await client.SendAsync(request);
httpResponse.EnsureSuccessStatusCode();
var c = await httpResponse.Content.ReadAsStringAsync();
tweet = JsonDocument.Parse(c);
var timeline = tweet.RootElement.GetProperty("data").GetProperty("threaded_conversation_with_injections_v2")
@ -77,6 +75,7 @@ namespace BirdsiteLive.Twitter
catch (Exception e)
{
_logger.LogError(e, "Error retrieving tweet {TweetId}", statusId);
await _twitterAuthenticationInitializer.RefreshClient(request);
return null;
}
}
@ -105,7 +104,7 @@ namespace BirdsiteLive.Twitter
userId + "%22,%22count%22%3A40,%22includePromotedContent%22%3Atrue,%22withCommunity%22%3Atrue,%22withSuperFollowsUserFields%22%3Atrue,%22withDownvotePerspective%22%3Afalse,%22withReactionsMetadata%22%3Afalse,%22withReactionsPerspective%22%3Afalse,%22withSuperFollowsTweetFields%22%3Atrue,%22withVoice%22%3Atrue,%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue,%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue,%22verified_phone_label_enabled%22%3Afalse,%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue,%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse,%22tweetypie_unmention_optimization_enabled%22%3Atrue,%22vibe_api_enabled%22%3Atrue,%22responsive_web_edit_tweet_api_enabled%22%3Atrue,%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue,%22view_counts_everywhere_api_enabled%22%3Atrue,%22longform_notetweets_consumption_enabled%22%3Atrue,%22tweet_awards_web_tipping_enabled%22%3Afalse,%22freedom_of_speech_not_reach_fetch_enabled%22%3Afalse,%22standardized_nudges_misinfo%22%3Atrue,%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse,%22interactive_text_enabled%22%3Atrue,%22responsive_web_text_conversations_enabled%22%3Afalse,%22longform_notetweets_richtext_consumption_enabled%22%3Afalse,%22responsive_web_enhance_cards_enabled%22%3Afalse%7D";
JsonDocument results;
List<ExtractedTweet> extractedTweets = new List<ExtractedTweet>();
using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL);
using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL, true);
try
{
@ -119,7 +118,7 @@ namespace BirdsiteLive.Twitter
catch (HttpRequestException e)
{
_logger.LogError(e, "Error retrieving timeline of {Username}; refreshing client", username);
await _twitterAuthenticationInitializer.RefreshClient(request);
//await _twitterAuthenticationInitializer.RefreshClient(request);
return null;
}
catch (Exception e)

View file

@ -40,7 +40,7 @@ namespace BirdsiteLive.Twitter
JsonDocument res;
var client = await _twitterAuthenticationInitializer.MakeHttpClient();
using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), endpoint.Replace("elonmusk", username));
using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), endpoint.Replace("elonmusk", username), true);
try
{

View file

@ -10,7 +10,6 @@ using BirdsiteLive.Common.Settings;
using BirdsiteLive.Domain;
using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.Mvc.ViewFeatures;
using Newtonsoft.Json;
namespace BirdsiteLive.Controllers
{

View file

@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Net.Mime;
using System.Text.RegularExpressions;
using System.Threading;
@ -19,13 +20,12 @@ using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Primitives;
using Newtonsoft.Json;
namespace BirdsiteLive.Controllers
{
public class UsersController : Controller
{
private readonly ITwitterUserService _twitterUserService;
private readonly ICachedTwitterUserService _twitterUserService;
private readonly ICachedTwitterTweetsService _twitterTweetService;
private readonly IUserService _userService;
private readonly IStatusService _statusService;
@ -33,7 +33,7 @@ namespace BirdsiteLive.Controllers
private readonly ILogger<UsersController> _logger;
#region Ctor
public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ICachedTwitterTweetsService twitterTweetService, ILogger<UsersController> logger)
public UsersController(ICachedTwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ICachedTwitterTweetsService twitterTweetService, ILogger<UsersController> logger)
{
_twitterUserService = twitterUserService;
_userService = userService;
@ -111,7 +111,7 @@ namespace BirdsiteLive.Controllers
if (isSaturated) return new ObjectResult("Too Many Requests") { StatusCode = 429 };
if (notFound) return NotFound();
var apUser = _userService.GetUser(user);
var jsonApUser = JsonConvert.SerializeObject(apUser);
var jsonApUser = System.Text.Json.JsonSerializer.Serialize(apUser);
return Content(jsonApUser, "application/activity+json; charset=utf-8");
}
}
@ -155,7 +155,7 @@ namespace BirdsiteLive.Controllers
if (r.Contains("application/activity+json"))
{
var jsonApUser = JsonConvert.SerializeObject(status);
var jsonApUser = JsonSerializer.Serialize(status);
return Content(jsonApUser, "application/activity+json; charset=utf-8");
}
}
@ -185,7 +185,7 @@ namespace BirdsiteLive.Controllers
var status = _statusService.GetActivity(id, tweet);
var jsonApUser = JsonConvert.SerializeObject(status);
var jsonApUser = JsonSerializer.Serialize(status);
return Content(jsonApUser, "application/activity+json; charset=utf-8");
}
@ -269,7 +269,7 @@ namespace BirdsiteLive.Controllers
{
id = $"https://{_instanceSettings.Domain}/users/{id}/followers"
};
var jsonApUser = JsonConvert.SerializeObject(followers);
var jsonApUser = JsonSerializer.Serialize(followers);
return Content(jsonApUser, "application/activity+json; charset=utf-8");
}
}

View file

@ -19,7 +19,8 @@
"commandName": "Project",
"launchBrowser": true,
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
"ASPNETCORE_ENVIRONMENT": "Development",
"Instance__ParallelTwitterRequests": "0"
},
"applicationUrl": "http://localhost:5000"
},

View file

@ -15,7 +15,7 @@ namespace BirdsiteLive.Services
private readonly ITwitterUserDal _twitterUserDal;
private readonly IFollowersDal _followersDal;
private static CachedStatistics _cachedStatistics;
private static Task<CachedStatistics> _cachedStatistics;
private readonly InstanceSettings _instanceSettings;
#region Ctor
@ -24,28 +24,36 @@ namespace BirdsiteLive.Services
_twitterUserDal = twitterUserDal;
_instanceSettings = instanceSettings;
_followersDal = followersDal;
_cachedStatistics = CreateStats();
}
#endregion
public async Task<CachedStatistics> GetStatisticsAsync()
{
if (_cachedStatistics == null ||
(DateTime.UtcNow - _cachedStatistics.RefreshedTime).TotalMinutes > 15)
var stats = await _cachedStatistics;
if ((DateTime.UtcNow - stats.RefreshedTime).TotalMinutes > 5)
{
var twitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync();
var twitterSyncLag = await _twitterUserDal.GetTwitterSyncLag();
var fediverseUsers = await _followersDal.GetFollowersCountAsync();
_cachedStatistics = new CachedStatistics
{
RefreshedTime = DateTime.UtcNow,
SyncLag = twitterSyncLag,
TwitterUsers = twitterUserCount,
FediverseUsers = fediverseUsers
};
_cachedStatistics = CreateStats();
}
return _cachedStatistics;
return stats;
}
private async Task<CachedStatistics> CreateStats()
{
var twitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync();
var twitterSyncLag = await _twitterUserDal.GetTwitterSyncLag();
var fediverseUsers = await _followersDal.GetFollowersCountAsync();
var stats = new CachedStatistics
{
RefreshedTime = DateTime.UtcNow,
SyncLag = twitterSyncLag,
TwitterUsers = twitterUserCount,
FediverseUsers = fediverseUsers
};
return stats;
}
}

View file

@ -10,6 +10,7 @@ using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Postgres.DataAccessLayers;
using BirdsiteLive.DAL.Postgres.Settings;
using BirdsiteLive.Models;
using BirdsiteLive.Services;
using BirdsiteLive.Twitter;
using BirdsiteLive.Twitter.Tools;
using Lamar;
@ -98,6 +99,8 @@ namespace BirdsiteLive
services.For<ITwitterUserService>().Use<TwitterUserService>().Singleton();
services.For<ITwitterAuthenticationInitializer>().Use<TwitterAuthenticationInitializer>().Singleton();
services.For<ICachedStatisticsService>().Use<CachedStatisticsService>().Singleton();
services.Scan(_ =>
{

View file

@ -118,14 +118,20 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
}
}
public async Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber)
public async Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber, int nStart, int nEnd, int m)
{
var query = "SELECT * FROM (SELECT unnest(followings) as follow FROM followers GROUP BY follow) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id ORDER BY lastSync ASC NULLS FIRST LIMIT $1";
var query = "SELECT * FROM (SELECT unnest(followings) as follow FROM followers GROUP BY follow) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id WHERE mod(id, $2) >= $3 AND mod(id, $2) <= $4 ORDER BY lastSync ASC NULLS FIRST LIMIT $1";
await using var connection = DataSource.CreateConnection();
await connection.OpenAsync();
await using var command = new NpgsqlCommand(query, connection) {
Parameters = { new() { Value = maxNumber}}
Parameters =
{
new() { Value = maxNumber},
new() { Value = m},
new() { Value = nStart},
new() { Value = nEnd}
}
};
var reader = await command.ExecuteReaderAsync();
var results = new List<SyncTwitterUser>();

View file

@ -9,7 +9,7 @@ namespace BirdsiteLive.DAL.Contracts
Task CreateTwitterUserAsync(string acct, long lastTweetPostedId);
Task<SyncTwitterUser> GetTwitterUserAsync(string acct);
Task<SyncTwitterUser> GetTwitterUserAsync(int id);
Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber);
Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber, int nStart, int nEnd, int m);
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber);
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync();
Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync);

View file

@ -3,31 +3,21 @@ using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub.Tests
{
//[TestClass]
//public class ActivityTests
//{
// [TestMethod]
// public void FollowDeserializationTest()
// {
// var json = "{ \"@context\":\"https://www.w3.org/ns/activitystreams\",\"id\":\"https://mastodon.technology/c94567cf-1fda-42ba-82fc-a0f82f63ccbe\",\"type\":\"Follow\",\"actor\":\"https://mastodon.technology/users/testtest\",\"object\":\"https://4a120ca2680e.ngrok.io/users/manu\"}";
[TestClass]
public class ActivityTests
{
[TestMethod]
public void Serialize()
{
var obj = new Actor
{
type = "Person",
id = "id"
};
// var data = JsonConvert.DeserializeObject<Activity>(json);
var json = JsonConvert.SerializeObject(obj);
// Assert.AreEqual("https://mastodon.technology/c94567cf-1fda-42ba-82fc-a0f82f63ccbe", data.id);
// Assert.AreEqual("Follow", data.type);
// Assert.AreEqual("https://4a120ca2680e.ngrok.io/users/manu", data.apObject);
// }
// [TestMethod]
// public void UndoDeserializationTest()
// {
// var json =
// "{\"@context\":\"https://www.w3.org/ns/activitystreams\",\"id\":\"https://mastodon.technology/users/testtest#follows/225982/undo\",\"type\":\"Undo\",\"actor\":\"https://mastodon.technology/users/testtest\",\"object\":{\"id\":\"https://mastodon.technology/c94567cf-1fda-42ba-82fc-a0f82f63ccbe\",\"type\":\"Follow\",\"actor\":\"https://mastodon.technology/users/testtest\",\"object\":\"https://4a120ca2680e.ngrok.io/users/manu\"}}";
// var data = JsonConvert.DeserializeObject<Activity>(json);
// Assert.AreEqual("https://mastodon.technology/users/testtest#follows/225982/undo", data.id);
// Assert.AreEqual("Undo", data.type);
// Assert.AreEqual("https://4a120ca2680e.ngrok.io/users/manu", data.apObject);
// }
//}
}
}
}

View file

@ -1,6 +1,6 @@
using BirdsiteLive.ActivityPub.Models;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Newtonsoft.Json;
using System.Text.Json;
namespace BirdsiteLive.ActivityPub.Tests
{
@ -28,9 +28,11 @@ namespace BirdsiteLive.ActivityPub.Tests
var data = ApDeserializer.ProcessActivity(json) as ActivityUndoFollow;
Assert.AreEqual("https://mastodon.technology/users/testtest#follows/225982/undo", data.id);
Assert.AreEqual("Undo", data.type);
Assert.AreEqual("https://www.w3.org/ns/activitystreams", data.context);
Assert.AreEqual("Follow", data.apObject.type);
Assert.AreEqual("https://mastodon.technology/users/testtest", data.apObject.actor);
Assert.AreEqual("https://4a120ca2680e.ngrok.io/users/manu", data.apObject.apObject);
Assert.AreEqual(null, data.apObject.context);
}
[TestMethod]
@ -62,14 +64,100 @@ namespace BirdsiteLive.ActivityPub.Tests
Assert.AreEqual("https://mastodon.technology/users/deleteduser", data.actor);
Assert.AreEqual("https://mastodon.technology/users/deleteduser", data.apObject);
}
// {"object":{"object":"https://bird.makeup/users/spectatorindex","id":"https://masto.ai/b89eb86e-c902-48bc-956f-94f081617f18","type":"Follow","actor":"https://masto.ai/users/singha"},"@context":"https://www.w3.org/ns/activitystreams","id":"https://bird.makeup/users/spectatorindex#accepts/follows/27363118-e61e-4710-a41c-75dd5d54912f","type":"Accept","actor":"https://bird.makeup/users/spectatorindex"}
// {"object":{"object":"https://bird.makeup/users/moltke","id":"https://universeodon.com/81cddd78-d7d6-4665-aa21-7bcfbea82b6b","type":"Follow","actor":"https://universeodon.com/users/amhrasmussen"},"@context":"https://www.w3.org/ns/activitystreams","id":"https://bird.makeup/users/moltke#accepts/follows/d28146be-e884-4e91-8385-19fa004f35b3","type":"Accept","actor":"https://bird.makeup/users/moltke"}
//[TestMethod]
//public void NoteDeserializationTest()
//{
// var json =
// "{\"@context\":[\"https://www.w3.org/ns/activitystreams\",{\"ostatus\":\"http://ostatus.org#\",\"atomUri\":\"ostatus:atomUri\",\"inReplyToAtomUri\":\"ostatus:inReplyToAtomUri\",\"conversation\":\"ostatus:conversation\",\"sensitive\":\"as:sensitive\",\"toot\":\"http://joinmastodon.org/ns#\",\"votersCount\":\"toot:votersCount\"}],\"id\":\"https://mastodon.technology/users/testtest/statuses/104424839893177182/activity\",\"type\":\"Create\",\"actor\":\"https://mastodon.technology/users/testtest\",\"published\":\"2020-06-29T02:10:04Z\",\"to\":[\"https://mastodon.technology/users/testtest/followers\"],\"cc\":[],\"object\":{\"id\":\"https://mastodon.technology/users/testtest/statuses/104424839893177182\",\"type\":\"Note\",\"summary\":null,\"inReplyTo\":null,\"published\":\"2020-06-29T02:10:04Z\",\"url\":\"https://mastodon.technology/@testtest/104424839893177182\",\"attributedTo\":\"https://mastodon.technology/users/testtest\",\"to\":[\"https://mastodon.technology/users/testtest/followers\"],\"cc\":[],\"sensitive\":false,\"atomUri\":\"https://mastodon.technology/users/testtest/statuses/104424839893177182\",\"inReplyToAtomUri\":null,\"conversation\":\"tag:mastodon.technology,2020-06-29:objectId=34900058:objectType=Conversation\",\"content\":\"<p>test</p>\",\"contentMap\":{\"en\":\"<p>test</p>\"},\"attachment\":[],\"tag\":[],\"replies\":{\"id\":\"https://mastodon.technology/users/testtest/statuses/104424839893177182/replies\",\"type\":\"Collection\",\"first\":{\"type\":\"CollectionPage\",\"next\":\"https://mastodon.technology/users/testtest/statuses/104424839893177182/replies?only_other_accounts=true&page=true\",\"partOf\":\"https://mastodon.technology/users/testtest/statuses/104424839893177182/replies\",\"items\":[]}}}}";
// var data = ApDeserializer.ProcessActivity(json) as ActivityAcceptFollow;
//}
[TestMethod]
public void ActorDeserializationTest()
{
var json = """
{
"@context": [
"https://www.w3.org/ns/activitystreams",
"https://w3id.org/security/v1",
{
"manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
"toot": "http://joinmastodon.org/ns#",
"featured": {
"@id": "toot:featured",
"@type": "@id"
},
"featuredTags": {
"@id": "toot:featuredTags",
"@type": "@id"
},
"alsoKnownAs": {
"@id": "as:alsoKnownAs",
"@type": "@id"
},
"movedTo": {
"@id": "as:movedTo",
"@type": "@id"
},
"schema": "http://schema.org#",
"PropertyValue": "schema:PropertyValue",
"value": "schema:value",
"discoverable": "toot:discoverable",
"Device": "toot:Device",
"Ed25519Signature": "toot:Ed25519Signature",
"Ed25519Key": "toot:Ed25519Key",
"Curve25519Key": "toot:Curve25519Key",
"EncryptedMessage": "toot:EncryptedMessage",
"publicKeyBase64": "toot:publicKeyBase64",
"deviceId": "toot:deviceId",
"claim": {
"@type": "@id",
"@id": "toot:claim"
},
"fingerprintKey": {
"@type": "@id",
"@id": "toot:fingerprintKey"
},
"identityKey": {
"@type": "@id",
"@id": "toot:identityKey"
},
"devices": {
"@type": "@id",
"@id": "toot:devices"
},
"messageFranking": "toot:messageFranking",
"messageType": "toot:messageType",
"cipherText": "toot:cipherText",
"suspended": "toot:suspended"
}
],
"id": "https://mastodon.online/users/devvincent",
"type": "Person",
"following": "https://mastodon.online/users/devvincent/following",
"followers": "https://mastodon.online/users/devvincent/followers",
"inbox": "https://mastodon.online/users/devvincent/inbox",
"outbox": "https://mastodon.online/users/devvincent/outbox",
"featured": "https://mastodon.online/users/devvincent/collections/featured",
"featuredTags": "https://mastodon.online/users/devvincent/collections/tags",
"preferredUsername": "devvincent",
"name": "",
"summary": "",
"url": "https://mastodon.online/@devvincent",
"manuallyApprovesFollowers": false,
"discoverable": false,
"published": "2022-05-08T00:00:00Z",
"devices": "https://mastodon.online/users/devvincent/collections/devices",
"publicKey": {
"id": "https://mastodon.online/users/devvincent#main-key",
"owner": "https://mastodon.online/users/devvincent",
"publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA7U07uS4zu5jeZSBVZ072\naXcTeVQc0baM8BBUvJkpX+mV2vh+V4yfqN44KzFxlkk8XcAoidt8HBAvpQ/5yCwZ\neGS2ySCxC+sqvErIbaYadWVHGJhZjLYPVa0n8wvkqRQ0aUJ8K17/wY+/YYfukgeC\nTGHGoyzDDZZxrR1Z8LTvImSEkYooTvvzaaFaTUnFwCKepxftKLdJAfp4sP4l1Zom\nUZGwaYimuJmN1bfhet/2v0S7M7/XPlmVRpfUluE2vYE0RtJt3BVDZfoWEGJPk9us\nN/JHu6UBUh6UM6ASFy5MlDLh36OxyO9sVx1WgQlNDmu2qcGUIkIgqTKppDCIP3Xk\nVQIDAQAB\n-----END PUBLIC KEY-----\n"
},
"tag": [],
"attachment": [],
"endpoints": {
"sharedInbox": "https://mastodon.online/inbox"
}
}
""";
var actor = JsonSerializer.Deserialize<Actor>(json);
}
}
}

View file

@ -4,6 +4,8 @@
<TargetFramework>net6</TargetFramework>
<IsPackable>false</IsPackable>
<LangVersion>11</LangVersion>
</PropertyGroup>
<ItemGroup>

View file

@ -0,0 +1,96 @@
using System.Net.Http;
using System.Threading.Tasks;
using BirdsiteLive.ActivityPub;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.Domain.Factories;
using Microsoft.Extensions.Logging;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Moq;
using System.Text.Json;
namespace BirdsiteLive.Domain.Tests
{
[TestClass]
public class ActivityServiceTests
{
private readonly InstanceSettings _settings;
#region Ctor
public ActivityServiceTests()
{
_settings = new InstanceSettings
{
Domain = "domain.name"
};
}
#endregion
[TestMethod]
public async Task ActivityTest()
{
var logger1 = new Mock<ILogger<ActivityPubService>>();
var httpFactory = new Mock<IHttpClientFactory>();
var keyFactory = new Mock<MagicKeyFactory>();
var cryptoService = new CryptoService(keyFactory.Object);
httpFactory.Setup(_ => _.CreateClient(string.Empty)).Returns(new HttpClient());
var service = new ActivityPubService(cryptoService, _settings, httpFactory.Object, logger1.Object);
var activity = new ActivityAcceptFollow()
{
id = "awef",
context = "https://www.w3.org/ns/activitystreams",
type = "Accept",
actor = "https://mastodon.technology/users/testtest",
apObject = new ActivityFollow()
{
context = "https://www.w3.org/ns/activitystreams",
id = "abc",
type = "Follow",
actor = "https://mastodon.technology/users/testtest2",
apObject = "https://mastodon.technology/users/testtest3",
}
};
var json =
"""{"object":{"object":"https://mastodon.technology/users/testtest3","@context":"https://www.w3.org/ns/activitystreams","id":"abc","type":"Follow","actor":"https://mastodon.technology/users/testtest2"},"@context":"https://www.w3.org/ns/activitystreams","id":"awef","type":"Accept","actor":"https://mastodon.technology/users/testtest"}""";
#region Validations
var req = service.BuildRequest(activity, "google.com", "tata", "awef");
Assert.AreEqual(await req.Content.ReadAsStringAsync(), json);
#endregion
}
[TestMethod]
public async Task AcceptFollow()
{
var logger1 = new Mock<ILogger<ActivityPubService>>();
var httpFactory = new Mock<IHttpClientFactory>();
var keyFactory = new Mock<MagicKeyFactory>();
var cryptoService = new CryptoService(keyFactory.Object);
httpFactory.Setup(_ => _.CreateClient(string.Empty)).Returns(new HttpClient());
var service = new ActivityPubService(cryptoService, _settings, httpFactory.Object, logger1.Object);
var json = "{ \"@context\":\"https://www.w3.org/ns/activitystreams\",\"id\":\"https://mastodon.technology/c94567cf-1fda-42ba-82fc-a0f82f63ccbe\",\"type\":\"Follow\",\"actor\":\"https://mastodon.technology/users/testtest\",\"object\":\"https://4a120ca2680e.ngrok.io/users/manu\"}";
var activity = ApDeserializer.ProcessActivity(json) as ActivityFollow;
var jsonres =
"{\"object\":{\"id\":\"https://mastodon.technology/c94567cf-1fda-42ba-82fc-a0f82f63ccbe\",\"type\":\"Follow\",\"actor\":\"https://mastodon.technology/users/testtest\",\"object\":\"https://4a120ca2680e.ngrok.io/users/manu\"},\"@context\":\"https://www.w3.org/ns/activitystreams\",\"id\":\"https://4a120ca2680e.ngrok.io/users/manu#accepts/follows/32e5fbda-9159-4ede-8249-9d008092d26f\",\"type\":\"Accept\",\"actor\":\"https://4a120ca2680e.ngrok.io/users/manu\"}";
var activityRes = ApDeserializer.ProcessActivity(jsonres) as ActivityAcceptFollow;
#region Validations
var req = service.BuildAcceptFollow(activity);
string s = JsonSerializer.Serialize(req);
Assert.AreEqual(req.actor, activityRes.actor);
Assert.AreEqual(req.context, activityRes.context);
#endregion
}
}
}

View file

@ -2,7 +2,7 @@
<PropertyGroup>
<TargetFramework>net6</TargetFramework>
<LangVersion>11</LangVersion>
<IsPackable>false</IsPackable>
</PropertyGroup>

View file

@ -14,6 +14,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
[TestClass]
public class RetrieveFollowersProcessorTests
{
[Ignore]
[TestMethod]
public async Task ProcessAsync_Test()
{

View file

@ -29,19 +29,32 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
new SyncTwitterUser(),
};
var maxUsers = 1000;
var instanceSettings = new InstanceSettings()
{
n_start = 1,
};
#endregion
#region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersWithFollowersAsync(
It.Is<int>(y => true),
It.Is<int>(y => true),
It.Is<int>(y => true),
It.Is<int>(y => true)))
.ReturnsAsync(users);
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
followersDalMock
.Setup(x => x.GetFollowersAsync(It.Is<int>(x => true)))
.ReturnsAsync(new Follower[] {});
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
processor.WaitFactor = 10;
var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
@ -66,12 +79,19 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
users.Add(new SyncTwitterUser());
var maxUsers = 1000;
var instanceSettings = new InstanceSettings()
{
n_start = 1,
};
#endregion
#region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.SetupSequence(x => x.GetAllTwitterUsersWithFollowersAsync(
It.Is<int>(y => true),
It.Is<int>(y => true),
It.Is<int>(y => true),
It.Is<int>(y => true)))
.ReturnsAsync(users.ToArray())
.ReturnsAsync(new SyncTwitterUser[0])
@ -79,10 +99,15 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
.ReturnsAsync(new SyncTwitterUser[0])
.ReturnsAsync(new SyncTwitterUser[0]);
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
followersDalMock
.Setup(x => x.GetFollowersAsync(It.Is<int>(x => true)))
.ReturnsAsync(new Follower[] {});
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
processor.WaitFactor = 2;
var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
@ -107,12 +132,19 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
users.Add(new SyncTwitterUser());
var maxUsers = 1000;
var instanceSettings = new InstanceSettings()
{
n_start = 1,
};
#endregion
#region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.SetupSequence(x => x.GetAllTwitterUsersWithFollowersAsync(
It.Is<int>(y => true),
It.Is<int>(y => true),
It.Is<int>(y => true),
It.Is<int>(y => true)))
.ReturnsAsync(users.ToArray())
.ReturnsAsync(new SyncTwitterUser[0])
@ -120,10 +152,15 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
.ReturnsAsync(new SyncTwitterUser[0])
.ReturnsAsync(new SyncTwitterUser[0]);
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
followersDalMock
.Setup(x => x.GetFollowersAsync(It.Is<int>(x => true)))
.ReturnsAsync(new Follower[] {});
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
processor.WaitFactor = 2;
var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
@ -144,6 +181,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var buffer = new BufferBlock<UserWithDataToSync[]>();
var maxUsers = 1000;
var instanceSettings = new InstanceSettings()
{
n_start = 1,
};
#endregion
#region Mocks
@ -151,13 +192,21 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersWithFollowersAsync(
It.Is<int>(y => true),
It.Is<int>(y => true),
It.Is<int>(y => true),
It.Is<int>(y => true)))
.ReturnsAsync(new SyncTwitterUser[0]);
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
followersDalMock
.Setup(x => x.GetFollowersAsync(It.Is<int>(x => true)))
.ReturnsAsync(new Follower[] {});
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
processor.WaitFactor = 1;
var t =processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
@ -176,19 +225,31 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var buffer = new BufferBlock<UserWithDataToSync[]>();
var maxUsers = 1000;
var instanceSettings = new InstanceSettings()
{
n_start = 1,
};
#endregion
#region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersWithFollowersAsync(
It.Is<int>(y => true),
It.Is<int>(y => true),
It.Is<int>(y => true),
It.Is<int>(y => true)))
.Returns(async () => await DelayFaultedTask<SyncTwitterUser[]>(new Exception()));
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
followersDalMock
.Setup(x => x.GetFollowersAsync(It.Is<int>(x => true)))
.ReturnsAsync(new Follower[] {});
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
processor.WaitFactor = 10;
var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
@ -210,15 +271,24 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
canTokenS.Cancel();
var maxUsers = 1000;
var instanceSettings = new InstanceSettings()
{
n_start = 1,
};
#endregion
#region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
followersDalMock
.Setup(x => x.GetFollowersAsync(It.Is<int>(x => true)))
.ReturnsAsync(new Follower[] {});
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
processor.WaitFactor = 1;
await processor.GetTwitterUsersAsync(buffer, canTokenS.Token);
}

View file

@ -88,7 +88,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
await processor.ProcessAsync(userWithTweets, CancellationToken.None);
await processor.ProcessAsync(new[] {userWithTweets}, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
@ -170,7 +170,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
await processor.ProcessAsync(userWithTweets, CancellationToken.None);
await processor.ProcessAsync(new[] {userWithTweets}, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
@ -261,7 +261,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
await processor.ProcessAsync(userWithTweets, CancellationToken.None);
await processor.ProcessAsync(new[] {userWithTweets}, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
@ -353,7 +353,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
await processor.ProcessAsync(userWithTweets, CancellationToken.None);
await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
@ -450,7 +450,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
await processor.ProcessAsync(userWithTweets, CancellationToken.None);
await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
@ -530,7 +530,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
await processor.ProcessAsync(userWithTweets, CancellationToken.None);
await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
@ -611,7 +611,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
await processor.ProcessAsync(userWithTweets, CancellationToken.None);
await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
@ -700,7 +700,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
await processor.ProcessAsync(userWithTweets, CancellationToken.None);
await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
@ -790,7 +790,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
await processor.ProcessAsync(userWithTweets, CancellationToken.None);
await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
@ -880,7 +880,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
await processor.ProcessAsync(userWithTweets, CancellationToken.None);
await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
@ -970,7 +970,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
await processor.ProcessAsync(userWithTweets, CancellationToken.None);
await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();
@ -1065,7 +1065,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
#endregion
var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
await processor.ProcessAsync(userWithTweets, CancellationToken.None);
await processor.ProcessAsync(new [] { userWithTweets }, CancellationToken.None);
#region Validations
sendTweetsToInboxTaskMock.VerifyAll();

View file

@ -37,10 +37,11 @@ namespace BirdsiteLive.ActivityPub.Tests
))
.ReturnsAsync(new SyncTwitterUser { TwitterUserId = default });
ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(httpFactory.Object, logger1.Object);
ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(httpFactory.Object, settings, logger1.Object);
ITwitterUserService user = new TwitterUserService(auth, stats.Object, logger2.Object);
ICachedTwitterUserService user2 = new CachedTwitterUserService(user, settings);
_tweetService = new TwitterTweetsService(auth, stats.Object, user2, twitterDal.Object, settings, logger3.Object);
}
[TestMethod]
@ -48,7 +49,7 @@ namespace BirdsiteLive.ActivityPub.Tests
{
var tweets = await _tweetService.GetTimelineAsync("kobebryant", 1218020971346444288);
Assert.AreEqual(tweets[0].MessageContent, "Continuing to move the game forward @KingJames. Much respect my brother 💪🏾 #33644");
Assert.AreEqual(tweets.Length, 8);
Assert.IsTrue(tweets.Length > 5);
}
[TestMethod]

View file

@ -1,4 +1,6 @@
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System;
using System.IO;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Microsoft.Extensions.Logging;
using System.Threading.Tasks;
using BirdsiteLive.Twitter;
@ -18,6 +20,7 @@ namespace BirdsiteLive.ActivityPub.Tests
[TestInitialize]
public async Task TestInit()
{
var logger1 = new Mock<ILogger<TwitterAuthenticationInitializer>>(MockBehavior.Strict);
var logger2 = new Mock<ILogger<TwitterUserService>>(MockBehavior.Strict);
var logger3 = new Mock<ILogger<TwitterTweetsService>>();
@ -29,10 +32,11 @@ namespace BirdsiteLive.ActivityPub.Tests
{
Domain = "domain.name"
};
ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(httpFactory.Object, logger1.Object);
ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(httpFactory.Object, settings, logger1.Object);
ITwitterUserService user = new TwitterUserService(auth, stats.Object, logger2.Object);
ICachedTwitterUserService user2 = new CachedTwitterUserService(user, settings);
_tweetService = new TwitterTweetsService(auth, stats.Object, user2, twitterDal.Object, settings, logger3.Object);
}
[TestMethod]
@ -74,6 +78,7 @@ namespace BirdsiteLive.ActivityPub.Tests
Assert.IsTrue(tweet.Media[0].Url.StartsWith("https://video.twimg.com/"));
}
[Ignore]
[TestMethod]
public async Task GifAndQT()
{

View file

@ -6,6 +6,7 @@ using BirdsiteLive.Twitter.Tools;
using BirdsiteLive.Statistics.Domain;
using Moq;
using System.Net.Http;
using BirdsiteLive.Common.Settings;
namespace BirdsiteLive.ActivityPub.Tests
{
@ -21,8 +22,12 @@ namespace BirdsiteLive.ActivityPub.Tests
var logger3 = new Mock<ILogger<TwitterUserService>>();
var stats = new Mock<ITwitterStatisticsHandler>();
var httpFactory = new Mock<IHttpClientFactory>();
var settings = new InstanceSettings
{
Domain = "domain.name"
};
httpFactory.Setup(_ => _.CreateClient(string.Empty)).Returns(new HttpClient());
ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(httpFactory.Object, logger1.Object);
ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(httpFactory.Object, settings, logger1.Object);
_tweetService = new TwitterUserService(auth, stats.Object, logger3.Object);
}