Merge branch 'master' of bird.makeup into makeup

.builds/arch.yml

@@ -13,8 +13,12 @@ tasks:
       sudo docker run -d -p 5432:5432 -e POSTGRES_PASSWORD=birdsitelive -e POSTGRES_USER=birdsitelive -e POSTGRES_DB=birdsitelive postgres:15
       cd bird.makeup/src
       dotnet test
-
-  - publish: |
+  - publish-arm: |
+      cd bird.makeup/src/BirdsiteLive
+      dotnet publish --os linux --arch arm64 /t:PublishContainer -c Release
+      docker tag cloutier/bird.makeup:1.0 cloutier/bird.makeup:latest-arm
+      docker push cloutier/bird.makeup:latest-arm
+  - publish-x64: |
       cd bird.makeup/src/BirdsiteLive
       dotnet publish --os linux --arch x64 /t:PublishContainer -c Release
       docker tag cloutier/bird.makeup:1.0 cloutier/bird.makeup:latest

src/BSLManager/Tools/SettingsManager.cs

@@ -1,8 +1,8 @@
 using System;
 using System.IO;
+using System.Text.Json;
 using System.Runtime.CompilerServices;
 using BirdsiteLive.Common.Settings;
-using Newtonsoft.Json;
 
 namespace BSLManager.Tools
 {
@@ -94,7 +94,7 @@ namespace BSLManager.Tools
                 if (!File.Exists(LocalFileName)) return null;
 
                 var jsonContent = File.ReadAllText(LocalFileName);
-                var content = JsonConvert.DeserializeObject<LocalSettingsData>(jsonContent);
+                var content = JsonSerializer.Deserialize<LocalSettingsData>(jsonContent);
                 return content;
             }
             catch (Exception)
@@ -105,7 +105,7 @@ namespace BSLManager.Tools
 
         private void SaveLocalSettings(LocalSettingsData data)
         {
-            var jsonContent = JsonConvert.SerializeObject(data);
+            var jsonContent = JsonSerializer.Serialize(data);
             File.WriteAllText(LocalFileName, jsonContent);
         }
     }

src/BirdsiteLive.ActivityPub/ApDeserializer.cs

@@ -1,6 +1,7 @@
 using System;
 using BirdsiteLive.ActivityPub.Models;
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
+using System.Text.Json;
 
 namespace BirdsiteLive.ActivityPub
 {
@@ -10,22 +11,21 @@ namespace BirdsiteLive.ActivityPub
         {
             try
             {
-                var activity = JsonConvert.DeserializeObject<Activity>(json);
+                var activity = JsonSerializer.Deserialize<Activity>(json);
                 switch (activity.type)
                 {
                     case "Follow":
-                        return JsonConvert.DeserializeObject<ActivityFollow>(json);
+                        return JsonSerializer.Deserialize<ActivityFollow>(json);
                     case "Undo":
-                        var a = JsonConvert.DeserializeObject<ActivityUndo>(json);
+                        var a = JsonSerializer.Deserialize<ActivityUndo>(json);
                         if(a.apObject.type == "Follow")
-                            return JsonConvert.DeserializeObject<ActivityUndoFollow>(json);
+                            return JsonSerializer.Deserialize<ActivityUndoFollow>(json);
                         break;
                     case "Delete":
-                        return JsonConvert.DeserializeObject<ActivityDelete>(json);
+                        return JsonSerializer.Deserialize<ActivityDelete>(json);
                     case "Accept":
-                        var accept = JsonConvert.DeserializeObject<ActivityAccept>(json);
-                        //var acceptType = JsonConvert.DeserializeObject<Activity>(accept.apObject);
-                        switch ((accept.apObject as dynamic).type.ToString())
+                        var accept = JsonSerializer.Deserialize<ActivityAccept>(json);
+                        switch (accept.apObject.type)
                         {
                             case "Follow":
                                 var acceptFollow = new ActivityAcceptFollow()
@@ -36,11 +36,12 @@ namespace BirdsiteLive.ActivityPub
                                     context = accept.context,
                                     apObject = new ActivityFollow()
                                     {
-                                        id = (accept.apObject as dynamic).id?.ToString(),
-                                        type = (accept.apObject as dynamic).type?.ToString(),
-                                        actor = (accept.apObject as dynamic).actor?.ToString(),
-                                        context = (accept.apObject as dynamic).context?.ToString(),
-                                        apObject = (accept.apObject as dynamic).@object?.ToString()
+
+                                        id = accept.apObject.id,
+                                        type = accept.apObject.type,
+                                        actor = accept.apObject.actor,
+                                        context = accept.apObject.context?.ToString(),
+                                        apObject = accept.apObject.apObject,
                                     }
                                 };
                                 return acceptFollow;
@@ -56,10 +57,5 @@ namespace BirdsiteLive.ActivityPub
             return null;
         }
 
-        private class Ac : Activity
-        {
-            [JsonProperty("object")]
-            public Activity apObject { get; set; }
-        }
     }
 }

src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj

@@ -6,7 +6,6 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
   </ItemGroup>
 
 </Project>

src/BirdsiteLive.ActivityPub/Converters/ContextArrayConverter.cs

@@ -1,39 +0,0 @@
-using System;
-using System.Collections.Generic;
-using Newtonsoft.Json;
-
-namespace BirdsiteLive.ActivityPub.Converters
-{
-    public class ContextArrayConverter : JsonConverter
-    {
-        public override bool CanWrite { get { return false; } }
-        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
-        {
-            throw new NotImplementedException();
-        }
-
-        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
-        {
-            var result = new List<string>();
-
-            var list = serializer.Deserialize<List<object>>(reader);
-            foreach (var l in list)
-            {
-                if (l is string s)
-                    result.Add(s);
-                else
-                {
-                    var str = JsonConvert.SerializeObject(l);
-                    result.Add(str);
-                }
-            }
-
-            return result.ToArray();
-        }
-
-        public override bool CanConvert(Type objectType)
-        {
-            throw new NotImplementedException();
-        }
-    }
-}

src/BirdsiteLive.ActivityPub/Models/Activity.cs

@@ -1,36 +1,17 @@
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.Text.Json.Serialization;
-using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class Activity
     {
-		[Newtonsoft.Json.JsonIgnore]
-		public static readonly object[] DefaultContext = new object[] {
-            "https://www.w3.org/ns/activitystreams",
-            "https://w3id.org/security/v1",
-            new Dictionary<string, string>
-            {
-                { "Emoji", "toot:Emoji" },
-                { "Hashtag", "as:Hashtag" },
-                { "PropertyValue", "schema:PropertyValue" },
-                { "value", "schema:value" },
-                { "sensitive", "as:sensitive" },
-                { "quoteUrl", "as:quoteUrl" },
-
-                { "schema", "http://schema.org#" },
-                { "toot", "https://joinmastodon.org/ns#" }
-            }
-        };
-
-        [JsonProperty("@context")]
-        public object context { get; set; } = DefaultContext;
+        [JsonPropertyName("@context")]
+        public string context { get; set; }
         public string id { get; set; }
         public string type { get; set; }
         public string actor { get; set; }
 
-        //[JsonProperty("object")]
+        //[JsonPropertyName("object")]
         //public string apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityAccept.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityAccept : Activity
     {
-        [JsonProperty("object")]
-        public object apObject { get; set; }
+        [JsonPropertyName("object")]
+        public NestedActivity apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityAcceptFollow.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityAcceptFollow : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public ActivityFollow apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityAcceptUndoFollow.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityAcceptUndoFollow : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public ActivityUndoFollow apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityCreateNote.cs

@@ -1,6 +1,5 @@
-using System;
-using BirdsiteLive.ActivityPub.Models;
-using Newtonsoft.Json;
+using BirdsiteLive.ActivityPub.Models;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
@@ -10,7 +9,7 @@ namespace BirdsiteLive.ActivityPub
         public string[] to { get; set; }
         public string[] cc { get; set; }
 
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public Note apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub.Models
 {
     public class ActivityDelete : Activity
     {
-        [JsonProperty("object")]
-        public object apObject { get; set; }
+        [JsonPropertyName("object")]
+        public string apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityFollow.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityFollow : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public string apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityRejectFollow.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityRejectFollow : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public ActivityFollow apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityUndo.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityUndo : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public Activity apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityUndoFollow.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityUndoFollow : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public ActivityFollow apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/Actor.cs

@@ -1,15 +1,13 @@
 using System.Net;
 using BirdsiteLive.ActivityPub.Converters;
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class Actor
     {
-        //[JsonPropertyName("@context")]
-        [JsonProperty("@context")]
-        [JsonConverter(typeof(ContextArrayConverter))]
-        public object[] context { get; set; } = Activity.DefaultContext;
+        [JsonPropertyName("@context")]
+        public object[] context { get; set; } = new string[] { "https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1" };
         public string id { get; set; }
         public string type { get; set; }
         public string followers { get; set; }

src/BirdsiteLive.ActivityPub/Models/Followers.cs

@@ -1,13 +1,12 @@
 using BirdsiteLive.ActivityPub.Converters;
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub.Models
 {
     public class Followers
     {
-        [JsonProperty("@context")]
-        [JsonConverter(typeof(ContextArrayConverter))]
-        public object[] context { get; set; } = Activity.DefaultContext;
+        [JsonPropertyName("@context")]
+        public string context { get; set; } = "https://www.w3.org/ns/activitystreams";
 
         public string id { get; set; }
         public string type { get; set; } = "OrderedCollection";

src/BirdsiteLive.ActivityPub/Models/NestedActivity.cs

@@ -0,0 +1,17 @@
+using System.Text.Json.Serialization;
+
+namespace BirdsiteLive.ActivityPub
+{
+    public class NestedActivity
+    {
+        [JsonPropertyName("@context")]
+        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+        public object context { get; set; }
+        public string id { get; set; }
+        public string type { get; set; }
+        public string actor { get; set; }
+
+        [JsonPropertyName("object")]
+        public string apObject { get; set; }
+    }
+}

src/BirdsiteLive.ActivityPub/Models/Note.cs

@@ -1,13 +1,11 @@
-using BirdsiteLive.ActivityPub.Converters;
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub.Models
 {
     public class Note
     {
-        [JsonProperty("@context")]
-        [JsonConverter(typeof(ContextArrayConverter))]
-        public object[] context { get; set; } = Activity.DefaultContext;
+        [JsonPropertyName("@context")]
+        public string[] context { get; set; } = new[] { "https://www.w3.org/ns/activitystreams" };
 
         public string id { get; set; }
         public string announceId { get; set; }

src/BirdsiteLive.Common/Settings/InstanceSettings.cs

@@ -14,8 +14,13 @@
         public int FailingTwitterUserCleanUpThreshold { get; set; }
         public int FailingFollowerCleanUpThreshold { get; set; } = -1;
 
-        public int UserCacheCapacity { get; set; } = 20_000;
+        public int UserCacheCapacity { get; set; } = 40_000;
         public int TweetCacheCapacity { get; set; } = 20_000;
+        // "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
+        public string TwitterBearerToken { get; set; } = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA";
+        public int m { get; set; } = 1;
+        public int n_start { get; set; } = 0;
+        public int n_end { get; set; } = 1;
         public int ParallelTwitterRequests { get; set; } = 10;
         public int ParallelFediversePosts { get; set; } = 10;
     }

src/BirdsiteLive.Domain/ActivityPubService.cs

@@ -5,13 +5,14 @@ using System.Net.Http;
 using System.Runtime.InteropServices;
 using System.Security.Cryptography;
 using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
 using System.Threading.Tasks;
 using BirdsiteLive.ActivityPub;
 using BirdsiteLive.ActivityPub.Converters;
 using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.Common.Settings;
 using Microsoft.Extensions.Logging;
-using Newtonsoft.Json;
 
 namespace BirdsiteLive.Domain
 {
@@ -21,6 +22,8 @@ namespace BirdsiteLive.Domain
         Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null);
         Task PostNewActivity(ActivityCreateNote note, string username, string noteId, string targetHost,
             string targetInbox);
+
+        ActivityAcceptFollow BuildAcceptFollow(ActivityFollow activity);
     }
 
     public class ActivityPubService : IActivityPubService
@@ -53,7 +56,7 @@ namespace BirdsiteLive.Domain
 
             var content = await result.Content.ReadAsStringAsync();
 
-            var actor = JsonConvert.DeserializeObject<Actor>(content);
+            var actor = JsonSerializer.Deserialize<Actor>(content);
             if (string.IsNullOrWhiteSpace(actor.url)) actor.url = objectId;
             return actor;
         }
@@ -73,13 +76,32 @@ namespace BirdsiteLive.Domain
             }
         }
 
-        public async Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null)
+        public ActivityAcceptFollow BuildAcceptFollow(ActivityFollow activity)
+        {
+            var acceptFollow = new ActivityAcceptFollow()
+            {
+                context = "https://www.w3.org/ns/activitystreams",
+                id = $"{activity.apObject}#accepts/follows/{Guid.NewGuid()}",
+                type = "Accept",
+                actor = activity.apObject,
+                apObject = new ActivityFollow()
+                {
+                    id = activity.id,
+                    type = activity.type,
+                    actor = activity.actor,
+                    apObject = activity.apObject
+                }
+            };
+            return acceptFollow;
+        }
+        public HttpRequestMessage BuildRequest<T>(T data, string targetHost, string actorUrl,
+            string inbox = null)
         {
             var usedInbox = $"/inbox";
             if (!string.IsNullOrWhiteSpace(inbox))
                 usedInbox = inbox;
 
-            var json = JsonConvert.SerializeObject(data, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore });
+            var json = JsonSerializer.Serialize(data);
 
             var date = DateTime.UtcNow.ToUniversalTime();
             var httpDate = date.ToString("r");
@@ -88,22 +110,30 @@ namespace BirdsiteLive.Domain
 
             var signature = _cryptoService.SignAndGetSignatureHeader(date, actorUrl, targetHost, digest, usedInbox);
 
-            var client = _httpClientFactory.CreateClient("BirdsiteLIVE");
-            client.Timeout = TimeSpan.FromSeconds(2);
             var httpRequestMessage = new HttpRequestMessage
             {
                 Method = HttpMethod.Post,
                 RequestUri = new Uri($"https://{targetHost}{usedInbox}"),
                 Headers =
                 {
-                    {"Host", targetHost},
-                    {"Date", httpDate},
-                    {"Signature", signature},
-                    {"Digest", $"SHA-256={digest}"}
+                    { "Host", targetHost },
+                    { "Date", httpDate },
+                    { "Signature", signature },
+                    { "Digest", $"SHA-256={digest}" }
                 },
                 Content = new StringContent(json, Encoding.UTF8, "application/ld+json")
             };
 
+            return httpRequestMessage;
+        }
+
+        public async Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null)
+        {
+            var httpRequestMessage = BuildRequest(data, targetHost, actorUrl, inbox);
+
+            var client = _httpClientFactory.CreateClient("BirdsiteLIVE");
+            client.Timeout = TimeSpan.FromSeconds(2);
+
             var response = await client.SendAsync(httpRequestMessage);
             response.EnsureSuccessStatusCode();
 

src/BirdsiteLive.Domain/UserService.cs

@@ -183,30 +183,11 @@ namespace BirdsiteLive.Domain
         
         private async Task<bool> SendAcceptFollowAsync(ActivityFollow activity, string followerHost)
         {
-            var acceptFollow = new ActivityAcceptFollow()
-            {
-                context = "https://www.w3.org/ns/activitystreams",
-                id = $"{activity.apObject}#accepts/follows/{Guid.NewGuid()}",
-                type = "Accept",
-                actor = activity.apObject,
-                apObject = new ActivityFollow()
-                {
-                    id = activity.id,
-                    type = activity.type,
-                    actor = activity.actor,
-                    apObject = activity.apObject
-                }
-            };
-            try
-            {
-                var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject);
-                return result == HttpStatusCode.Accepted ||
-                       result == HttpStatusCode.OK; 
-            }
-            catch (Exception e)
-            {
-                return false;
-            }
+            var acceptFollow = _activityPubService.BuildAcceptFollow(activity);
+            var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject);
+            return result == HttpStatusCode.Accepted ||
+                   result == HttpStatusCode.OK; //TODO: revamp this for better error handling
+
         }
 
         public async Task<bool> SendRejectFollowAsync(ActivityFollow activity, string followerHost)
@@ -264,10 +245,11 @@ namespace BirdsiteLive.Domain
                 actor = activity.apObject.apObject,
                 apObject = new ActivityUndoFollow()
                 {
-                    id = activity.id,
-                    type = activity.type,
-                    actor = activity.actor,
-                    apObject = activity.apObject
+                    id = (activity.apObject as dynamic).id?.ToString(),
+                    type = (activity.apObject as dynamic).type?.ToString(),
+                    actor = (activity.apObject as dynamic).actor?.ToString(),
+                    context = (activity.apObject as dynamic).context?.ToString(),
+                    apObject = (activity.apObject as dynamic).@object?.ToString()
                 }
             };
             var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject.apObject);

src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs

@@ -6,6 +6,6 @@ namespace BirdsiteLive.Pipeline.Contracts
 {
     public interface ISendTweetsToFollowersProcessor
     {
-        Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct);
+        Task ProcessAsync(UserWithDataToSync[] usersWithTweetsToSync, CancellationToken ct);
     }
 }

src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs

@@ -21,18 +21,18 @@ namespace BirdsiteLive.Pipeline.Processors
 
         public async Task<IEnumerable<UserWithDataToSync>> ProcessAsync(UserWithDataToSync[] userWithTweetsToSyncs, CancellationToken ct)
         {
-            List<Task> todo = new List<Task>();
-            foreach (var user in userWithTweetsToSyncs)
-            {
-                var t = Task.Run( 
-                async() => {
-                    var followers = await _followersDal.GetFollowersAsync(user.User.Id);
-                    user.Followers = followers;
-                });
-                todo.Add(t);
-            }
-            
-            await Task.WhenAll(todo);
+            //List<Task> todo = new List<Task>();
+            //foreach (var user in userWithTweetsToSyncs)
+            //{
+            //    var t = Task.Run( 
+            //    async() => {
+            //        var followers = await _followersDal.GetFollowersAsync(user.User.Id);
+            //        user.Followers = followers;
+            //    });
+            //    todo.Add(t);
+            //}
+            //
+            //await Task.WhenAll(todo);
 
             return userWithTweetsToSyncs;
         }

src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -15,15 +16,19 @@ namespace BirdsiteLive.Pipeline.Processors
     public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor
     {
         private readonly ITwitterUserDal _twitterUserDal;
+        private readonly IFollowersDal _followersDal;
+        private readonly InstanceSettings _instanceSettings;
         private readonly ILogger<RetrieveTwitterUsersProcessor> _logger;
         private static Random rng = new Random();
         
         public int WaitFactor = 1000 * 60; //1 min
 
         #region Ctor
-        public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, ILogger<RetrieveTwitterUsersProcessor> logger)
+        public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, InstanceSettings instanceSettings, ILogger<RetrieveTwitterUsersProcessor> logger)
         {
             _twitterUserDal = twitterUserDal;
+            _followersDal = followersDal;
+            _instanceSettings = instanceSettings;
             _logger = logger;
         }
         #endregion
@@ -34,27 +39,37 @@ namespace BirdsiteLive.Pipeline.Processors
             {
                 ct.ThrowIfCancellationRequested();
 
-                try
+                if (_instanceSettings.ParallelTwitterRequests == 0)
                 {
-                    var users = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000);
+                    while (true)
+                        await Task.Delay(10000);
+                }
+                
+                var usersDal = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000, _instanceSettings.n_start, _instanceSettings.n_end, _instanceSettings.m);
 
-                    var userCount = users.Any() ? Math.Min(users.Length, 200) : 1;
-                    var splitUsers = users.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList();
+                var userCount = usersDal.Any() ? Math.Min(usersDal.Length, 200) : 1;
+                var splitUsers = usersDal.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList();
 
-                    foreach (var u in splitUsers)
+                foreach (var users in splitUsers)
+                {
+                    ct.ThrowIfCancellationRequested();
+                    List<UserWithDataToSync> toSync = new List<UserWithDataToSync>();
+                    foreach (var u in users)
                     {
-                        ct.ThrowIfCancellationRequested();
-                        UserWithDataToSync[] toSync = u.Select(x => new UserWithDataToSync { User = x }).ToArray();
-
-                        await twitterUsersBufferBlock.SendAsync(toSync, ct);
+                        var followers = await _followersDal.GetFollowersAsync(u.Id);
+                        toSync.Add( new UserWithDataToSync()
+                        {
+                            User = u,
+                            Followers = followers
+                        });
+                        
                     }
 
-                    await Task.Delay(10, ct); // this is somehow necessary
-                }
-                catch (Exception e)
-                {
-                    _logger.LogError(e, "Failing retrieving Twitter Users.");
+                    await twitterUsersBufferBlock.SendAsync(toSync.ToArray(), ct);
+
                 }
+                
+                await Task.Delay(10, ct); // this is somehow necessary
             }
         }
     }

src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs

@@ -41,34 +41,40 @@ namespace BirdsiteLive.Pipeline.Processors
         }
         #endregion
 
-        public async Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
+        public async Task ProcessAsync(UserWithDataToSync[] usersWithTweetsToSync, CancellationToken ct)
         {
-            var user = userWithTweetsToSync.User;
-
-            _todo = _todo.Where(x => !x.IsCompleted).ToList();
-            
-            var t = Task.Run( async () => 
+            foreach (var userWithTweetsToSync in usersWithTweetsToSync)
             {
-                // Process Shared Inbox
-                var followersWtSharedInbox = userWithTweetsToSync.Followers
-                    .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
-                    .ToList();
-                await ProcessFollowersWithSharedInboxAsync(userWithTweetsToSync.Tweets, followersWtSharedInbox, user);
+                var user = userWithTweetsToSync.User;
 
-                // Process Inbox
-                var followerWtInbox = userWithTweetsToSync.Followers
-                    .Where(x => string.IsNullOrWhiteSpace(x.SharedInboxRoute))
-                    .ToList();
-                await ProcessFollowersWithInboxAsync(userWithTweetsToSync.Tweets, followerWtInbox, user);
-            });
-            _todo.Add(t);
+                _todo = _todo.Where(x => !x.IsCompleted).ToList();
+                
+                var t = Task.Run( async () => 
+                {
+                    // Process Shared Inbox
+                    var followersWtSharedInbox = userWithTweetsToSync.Followers
+                        .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
+                        .ToList();
+                    await ProcessFollowersWithSharedInboxAsync(userWithTweetsToSync.Tweets, followersWtSharedInbox, user);
 
-            if (_todo.Count >= _instanceSettings.ParallelFediversePosts)
-            {
-                await Task.WhenAny(_todo);
+                    // Process Inbox
+                    var followerWtInbox = userWithTweetsToSync.Followers
+                        .Where(x => string.IsNullOrWhiteSpace(x.SharedInboxRoute))
+                        .ToList();
+                    await ProcessFollowersWithInboxAsync(userWithTweetsToSync.Tweets, followerWtInbox, user);
+                    
+                    _logger.LogInformation("Done sending " + userWithTweetsToSync.Tweets.Length + " tweets for "
+                        + userWithTweetsToSync.Followers.Length + "followers for user " + userWithTweetsToSync.User.Acct);
+                }, ct);
+                _todo.Add(t);
+
+                if (_todo.Count >= _instanceSettings.ParallelFediversePosts)
+                {
+                    await Task.WhenAny(_todo);
+                }
+                
+                
             }
-            
-            _logger.LogInformation("Done sending " + userWithTweetsToSync.Followers.Length + "tweets for user " + userWithTweetsToSync.User.Acct);
 
         }
 

src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs

@@ -43,7 +43,6 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
             var tweetsToSend = tweets
                 .OrderBy(x => x.Id)
                 .ToList();
-            _logger.LogInformation("After filtering, there were " + tweetsToSend.Count() + " tweets left to send");
 
             foreach (var tweet in tweetsToSend)
             {

src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs

@@ -44,17 +44,15 @@ namespace BirdsiteLive.Pipeline
             var twitterUserToRefreshBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions
                 { BoundedCapacity = 1, CancellationToken = ct });
             var retrieveTweetsBlock = new TransformBlock<UserWithDataToSync[], UserWithDataToSync[]>(async x =>  await _retrieveTweetsProcessor.ProcessAsync(x, ct), standardBlockOptions );
-            var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
-            var retrieveFollowersBlock = new TransformManyBlock<UserWithDataToSync[], UserWithDataToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { BoundedCapacity = 1 } );
-            var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 500, CancellationToken = ct });
-            var sendTweetsToFollowersBlock = new ActionBlock<UserWithDataToSync>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), standardBlockOptions);
+            var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 2, CancellationToken = ct });
+           // var retrieveFollowersBlock = new TransformManyBlock<UserWithDataToSync[], UserWithDataToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { BoundedCapacity = 1 } );
+           // var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 500, CancellationToken = ct });
+            var sendTweetsToFollowersBlock = new ActionBlock<UserWithDataToSync[]>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), standardBlockOptions);
 
             // Link pipeline
             twitterUserToRefreshBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions { PropagateCompletion = true });
             retrieveTweetsBlock.LinkTo(retrieveTweetsBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
-            retrieveTweetsBufferBlock.LinkTo(retrieveFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
-            retrieveFollowersBlock.LinkTo(retrieveFollowersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
-            retrieveFollowersBufferBlock.LinkTo(sendTweetsToFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
+            retrieveTweetsBufferBlock.LinkTo(sendTweetsToFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
 
             // Launch twitter user retriever after a little delay
             // to give time for the Tweet cache to fill

src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj

@@ -7,6 +7,7 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Http" Version="5.0.0" />
+    <PackageReference Include="System.Threading.RateLimiting" Version="7.0.0" />
   </ItemGroup>
 
   <ItemGroup>

src/BirdsiteLive.Twitter/CachedTwitterService.cs

@@ -11,6 +11,7 @@ namespace BirdsiteLive.Twitter
     {
         void PurgeUser(string username);
         void AddUser(TwitterUser user);
+        bool UserIsCached(string username);
     }
 
     public class CachedTwitterUserService : ICachedTwitterUserService
@@ -39,6 +40,10 @@ namespace BirdsiteLive.Twitter
         }
         #endregion
 
+        public bool UserIsCached(string username)
+        {
+            return _userCache.TryGetValue(username, out _);
+        }
         public async Task<TwitterUser> GetUserAsync(string username)
         {
             if (!_userCache.TryGetValue(username, out Task<TwitterUser> user))

src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs

@@ -1,20 +1,24 @@
 using System;
 using System.Threading;
 using System.Collections.Generic;
+using System.Collections.Concurrent;
 using System.Linq;
 using System.Threading.Tasks;
 using BirdsiteLive.Common.Settings;
 using Microsoft.Extensions.Logging;
 using System.Net.Http;
 using System.Net;
+using System.Net.Http.Headers;
+using System.Text;
 using System.Text.Json;
+using System.Threading.RateLimiting;
 
 namespace BirdsiteLive.Twitter.Tools
 {
     public interface ITwitterAuthenticationInitializer
     {
         Task<HttpClient> MakeHttpClient();
-        HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint);
+        HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint, bool addToken);
         Task RefreshClient(HttpRequestMessage client);
     }
 
@@ -22,79 +26,82 @@ namespace BirdsiteLive.Twitter.Tools
     {
         private readonly ILogger<TwitterAuthenticationInitializer> _logger;
         private static bool _initialized;
-        private static System.Timers.Timer aTimer;
         private readonly IHttpClientFactory _httpClientFactory;
-        private List<HttpClient> _twitterClients = new List<HttpClient>();
-        private List<String> _tokens = new List<string>();
+        private ConcurrentDictionary<String, String> _token2 = new ConcurrentDictionary<string, string>();
         static Random rnd = new Random();
-        private const int _targetClients = 20;
+        private RateLimiter _rateLimiter;
+        private const int _targetClients = 3;
+        private InstanceSettings _instanceSettings;
+        private readonly (string, string)[] _apiKeys = new[]
+        {
+            ("IQKbtAYlXLripLGPWd0HUA", "GgDYlkSvaPxGxC4X8liwpUoqKwwr3lCADbz8A7ADU"), // iPhone
+            ("3nVuSoBZnx6U4vzUxf5w", "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"), // Android
+            ("CjulERsDeqhhjSme66ECg", "IQWdVyqFxghAtURHGeGiWAsmCAGmdW3WmbEx6Hck"), // iPad
+            ("3rJOl1ODzm9yZy63FACdg", "5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8"), // Mac
+        };
         public String BearerToken { 
-            get { return "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"; }
+            get
+            {
+                return _instanceSettings.TwitterBearerToken;
+            }
         }
 
         #region Ctor
-        public TwitterAuthenticationInitializer(IHttpClientFactory httpClientFactory, ILogger<TwitterAuthenticationInitializer> logger)
+        public TwitterAuthenticationInitializer(IHttpClientFactory httpClientFactory, InstanceSettings settings, ILogger<TwitterAuthenticationInitializer> logger)
         {
             _logger = logger;
+            _instanceSettings = settings;
             _httpClientFactory = httpClientFactory;
-
-            aTimer = new System.Timers.Timer();
-            aTimer.Interval = 20 * 1000; 
-            aTimer.Elapsed += async (sender, e) => await RefreshCred();
-            
-            aTimer.Start();
         }
         #endregion
 
+        private async Task<string> GenerateBearerToken()
+        {
+            var httpClient = _httpClientFactory.CreateClient();
+            using (var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.twitter.com/oauth2/token?grant_type=client_credentials"))
+            {
+                int r = rnd.Next(_apiKeys.Length);
+                var (login, password) = _apiKeys[r];
+                var authValue = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{login}:{password}")));
+                request.Headers.Authorization = authValue;
+
+                var httpResponse = await httpClient.SendAsync(request);
+
+                var c = await httpResponse.Content.ReadAsStringAsync();
+                httpResponse.EnsureSuccessStatusCode();
+                var doc = JsonDocument.Parse(c);
+                var token = doc.RootElement.GetProperty("access_token").GetString();
+                return token;
+            }
+            
+        }
+
 
         public async Task RefreshClient(HttpRequestMessage req)
         {
             string token = req.Headers.GetValues("x-guest-token").First();
 
-            var i = _tokens.IndexOf(token);
-
-            // this is prabably not thread save but yolo
-            try
-            {
-                _twitterClients.RemoveAt(i);
-                _tokens.RemoveAt(i);
-            }
-            catch (IndexOutOfRangeException _)
-            {
-                _logger.LogError("Error refreshing twitter token");
-            }
+            _token2.TryRemove(token, out _);
 
             await RefreshCred();
+            await Task.Delay(1000);
+            await RefreshCred();
         }
 
         private async Task RefreshCred()
         {
             (string bearer, string guest) = await GetCred();
-
-            HttpClient client = _httpClientFactory.CreateClient();
-            //HttpClient client = new HttpClient();
-            client.DefaultRequestHeaders.TryAddWithoutValidation("Authorization", $"Bearer " + bearer); 
-            client.DefaultRequestHeaders.TryAddWithoutValidation("x-guest-token", guest);
-            client.DefaultRequestHeaders.TryAddWithoutValidation("Referer", "https://twitter.com/");
-            client.DefaultRequestHeaders.TryAddWithoutValidation("x-twitter-active-user", "yes");
-
-            _twitterClients.Add(client);
-            _tokens.Add(guest);
-
-            if (_twitterClients.Count > _targetClients)
-            {
-                _twitterClients.RemoveAt(0);
-                _tokens.RemoveAt(0);
-            }
+            _token2.TryAdd(guest, bearer);
         }
 
         private async Task<(string, string)> GetCred()
         {
             string token;
             var httpClient = _httpClientFactory.CreateClient();
+            string bearer = await GenerateBearerToken();
             using (var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.twitter.com/1.1/guest/activate.json"))
             {
-                request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + BearerToken); 
+                request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer); 
 
                 var httpResponse = await httpClient.SendAsync(request);
 
@@ -104,40 +111,26 @@ namespace BirdsiteLive.Twitter.Tools
                 token = doc.RootElement.GetProperty("guest_token").GetString();
             }
 
-            return (BearerToken, token);
+            return (bearer, token);
 
         }
-        private async Task InitTwitterCredentials()
-        {
-            for (;;)
-            {
-                try
-                {
-                    await RefreshCred();
-                    _initialized = true;
-                    return;
-                }
-                catch (Exception e)
-                {
-                    _logger.LogError(e, "Twitter Authentication Failed");
-                    await Task.Delay(3600*1000);
-                }
-            }
-        }
 
         public async Task<HttpClient> MakeHttpClient()
         {
-            if (_twitterClients.Count < _targetClients)
+            if (_token2.Count < _targetClients)
                 await RefreshCred();
-            int r = rnd.Next(_twitterClients.Count);
-            return _twitterClients[r];
+            return _httpClientFactory.CreateClient();
         }
-        public HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint)
+        public HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint, bool addToken)
         {
             var request = new HttpRequestMessage(m, endpoint);
-            int r = rnd.Next(_twitterClients.Count);
-            request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + BearerToken); 
-            request.Headers.TryAddWithoutValidation("x-guest-token", _tokens[r]);
+            //(string bearer, string token) = _tokens[r];
+            (string token, string bearer) = _token2.MaxBy(x => rnd.Next());
+            request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer); 
+            request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");
+            request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes");
+            if (addToken)
+                request.Headers.TryAddWithoutValidation("x-guest-token", token);
             //request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");
             //request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes");
             return request;

src/BirdsiteLive.Twitter/TwitterTweetsService.cs

@@ -55,16 +55,14 @@ namespace BirdsiteLive.Twitter
                 "https://api.twitter.com/graphql/XjlydVWHFIDaAUny86oh2g/TweetDetail?variables=%7B%22focalTweetId%22%3A%22"
                 + statusId +
                 "%22,%22with_rux_injections%22%3Atrue,%22includePromotedContent%22%3Afalse,%22withCommunity%22%3Afalse,%22withQuickPromoteEligibilityTweetFields%22%3Afalse,%22withBirdwatchNotes%22%3Afalse,%22withSuperFollowsUserFields%22%3Afalse,%22withDownvotePerspective%22%3Afalse,%22withReactionsMetadata%22%3Afalse,%22withReactionsPerspective%22%3Afalse,%22withSuperFollowsTweetFields%22%3Afalse,%22withVoice%22%3Atrue,%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue,%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue,%22verified_phone_label_enabled%22%3Afalse,%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue,%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse,%22tweetypie_unmention_optimization_enabled%22%3Atrue,%22vibe_api_enabled%22%3Atrue,%22responsive_web_edit_tweet_api_enabled%22%3Atrue,%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Afalse,%22view_counts_everywhere_api_enabled%22%3Atrue,%22longform_notetweets_consumption_enabled%22%3Atrue,%22tweet_awards_web_tipping_enabled%22%3Afalse,%22freedom_of_speech_not_reach_fetch_enabled%22%3Afalse,%22standardized_nudges_misinfo%22%3Atrue,%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse,%22interactive_text_enabled%22%3Atrue,%22responsive_web_text_conversations_enabled%22%3Afalse,%22longform_notetweets_richtext_consumption_enabled%22%3Afalse,%22responsive_web_enhance_cards_enabled%22%3Atrue%7D";
+            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL, true);
             try
             {
                 JsonDocument tweet;
-                using (var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL))
-                {
-                    var httpResponse = await client.SendAsync(request);
-                    httpResponse.EnsureSuccessStatusCode();
-                    var c = await httpResponse.Content.ReadAsStringAsync();
-                    tweet = JsonDocument.Parse(c);
-                }
+                var httpResponse = await client.SendAsync(request);
+                httpResponse.EnsureSuccessStatusCode();
+                var c = await httpResponse.Content.ReadAsStringAsync();
+                tweet = JsonDocument.Parse(c);
 
 
                 var timeline = tweet.RootElement.GetProperty("data").GetProperty("threaded_conversation_with_injections_v2")
@@ -77,6 +75,7 @@ namespace BirdsiteLive.Twitter
             catch (Exception e)
             {
                 _logger.LogError(e, "Error retrieving tweet {TweetId}", statusId);
+                await _twitterAuthenticationInitializer.RefreshClient(request);
                 return null;
             }
         }
@@ -105,7 +104,7 @@ namespace BirdsiteLive.Twitter
                 userId + "%22,%22count%22%3A40,%22includePromotedContent%22%3Atrue,%22withCommunity%22%3Atrue,%22withSuperFollowsUserFields%22%3Atrue,%22withDownvotePerspective%22%3Afalse,%22withReactionsMetadata%22%3Afalse,%22withReactionsPerspective%22%3Afalse,%22withSuperFollowsTweetFields%22%3Atrue,%22withVoice%22%3Atrue,%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue,%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue,%22verified_phone_label_enabled%22%3Afalse,%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue,%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse,%22tweetypie_unmention_optimization_enabled%22%3Atrue,%22vibe_api_enabled%22%3Atrue,%22responsive_web_edit_tweet_api_enabled%22%3Atrue,%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue,%22view_counts_everywhere_api_enabled%22%3Atrue,%22longform_notetweets_consumption_enabled%22%3Atrue,%22tweet_awards_web_tipping_enabled%22%3Afalse,%22freedom_of_speech_not_reach_fetch_enabled%22%3Afalse,%22standardized_nudges_misinfo%22%3Atrue,%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse,%22interactive_text_enabled%22%3Atrue,%22responsive_web_text_conversations_enabled%22%3Afalse,%22longform_notetweets_richtext_consumption_enabled%22%3Afalse,%22responsive_web_enhance_cards_enabled%22%3Afalse%7D";
             JsonDocument results;
             List<ExtractedTweet> extractedTweets = new List<ExtractedTweet>();
-            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL);
+            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL, true);
             try
             {
 
@@ -119,7 +118,7 @@ namespace BirdsiteLive.Twitter
             catch (HttpRequestException e)
             {
                 _logger.LogError(e, "Error retrieving timeline of {Username}; refreshing client", username);
-                await _twitterAuthenticationInitializer.RefreshClient(request);
+                //await _twitterAuthenticationInitializer.RefreshClient(request);
                 return null;
             }
             catch (Exception e)

src/BirdsiteLive.Twitter/TwitterUserService.cs

@@ -40,7 +40,7 @@ namespace BirdsiteLive.Twitter
 
             JsonDocument res;
             var client = await _twitterAuthenticationInitializer.MakeHttpClient();
-            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), endpoint.Replace("elonmusk", username));
+            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), endpoint.Replace("elonmusk", username), true);
             try
             {
 

src/BirdsiteLive/Controllers/DebugingController.cs

@@ -10,7 +10,6 @@ using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Domain;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.Mvc.ViewFeatures;
-using Newtonsoft.Json;
 
 namespace BirdsiteLive.Controllers
 {

src/BirdsiteLive/Controllers/UsersController.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text.Json;
 using System.Net.Mime;
 using System.Text.RegularExpressions;
 using System.Threading;
@@ -19,13 +20,12 @@ using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Primitives;
-using Newtonsoft.Json;
 
 namespace BirdsiteLive.Controllers
 {
     public class UsersController : Controller
     {
-        private readonly ITwitterUserService _twitterUserService;
+        private readonly ICachedTwitterUserService _twitterUserService;
         private readonly ICachedTwitterTweetsService _twitterTweetService;
         private readonly IUserService _userService;
         private readonly IStatusService _statusService;
@@ -33,7 +33,7 @@ namespace BirdsiteLive.Controllers
         private readonly ILogger<UsersController> _logger;
 
         #region Ctor
-        public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ICachedTwitterTweetsService twitterTweetService, ILogger<UsersController> logger)
+        public UsersController(ICachedTwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ICachedTwitterTweetsService twitterTweetService, ILogger<UsersController> logger)
         {
             _twitterUserService = twitterUserService;
             _userService = userService;
@@ -111,7 +111,7 @@ namespace BirdsiteLive.Controllers
                     if (isSaturated) return new ObjectResult("Too Many Requests") { StatusCode = 429 };
                     if (notFound) return NotFound();
                     var apUser = _userService.GetUser(user);
-                    var jsonApUser = JsonConvert.SerializeObject(apUser);
+                    var jsonApUser = System.Text.Json.JsonSerializer.Serialize(apUser);
                     return Content(jsonApUser, "application/activity+json; charset=utf-8");
                 }
             }
@@ -155,7 +155,7 @@ namespace BirdsiteLive.Controllers
 
                 if (r.Contains("application/activity+json"))
                 {
-                    var jsonApUser = JsonConvert.SerializeObject(status);
+                    var jsonApUser = JsonSerializer.Serialize(status);
                     return Content(jsonApUser, "application/activity+json; charset=utf-8");
                 }
             }
@@ -185,7 +185,7 @@ namespace BirdsiteLive.Controllers
 
             var status = _statusService.GetActivity(id, tweet);
 
-            var jsonApUser = JsonConvert.SerializeObject(status);
+            var jsonApUser = JsonSerializer.Serialize(status);
             return Content(jsonApUser, "application/activity+json; charset=utf-8");
         }
 
@@ -269,7 +269,7 @@ namespace BirdsiteLive.Controllers
             {
                 id = $"https://{_instanceSettings.Domain}/users/{id}/followers"
             };
-            var jsonApUser = JsonConvert.SerializeObject(followers);
+            var jsonApUser = JsonSerializer.Serialize(followers);
             return Content(jsonApUser, "application/activity+json; charset=utf-8");
         }
     }

src/BirdsiteLive/Properties/launchSettings.json

@@ -19,7 +19,8 @@
       "commandName": "Project",
       "launchBrowser": true,
       "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development"
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "Instance__ParallelTwitterRequests": "0"
       },
       "applicationUrl": "http://localhost:5000"
     },

src/BirdsiteLive/Services/CachedStatisticsService.cs

@@ -15,7 +15,7 @@ namespace BirdsiteLive.Services
         private readonly ITwitterUserDal _twitterUserDal;
         private readonly IFollowersDal _followersDal;
 
-        private static CachedStatistics _cachedStatistics;
+        private static Task<CachedStatistics> _cachedStatistics;
         private readonly InstanceSettings _instanceSettings;
 
         #region Ctor
@@ -24,28 +24,36 @@ namespace BirdsiteLive.Services
             _twitterUserDal = twitterUserDal;
             _instanceSettings = instanceSettings;
             _followersDal = followersDal;
+            _cachedStatistics = CreateStats();
         }
         #endregion
 
         public async Task<CachedStatistics> GetStatisticsAsync()
         {
-            if (_cachedStatistics == null ||
-                (DateTime.UtcNow - _cachedStatistics.RefreshedTime).TotalMinutes > 15)
+            var stats = await _cachedStatistics;
+            if ((DateTime.UtcNow - stats.RefreshedTime).TotalMinutes > 5)
             {
-                var twitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync();
-                var twitterSyncLag = await _twitterUserDal.GetTwitterSyncLag();
-                var fediverseUsers = await _followersDal.GetFollowersCountAsync();
-
-                _cachedStatistics = new CachedStatistics
-                {
-                    RefreshedTime = DateTime.UtcNow,
-                    SyncLag = twitterSyncLag,
-                    TwitterUsers = twitterUserCount,
-                    FediverseUsers = fediverseUsers
-                };
+                _cachedStatistics = CreateStats();
             }
 
-            return _cachedStatistics;
+            return stats;
+        }
+
+        private async Task<CachedStatistics> CreateStats()
+        {
+            var twitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync();
+            var twitterSyncLag = await _twitterUserDal.GetTwitterSyncLag();
+            var fediverseUsers = await _followersDal.GetFollowersCountAsync();
+
+            var stats = new CachedStatistics
+            {
+                RefreshedTime = DateTime.UtcNow,
+                SyncLag = twitterSyncLag,
+                TwitterUsers = twitterUserCount,
+                FediverseUsers = fediverseUsers
+            };
+        
+            return stats;
         }
     }
 

src/BirdsiteLive/Startup.cs

@@ -10,6 +10,7 @@ using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Postgres.DataAccessLayers;
 using BirdsiteLive.DAL.Postgres.Settings;
 using BirdsiteLive.Models;
+using BirdsiteLive.Services;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Tools;
 using Lamar;
@@ -98,6 +99,8 @@ namespace BirdsiteLive
             services.For<ITwitterUserService>().Use<TwitterUserService>().Singleton();
 
             services.For<ITwitterAuthenticationInitializer>().Use<TwitterAuthenticationInitializer>().Singleton();
+            
+            services.For<ICachedStatisticsService>().Use<CachedStatisticsService>().Singleton();
 
             services.Scan(_ =>
             {

src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs

@@ -118,14 +118,20 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
             }
         }
 
-        public async Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber)
+        public async Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber, int nStart, int nEnd, int m)
         {
-            var query = "SELECT * FROM (SELECT unnest(followings) as follow FROM followers GROUP BY follow) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id  ORDER BY lastSync ASC NULLS FIRST LIMIT $1";
+            var query = "SELECT * FROM (SELECT unnest(followings) as follow FROM followers GROUP BY follow) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id WHERE mod(id, $2) >= $3 AND mod(id, $2) <= $4 ORDER BY lastSync ASC NULLS FIRST LIMIT $1";
 
             await using var connection = DataSource.CreateConnection();
             await connection.OpenAsync();
             await using var command = new NpgsqlCommand(query, connection) {
-                Parameters = { new() { Value = maxNumber}}
+                Parameters =
+                {
+                    new() { Value = maxNumber},
+                    new() { Value = m},
+                    new() { Value = nStart},
+                    new() { Value = nEnd}
+                }
             };
             var reader = await command.ExecuteReaderAsync();
             var results = new List<SyncTwitterUser>();

src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs

@@ -9,7 +9,7 @@ namespace BirdsiteLive.DAL.Contracts
         Task CreateTwitterUserAsync(string acct, long lastTweetPostedId);
         Task<SyncTwitterUser> GetTwitterUserAsync(string acct);
         Task<SyncTwitterUser> GetTwitterUserAsync(int id);
-        Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber);
+        Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber, int nStart, int nEnd, int m);
         Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber);
         Task<SyncTwitterUser[]> GetAllTwitterUsersAsync();
         Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync);

src/Tests/BirdsiteLive.ActivityPub.Tests/ActivityTests.cs

@@ -3,31 +3,21 @@ using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub.Tests
 {
-    //[TestClass]
-    //public class ActivityTests
-    //{
-    //    [TestMethod]
-    //    public void FollowDeserializationTest()
-    //    {
-    //        var json = "{ \"@context\":\"https://www.w3.org/ns/activitystreams\",\"id\":\"https://mastodon.technology/c94567cf-1fda-42ba-82fc-a0f82f63ccbe\",\"type\":\"Follow\",\"actor\":\"https://mastodon.technology/users/testtest\",\"object\":\"https://4a120ca2680e.ngrok.io/users/manu\"}";
+    [TestClass]
+    public class ActivityTests
+    {
+        [TestMethod]
+        public void Serialize()
+        {
+            var obj = new Actor
+            {
+                type = "Person",
+                id = "id"
+            };
 
-    //        var data = JsonConvert.DeserializeObject<Activity>(json);
+            var json = JsonConvert.SerializeObject(obj);
 
-    //        Assert.AreEqual("https://mastodon.technology/c94567cf-1fda-42ba-82fc-a0f82f63ccbe", data.id);
-    //        Assert.AreEqual("Follow", data.type);
-    //        Assert.AreEqual("https://4a120ca2680e.ngrok.io/users/manu", data.apObject);
-    //    }
 
-    //    [TestMethod]
-    //    public void UndoDeserializationTest()
-    //    {
-    //        var json =
-    //            "{\"@context\":\"https://www.w3.org/ns/activitystreams\",\"id\":\"https://mastodon.technology/users/testtest#follows/225982/undo\",\"type\":\"Undo\",\"actor\":\"https://mastodon.technology/users/testtest\",\"object\":{\"id\":\"https://mastodon.technology/c94567cf-1fda-42ba-82fc-a0f82f63ccbe\",\"type\":\"Follow\",\"actor\":\"https://mastodon.technology/users/testtest\",\"object\":\"https://4a120ca2680e.ngrok.io/users/manu\"}}";
-
-    //        var data = JsonConvert.DeserializeObject<Activity>(json);
-    //        Assert.AreEqual("https://mastodon.technology/users/testtest#follows/225982/undo", data.id);
-    //        Assert.AreEqual("Undo", data.type);
-    //        Assert.AreEqual("https://4a120ca2680e.ngrok.io/users/manu", data.apObject);
-    //    }
-    //}
+        }
+    }
 }

src/Tests/BirdsiteLive.ActivityPub.Tests/ApDeserializerTests.cs

@@ -1,6 +1,6 @@
 using BirdsiteLive.ActivityPub.Models;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
-using Newtonsoft.Json;
+using System.Text.Json;
 
 namespace BirdsiteLive.ActivityPub.Tests
 {
@@ -28,9 +28,11 @@ namespace BirdsiteLive.ActivityPub.Tests
             var data = ApDeserializer.ProcessActivity(json) as ActivityUndoFollow;
             Assert.AreEqual("https://mastodon.technology/users/testtest#follows/225982/undo", data.id);
             Assert.AreEqual("Undo", data.type);
+            Assert.AreEqual("https://www.w3.org/ns/activitystreams", data.context);
             Assert.AreEqual("Follow", data.apObject.type);
             Assert.AreEqual("https://mastodon.technology/users/testtest", data.apObject.actor);
             Assert.AreEqual("https://4a120ca2680e.ngrok.io/users/manu", data.apObject.apObject);
+            Assert.AreEqual(null, data.apObject.context);
         }
 
         [TestMethod]
@@ -62,14 +64,100 @@ namespace BirdsiteLive.ActivityPub.Tests
             Assert.AreEqual("https://mastodon.technology/users/deleteduser", data.actor);
             Assert.AreEqual("https://mastodon.technology/users/deleteduser", data.apObject);
         }
+        // {"object":{"object":"https://bird.makeup/users/spectatorindex","id":"https://masto.ai/b89eb86e-c902-48bc-956f-94f081617f18","type":"Follow","actor":"https://masto.ai/users/singha"},"@context":"https://www.w3.org/ns/activitystreams","id":"https://bird.makeup/users/spectatorindex#accepts/follows/27363118-e61e-4710-a41c-75dd5d54912f","type":"Accept","actor":"https://bird.makeup/users/spectatorindex"}
+        // {"object":{"object":"https://bird.makeup/users/moltke","id":"https://universeodon.com/81cddd78-d7d6-4665-aa21-7bcfbea82b6b","type":"Follow","actor":"https://universeodon.com/users/amhrasmussen"},"@context":"https://www.w3.org/ns/activitystreams","id":"https://bird.makeup/users/moltke#accepts/follows/d28146be-e884-4e91-8385-19fa004f35b3","type":"Accept","actor":"https://bird.makeup/users/moltke"}
 
-        //[TestMethod]
-        //public void NoteDeserializationTest()
-        //{
-        //    var json =
-        //        "{\"@context\":[\"https://www.w3.org/ns/activitystreams\",{\"ostatus\":\"http://ostatus.org#\",\"atomUri\":\"ostatus:atomUri\",\"inReplyToAtomUri\":\"ostatus:inReplyToAtomUri\",\"conversation\":\"ostatus:conversation\",\"sensitive\":\"as:sensitive\",\"toot\":\"http://joinmastodon.org/ns#\",\"votersCount\":\"toot:votersCount\"}],\"id\":\"https://mastodon.technology/users/testtest/statuses/104424839893177182/activity\",\"type\":\"Create\",\"actor\":\"https://mastodon.technology/users/testtest\",\"published\":\"2020-06-29T02:10:04Z\",\"to\":[\"https://mastodon.technology/users/testtest/followers\"],\"cc\":[],\"object\":{\"id\":\"https://mastodon.technology/users/testtest/statuses/104424839893177182\",\"type\":\"Note\",\"summary\":null,\"inReplyTo\":null,\"published\":\"2020-06-29T02:10:04Z\",\"url\":\"https://mastodon.technology/@testtest/104424839893177182\",\"attributedTo\":\"https://mastodon.technology/users/testtest\",\"to\":[\"https://mastodon.technology/users/testtest/followers\"],\"cc\":[],\"sensitive\":false,\"atomUri\":\"https://mastodon.technology/users/testtest/statuses/104424839893177182\",\"inReplyToAtomUri\":null,\"conversation\":\"tag:mastodon.technology,2020-06-29:objectId=34900058:objectType=Conversation\",\"content\":\"<p>test</p>\",\"contentMap\":{\"en\":\"<p>test</p>\"},\"attachment\":[],\"tag\":[],\"replies\":{\"id\":\"https://mastodon.technology/users/testtest/statuses/104424839893177182/replies\",\"type\":\"Collection\",\"first\":{\"type\":\"CollectionPage\",\"next\":\"https://mastodon.technology/users/testtest/statuses/104424839893177182/replies?only_other_accounts=true&page=true\",\"partOf\":\"https://mastodon.technology/users/testtest/statuses/104424839893177182/replies\",\"items\":[]}}}}";
 
-        //    var data = ApDeserializer.ProcessActivity(json) as ActivityAcceptFollow;
-        //}
+        [TestMethod]
+        public void ActorDeserializationTest()
+        {
+            var json = """
+					{
+						"@context": [
+							"https://www.w3.org/ns/activitystreams",
+							"https://w3id.org/security/v1",
+							{
+								"manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+								"toot": "http://joinmastodon.org/ns#",
+								"featured": {
+									"@id": "toot:featured",
+									"@type": "@id"
+								},
+								"featuredTags": {
+									"@id": "toot:featuredTags",
+									"@type": "@id"
+								},
+								"alsoKnownAs": {
+									"@id": "as:alsoKnownAs",
+									"@type": "@id"
+								},
+								"movedTo": {
+									"@id": "as:movedTo",
+									"@type": "@id"
+								},
+								"schema": "http://schema.org#",
+								"PropertyValue": "schema:PropertyValue",
+								"value": "schema:value",
+								"discoverable": "toot:discoverable",
+								"Device": "toot:Device",
+								"Ed25519Signature": "toot:Ed25519Signature",
+								"Ed25519Key": "toot:Ed25519Key",
+								"Curve25519Key": "toot:Curve25519Key",
+								"EncryptedMessage": "toot:EncryptedMessage",
+								"publicKeyBase64": "toot:publicKeyBase64",
+								"deviceId": "toot:deviceId",
+								"claim": {
+									"@type": "@id",
+									"@id": "toot:claim"
+								},
+								"fingerprintKey": {
+									"@type": "@id",
+									"@id": "toot:fingerprintKey"
+								},
+								"identityKey": {
+									"@type": "@id",
+									"@id": "toot:identityKey"
+								},
+								"devices": {
+									"@type": "@id",
+									"@id": "toot:devices"
+								},
+								"messageFranking": "toot:messageFranking",
+								"messageType": "toot:messageType",
+								"cipherText": "toot:cipherText",
+								"suspended": "toot:suspended"
+							}
+						],
+						"id": "https://mastodon.online/users/devvincent",
+						"type": "Person",
+						"following": "https://mastodon.online/users/devvincent/following",
+						"followers": "https://mastodon.online/users/devvincent/followers",
+						"inbox": "https://mastodon.online/users/devvincent/inbox",
+						"outbox": "https://mastodon.online/users/devvincent/outbox",
+						"featured": "https://mastodon.online/users/devvincent/collections/featured",
+						"featuredTags": "https://mastodon.online/users/devvincent/collections/tags",
+						"preferredUsername": "devvincent",
+						"name": "",
+						"summary": "",
+						"url": "https://mastodon.online/@devvincent",
+						"manuallyApprovesFollowers": false,
+						"discoverable": false,
+						"published": "2022-05-08T00:00:00Z",
+						"devices": "https://mastodon.online/users/devvincent/collections/devices",
+						"publicKey": {
+							"id": "https://mastodon.online/users/devvincent#main-key",
+							"owner": "https://mastodon.online/users/devvincent",
+							"publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA7U07uS4zu5jeZSBVZ072\naXcTeVQc0baM8BBUvJkpX+mV2vh+V4yfqN44KzFxlkk8XcAoidt8HBAvpQ/5yCwZ\neGS2ySCxC+sqvErIbaYadWVHGJhZjLYPVa0n8wvkqRQ0aUJ8K17/wY+/YYfukgeC\nTGHGoyzDDZZxrR1Z8LTvImSEkYooTvvzaaFaTUnFwCKepxftKLdJAfp4sP4l1Zom\nUZGwaYimuJmN1bfhet/2v0S7M7/XPlmVRpfUluE2vYE0RtJt3BVDZfoWEGJPk9us\nN/JHu6UBUh6UM6ASFy5MlDLh36OxyO9sVx1WgQlNDmu2qcGUIkIgqTKppDCIP3Xk\nVQIDAQAB\n-----END PUBLIC KEY-----\n"
+						},
+						"tag": [],
+						"attachment": [],
+						"endpoints": {
+							"sharedInbox": "https://mastodon.online/inbox"
+						}
+					}
+				""";
+
+            var actor = JsonSerializer.Deserialize<Actor>(json);
+        }
     }
 }

src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj

@@ -4,6 +4,8 @@
     <TargetFramework>net6</TargetFramework>
 
     <IsPackable>false</IsPackable>
+
+    <LangVersion>11</LangVersion>
   </PropertyGroup>
 
   <ItemGroup>

src/Tests/BirdsiteLive.Domain.Tests/ActivityServiceTests.cs

@@ -0,0 +1,96 @@
+using System.Net.Http;
+using System.Threading.Tasks;
+using BirdsiteLive.ActivityPub;
+using BirdsiteLive.Common.Settings;
+using BirdsiteLive.Domain.Factories;
+using Microsoft.Extensions.Logging;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Moq;
+using System.Text.Json;
+
+
+namespace BirdsiteLive.Domain.Tests
+{
+    [TestClass]
+    public class ActivityServiceTests
+    {
+        private readonly InstanceSettings _settings;
+
+        #region Ctor
+        public ActivityServiceTests()
+        {
+            _settings = new InstanceSettings
+            {
+                Domain = "domain.name"
+            };
+        }
+        #endregion
+
+        [TestMethod]
+        public async Task ActivityTest()
+        {
+            var logger1 = new Mock<ILogger<ActivityPubService>>();
+            var httpFactory = new Mock<IHttpClientFactory>();
+            var keyFactory = new Mock<MagicKeyFactory>();
+            var cryptoService = new CryptoService(keyFactory.Object);
+            httpFactory.Setup(_ => _.CreateClient(string.Empty)).Returns(new HttpClient());
+            var service = new ActivityPubService(cryptoService, _settings, httpFactory.Object, logger1.Object);
+
+            var activity = new ActivityAcceptFollow()
+            {
+                id = "awef",
+                context = "https://www.w3.org/ns/activitystreams",
+                type = "Accept",
+                actor = "https://mastodon.technology/users/testtest",
+                apObject = new ActivityFollow()
+                {
+                    context = "https://www.w3.org/ns/activitystreams",
+                    id = "abc",
+                    type = "Follow",
+                    actor = "https://mastodon.technology/users/testtest2",
+                    apObject = "https://mastodon.technology/users/testtest3",
+                }
+                
+            };
+            var json =
+                """{"object":{"object":"https://mastodon.technology/users/testtest3","@context":"https://www.w3.org/ns/activitystreams","id":"abc","type":"Follow","actor":"https://mastodon.technology/users/testtest2"},"@context":"https://www.w3.org/ns/activitystreams","id":"awef","type":"Accept","actor":"https://mastodon.technology/users/testtest"}""";
+            #region Validations
+
+            var req = service.BuildRequest(activity, "google.com", "tata", "awef");
+            
+            Assert.AreEqual(await req.Content.ReadAsStringAsync(), json);
+
+            #endregion
+        }
+        [TestMethod]
+        public async Task AcceptFollow()
+        {
+ 
+
+            var logger1 = new Mock<ILogger<ActivityPubService>>();
+            var httpFactory = new Mock<IHttpClientFactory>();
+            var keyFactory = new Mock<MagicKeyFactory>();
+            var cryptoService = new CryptoService(keyFactory.Object);
+            httpFactory.Setup(_ => _.CreateClient(string.Empty)).Returns(new HttpClient());
+            var service = new ActivityPubService(cryptoService, _settings, httpFactory.Object, logger1.Object);
+
+            var json = "{ \"@context\":\"https://www.w3.org/ns/activitystreams\",\"id\":\"https://mastodon.technology/c94567cf-1fda-42ba-82fc-a0f82f63ccbe\",\"type\":\"Follow\",\"actor\":\"https://mastodon.technology/users/testtest\",\"object\":\"https://4a120ca2680e.ngrok.io/users/manu\"}";
+            var activity = ApDeserializer.ProcessActivity(json) as ActivityFollow;
+
+            var jsonres =
+                "{\"object\":{\"id\":\"https://mastodon.technology/c94567cf-1fda-42ba-82fc-a0f82f63ccbe\",\"type\":\"Follow\",\"actor\":\"https://mastodon.technology/users/testtest\",\"object\":\"https://4a120ca2680e.ngrok.io/users/manu\"},\"@context\":\"https://www.w3.org/ns/activitystreams\",\"id\":\"https://4a120ca2680e.ngrok.io/users/manu#accepts/follows/32e5fbda-9159-4ede-8249-9d008092d26f\",\"type\":\"Accept\",\"actor\":\"https://4a120ca2680e.ngrok.io/users/manu\"}";
+            var activityRes = ApDeserializer.ProcessActivity(jsonres) as ActivityAcceptFollow;
+            #region Validations
+
+            var req = service.BuildAcceptFollow(activity);
+            
+            string s = JsonSerializer.Serialize(req);
+            
+            Assert.AreEqual(req.actor, activityRes.actor);
+            Assert.AreEqual(req.context, activityRes.context);
+
+            #endregion
+        }
+
+    }
+}

src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>net6</TargetFramework>
-
+    <LangVersion>11</LangVersion>
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveFollowersProcessorTests.cs

@@ -14,6 +14,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
     [TestClass]
     public class RetrieveFollowersProcessorTests
     {
+        [Ignore]
         [TestMethod]
         public async Task ProcessAsync_Test()
         {

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs

@@ -29,19 +29,32 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 new SyncTwitterUser(),
             };
             var maxUsers = 1000;
+            var instanceSettings = new InstanceSettings()
+            {
+                n_start = 1,
+            };
             #endregion
 
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .Setup(x => x.GetAllTwitterUsersWithFollowersAsync(
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
                     It.Is<int>(y => true)))
                 .ReturnsAsync(users);
             
+            var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
+            followersDalMock
+                .Setup(x => x.GetFollowersAsync(It.Is<int>(x => true)))
+                .ReturnsAsync(new Follower[] {});
+            
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
+            
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
             processor.WaitFactor = 10;
             var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -66,12 +79,19 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 users.Add(new SyncTwitterUser());
 
             var maxUsers = 1000;
+            var instanceSettings = new InstanceSettings()
+            {
+                n_start = 1,
+            };
             #endregion
 
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .SetupSequence(x => x.GetAllTwitterUsersWithFollowersAsync(
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
                     It.Is<int>(y => true)))
                 .ReturnsAsync(users.ToArray())
                 .ReturnsAsync(new SyncTwitterUser[0])
@@ -79,10 +99,15 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 .ReturnsAsync(new SyncTwitterUser[0])
                 .ReturnsAsync(new SyncTwitterUser[0]);
 
+            var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
+            followersDalMock
+                .Setup(x => x.GetFollowersAsync(It.Is<int>(x => true)))
+                .ReturnsAsync(new Follower[] {});
+            
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
             processor.WaitFactor = 2;
             var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -107,12 +132,19 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 users.Add(new SyncTwitterUser());
 
             var maxUsers = 1000;
+            var instanceSettings = new InstanceSettings()
+            {
+                n_start = 1,
+            };
             #endregion
 
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .SetupSequence(x => x.GetAllTwitterUsersWithFollowersAsync(
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
                     It.Is<int>(y => true)))
                 .ReturnsAsync(users.ToArray())
                 .ReturnsAsync(new SyncTwitterUser[0])
@@ -120,10 +152,15 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 .ReturnsAsync(new SyncTwitterUser[0])
                 .ReturnsAsync(new SyncTwitterUser[0]);
             
+            var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
+            followersDalMock
+                .Setup(x => x.GetFollowersAsync(It.Is<int>(x => true)))
+                .ReturnsAsync(new Follower[] {});
+            
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
             processor.WaitFactor = 2;
             var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -144,6 +181,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var buffer = new BufferBlock<UserWithDataToSync[]>();
 
             var maxUsers = 1000;
+            var instanceSettings = new InstanceSettings()
+            {
+                n_start = 1,
+            };
             #endregion
 
             #region Mocks
@@ -151,13 +192,21 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .Setup(x => x.GetAllTwitterUsersWithFollowersAsync(
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
                     It.Is<int>(y => true)))
                 .ReturnsAsync(new SyncTwitterUser[0]);
 
+            var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
+            followersDalMock
+                .Setup(x => x.GetFollowersAsync(It.Is<int>(x => true)))
+                .ReturnsAsync(new Follower[] {});
+            
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
             processor.WaitFactor = 1;
             var t =processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -176,19 +225,31 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var buffer = new BufferBlock<UserWithDataToSync[]>();
 
             var maxUsers = 1000;
+            var instanceSettings = new InstanceSettings()
+            {
+                n_start = 1,
+            };
             #endregion
 
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .Setup(x => x.GetAllTwitterUsersWithFollowersAsync(
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
                     It.Is<int>(y => true)))
                 .Returns(async () => await DelayFaultedTask<SyncTwitterUser[]>(new Exception()));
 
+            var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
+            followersDalMock
+                .Setup(x => x.GetFollowersAsync(It.Is<int>(x => true)))
+                .ReturnsAsync(new Follower[] {});
+            
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
             processor.WaitFactor = 10;
             var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -210,15 +271,24 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             canTokenS.Cancel();
 
             var maxUsers = 1000;
+            var instanceSettings = new InstanceSettings()
+            {
+                n_start = 1,
+            };
             #endregion
 
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             
+            var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
+            followersDalMock
+                .Setup(x => x.GetFollowersAsync(It.Is<int>(x => true)))
+                .ReturnsAsync(new Follower[] {});
+            
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
             processor.WaitFactor = 1;
             await processor.GetTwitterUsersAsync(buffer, canTokenS.Token);
         }

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs

@@ -88,7 +88,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new[] {userWithTweets}, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();
@@ -170,7 +170,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new[] {userWithTweets}, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();
@@ -261,7 +261,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new[] {userWithTweets}, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();
@@ -353,7 +353,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();
@@ -450,7 +450,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();
@@ -530,7 +530,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();
@@ -611,7 +611,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();
@@ -700,7 +700,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();
@@ -790,7 +790,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();
@@ -880,7 +880,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();
@@ -970,7 +970,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();
@@ -1065,7 +1065,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new [] { userWithTweets }, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();

src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs

@@ -37,10 +37,11 @@ namespace BirdsiteLive.ActivityPub.Tests
                 ))
                 .ReturnsAsync(new SyncTwitterUser { TwitterUserId = default });
 
-            ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(httpFactory.Object, logger1.Object);
+            ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(httpFactory.Object, settings, logger1.Object);
             ITwitterUserService user = new TwitterUserService(auth, stats.Object, logger2.Object);
             ICachedTwitterUserService user2 = new CachedTwitterUserService(user, settings);
             _tweetService = new TwitterTweetsService(auth, stats.Object, user2, twitterDal.Object, settings, logger3.Object);
+
         }
 
         [TestMethod]
@@ -48,7 +49,7 @@ namespace BirdsiteLive.ActivityPub.Tests
         {
             var tweets = await _tweetService.GetTimelineAsync("kobebryant", 1218020971346444288);
             Assert.AreEqual(tweets[0].MessageContent, "Continuing to move the game forward @KingJames. Much respect my brother 💪🏾 #33644");
-            Assert.AreEqual(tweets.Length, 8);
+            Assert.IsTrue(tweets.Length > 5);
         }
 
         [TestMethod]

src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs

@@ -1,4 +1,6 @@
-using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+using System.IO;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Microsoft.Extensions.Logging;
 using System.Threading.Tasks;
 using BirdsiteLive.Twitter;
@@ -18,6 +20,7 @@ namespace BirdsiteLive.ActivityPub.Tests
         [TestInitialize]
         public async Task TestInit()
         {
+            
             var logger1 = new Mock<ILogger<TwitterAuthenticationInitializer>>(MockBehavior.Strict);
             var logger2 = new Mock<ILogger<TwitterUserService>>(MockBehavior.Strict);
             var logger3 = new Mock<ILogger<TwitterTweetsService>>();
@@ -29,10 +32,11 @@ namespace BirdsiteLive.ActivityPub.Tests
             {
                 Domain = "domain.name"
             };
-            ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(httpFactory.Object, logger1.Object);
+            ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(httpFactory.Object, settings, logger1.Object);
             ITwitterUserService user = new TwitterUserService(auth, stats.Object, logger2.Object);
             ICachedTwitterUserService user2 = new CachedTwitterUserService(user, settings);
             _tweetService = new TwitterTweetsService(auth, stats.Object, user2, twitterDal.Object, settings, logger3.Object);
+
         }
 
         [TestMethod]
@@ -74,6 +78,7 @@ namespace BirdsiteLive.ActivityPub.Tests
             Assert.IsTrue(tweet.Media[0].Url.StartsWith("https://video.twimg.com/"));
         }
 
+        [Ignore]
         [TestMethod]
         public async Task GifAndQT()
         {

src/Tests/BirdsiteLive.Twitter.Tests/UsersTest.cs

@@ -6,6 +6,7 @@ using BirdsiteLive.Twitter.Tools;
 using BirdsiteLive.Statistics.Domain;
 using Moq;
 using System.Net.Http;
+using BirdsiteLive.Common.Settings;
 
 namespace BirdsiteLive.ActivityPub.Tests
 {
@@ -21,8 +22,12 @@ namespace BirdsiteLive.ActivityPub.Tests
             var logger3 = new Mock<ILogger<TwitterUserService>>();
             var stats = new Mock<ITwitterStatisticsHandler>();
             var httpFactory = new Mock<IHttpClientFactory>();
+            var settings = new InstanceSettings
+            {
+                Domain = "domain.name"
+            };
             httpFactory.Setup(_ => _.CreateClient(string.Empty)).Returns(new HttpClient());
-            ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(httpFactory.Object, logger1.Object);
+            ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(httpFactory.Object, settings, logger1.Object);
             _tweetService = new TwitterUserService(auth, stats.Object, logger3.Object);
         }