From 8b5d03e0f1964e66f5ee13ef33c5edb47ae96ecc Mon Sep 17 00:00:00 2001
From: Vincent Cloutier <vincent@cloutier.co>
Date: Sun, 12 Mar 2023 14:10:59 -0400
Subject: [PATCH 01/20] conversion to System.Text.Json

---
 src/BSLManager/Tools/SettingsManager.cs       |  6 +--
 .../ApDeserializer.cs                         | 32 +++++++--------
 .../BirdsiteLive.ActivityPub.csproj           |  1 -
 .../Converters/ContextArrayConverter.cs       | 39 -------------------
 .../Models/Activity.cs                        |  5 +--
 .../Models/ActivityAccept.cs                  |  6 +--
 .../Models/ActivityAcceptFollow.cs            |  6 +--
 .../Models/ActivityAcceptUndoFollow.cs        |  4 +-
 .../Models/ActivityCreateNote.cs              |  7 ++--
 .../Models/ActivityDelete.cs                  |  6 +--
 .../Models/ActivityFollow.cs                  |  4 +-
 .../Models/ActivityRejectFollow.cs            |  4 +-
 .../Models/ActivityUndo.cs                    |  4 +-
 .../Models/ActivityUndoFollow.cs              |  4 +-
 src/BirdsiteLive.ActivityPub/Models/Actor.cs  |  6 +--
 .../Models/Followers.cs                       |  5 +--
 .../Models/NestedActivity.cs                  | 16 ++++++++
 src/BirdsiteLive.ActivityPub/Models/Note.cs   |  6 +--
 src/BirdsiteLive.Domain/ActivityPubService.cs |  7 ++--
 src/BirdsiteLive.Domain/UserService.cs        |  2 +-
 .../Controllers/DebugingController.cs         |  1 -
 .../Controllers/UsersController.cs            | 10 ++---
 .../Properties/launchSettings.json            |  3 +-
 23 files changed, 77 insertions(+), 107 deletions(-)
 delete mode 100644 src/BirdsiteLive.ActivityPub/Converters/ContextArrayConverter.cs
 create mode 100644 src/BirdsiteLive.ActivityPub/Models/NestedActivity.cs

diff --git a/src/BSLManager/Tools/SettingsManager.cs b/src/BSLManager/Tools/SettingsManager.cs
index 4763822..87daeba 100644
--- a/src/BSLManager/Tools/SettingsManager.cs
+++ b/src/BSLManager/Tools/SettingsManager.cs
@@ -1,8 +1,8 @@
 using System;
 using System.IO;
+using System.Text.Json;
 using System.Runtime.CompilerServices;
 using BirdsiteLive.Common.Settings;
-using Newtonsoft.Json;
 
 namespace BSLManager.Tools
 {
@@ -94,7 +94,7 @@ namespace BSLManager.Tools
                 if (!File.Exists(LocalFileName)) return null;
 
                 var jsonContent = File.ReadAllText(LocalFileName);
-                var content = JsonConvert.DeserializeObject<LocalSettingsData>(jsonContent);
+                var content = JsonSerializer.Deserialize<LocalSettingsData>(jsonContent);
                 return content;
             }
             catch (Exception)
@@ -105,7 +105,7 @@ namespace BSLManager.Tools
 
         private void SaveLocalSettings(LocalSettingsData data)
         {
-            var jsonContent = JsonConvert.SerializeObject(data);
+            var jsonContent = JsonSerializer.Serialize(data);
             File.WriteAllText(LocalFileName, jsonContent);
         }
     }
diff --git a/src/BirdsiteLive.ActivityPub/ApDeserializer.cs b/src/BirdsiteLive.ActivityPub/ApDeserializer.cs
index 169bbe6..ba60c29 100644
--- a/src/BirdsiteLive.ActivityPub/ApDeserializer.cs
+++ b/src/BirdsiteLive.ActivityPub/ApDeserializer.cs
@@ -1,6 +1,7 @@
 using System;
 using BirdsiteLive.ActivityPub.Models;
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
+using System.Text.Json;
 
 namespace BirdsiteLive.ActivityPub
 {
@@ -10,22 +11,21 @@ namespace BirdsiteLive.ActivityPub
         {
             try
             {
-                var activity = JsonConvert.DeserializeObject<Activity>(json);
+                var activity = JsonSerializer.Deserialize<Activity>(json);
                 switch (activity.type)
                 {
                     case "Follow":
-                        return JsonConvert.DeserializeObject<ActivityFollow>(json);
+                        return JsonSerializer.Deserialize<ActivityFollow>(json);
                     case "Undo":
-                        var a = JsonConvert.DeserializeObject<ActivityUndo>(json);
+                        var a = JsonSerializer.Deserialize<ActivityUndo>(json);
                         if(a.apObject.type == "Follow")
-                            return JsonConvert.DeserializeObject<ActivityUndoFollow>(json);
+                            return JsonSerializer.Deserialize<ActivityUndoFollow>(json);
                         break;
                     case "Delete":
-                        return JsonConvert.DeserializeObject<ActivityDelete>(json);
+                        return JsonSerializer.Deserialize<ActivityDelete>(json);
                     case "Accept":
-                        var accept = JsonConvert.DeserializeObject<ActivityAccept>(json);
-                        //var acceptType = JsonConvert.DeserializeObject<Activity>(accept.apObject);
-                        switch ((accept.apObject as dynamic).type.ToString())
+                        var accept = JsonSerializer.Deserialize<ActivityAccept>(json);
+                        switch (accept.apObject.type)
                         {
                             case "Follow":
                                 var acceptFollow = new ActivityAcceptFollow()
@@ -34,13 +34,13 @@ namespace BirdsiteLive.ActivityPub
                                     id = accept.id,
                                     actor = accept.actor,
                                     context = accept.context,
-                                    apObject = new ActivityFollow()
+                                    apObject = new NestedActivity()
                                     {
-                                        id = (accept.apObject as dynamic).id?.ToString(),
-                                        type = (accept.apObject as dynamic).type?.ToString(),
-                                        actor = (accept.apObject as dynamic).actor?.ToString(),
-                                        context = (accept.apObject as dynamic).context?.ToString(),
-                                        apObject = (accept.apObject as dynamic).@object?.ToString()
+                                        id = accept.apObject.id,
+                                        type = accept.apObject.type,
+                                        actor = accept.apObject.actor,
+                                        context = accept.apObject.context,
+                                        apObject = accept.apObject.apObject,
                                     }
                                 };
                                 return acceptFollow;
@@ -58,7 +58,7 @@ namespace BirdsiteLive.ActivityPub
 
         private class Ac : Activity
         {
-            [JsonProperty("object")]
+            [JsonPropertyName("object")]
             public Activity apObject { get; set; }
         }
     }
diff --git a/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj b/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj
index e046b1c..ec55c83 100644
--- a/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj
+++ b/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj
@@ -6,7 +6,6 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
   </ItemGroup>
 
 </Project>
diff --git a/src/BirdsiteLive.ActivityPub/Converters/ContextArrayConverter.cs b/src/BirdsiteLive.ActivityPub/Converters/ContextArrayConverter.cs
deleted file mode 100644
index 21e139f..0000000
--- a/src/BirdsiteLive.ActivityPub/Converters/ContextArrayConverter.cs
+++ /dev/null
@@ -1,39 +0,0 @@
-using System;
-using System.Collections.Generic;
-using Newtonsoft.Json;
-
-namespace BirdsiteLive.ActivityPub.Converters
-{
-    public class ContextArrayConverter : JsonConverter
-    {
-        public override bool CanWrite { get { return false; } }
-        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
-        {
-            throw new NotImplementedException();
-        }
-
-        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
-        {
-            var result = new List<string>();
-
-            var list = serializer.Deserialize<List<object>>(reader);
-            foreach (var l in list)
-            {
-                if (l is string s)
-                    result.Add(s);
-                else
-                {
-                    var str = JsonConvert.SerializeObject(l);
-                    result.Add(str);
-                }
-            }
-
-            return result.ToArray();
-        }
-
-        public override bool CanConvert(Type objectType)
-        {
-            throw new NotImplementedException();
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/BirdsiteLive.ActivityPub/Models/Activity.cs b/src/BirdsiteLive.ActivityPub/Models/Activity.cs
index 8c78091..e6826fa 100644
--- a/src/BirdsiteLive.ActivityPub/Models/Activity.cs
+++ b/src/BirdsiteLive.ActivityPub/Models/Activity.cs
@@ -1,17 +1,16 @@
 using System.Text.Json.Serialization;
-using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class Activity
     {
-        [JsonProperty("@context")]
+        [JsonPropertyName("@context")]
         public object context { get; set; }
         public string id { get; set; }
         public string type { get; set; }
         public string actor { get; set; }
 
-        //[JsonProperty("object")]
+        //[JsonPropertyName("object")]
         //public string apObject { get; set; }
     }
 }
\ No newline at end of file
diff --git a/src/BirdsiteLive.ActivityPub/Models/ActivityAccept.cs b/src/BirdsiteLive.ActivityPub/Models/ActivityAccept.cs
index 30b8746..34891c7 100644
--- a/src/BirdsiteLive.ActivityPub/Models/ActivityAccept.cs
+++ b/src/BirdsiteLive.ActivityPub/Models/ActivityAccept.cs
@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityAccept : Activity
     {
-        [JsonProperty("object")]
-        public object apObject { get; set; }
+        [JsonPropertyName("object")]
+        public NestedActivity apObject { get; set; }
     }
 }
\ No newline at end of file
diff --git a/src/BirdsiteLive.ActivityPub/Models/ActivityAcceptFollow.cs b/src/BirdsiteLive.ActivityPub/Models/ActivityAcceptFollow.cs
index f833a43..91ec40e 100644
--- a/src/BirdsiteLive.ActivityPub/Models/ActivityAcceptFollow.cs
+++ b/src/BirdsiteLive.ActivityPub/Models/ActivityAcceptFollow.cs
@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityAcceptFollow : Activity
     {
-        [JsonProperty("object")]
-        public ActivityFollow apObject { get; set; }
+        [JsonPropertyName("object")]
+        public NestedActivity apObject { get; set; }
     }
 }
\ No newline at end of file
diff --git a/src/BirdsiteLive.ActivityPub/Models/ActivityAcceptUndoFollow.cs b/src/BirdsiteLive.ActivityPub/Models/ActivityAcceptUndoFollow.cs
index 9453f25..225e8e5 100644
--- a/src/BirdsiteLive.ActivityPub/Models/ActivityAcceptUndoFollow.cs
+++ b/src/BirdsiteLive.ActivityPub/Models/ActivityAcceptUndoFollow.cs
@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityAcceptUndoFollow : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public ActivityUndoFollow apObject { get; set; }
     }
 }
\ No newline at end of file
diff --git a/src/BirdsiteLive.ActivityPub/Models/ActivityCreateNote.cs b/src/BirdsiteLive.ActivityPub/Models/ActivityCreateNote.cs
index fe14ac7..9793317 100644
--- a/src/BirdsiteLive.ActivityPub/Models/ActivityCreateNote.cs
+++ b/src/BirdsiteLive.ActivityPub/Models/ActivityCreateNote.cs
@@ -1,6 +1,5 @@
-using System;
-using BirdsiteLive.ActivityPub.Models;
-using Newtonsoft.Json;
+using BirdsiteLive.ActivityPub.Models;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
@@ -10,7 +9,7 @@ namespace BirdsiteLive.ActivityPub
         public string[] to { get; set; }
         public string[] cc { get; set; }
 
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public Note apObject { get; set; }
     }
 }
\ No newline at end of file
diff --git a/src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs b/src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs
index deb7e7f..3c64c32 100644
--- a/src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs
+++ b/src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs
@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub.Models
 {
     public class ActivityDelete : Activity
     {
-        [JsonProperty("object")]
-        public object apObject { get; set; }
+        [JsonPropertyName("object")]
+        public string apObject { get; set; }
     }
 }
\ No newline at end of file
diff --git a/src/BirdsiteLive.ActivityPub/Models/ActivityFollow.cs b/src/BirdsiteLive.ActivityPub/Models/ActivityFollow.cs
index 26676f1..e718824 100644
--- a/src/BirdsiteLive.ActivityPub/Models/ActivityFollow.cs
+++ b/src/BirdsiteLive.ActivityPub/Models/ActivityFollow.cs
@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityFollow : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public string apObject { get; set; }
     }
 }
\ No newline at end of file
diff --git a/src/BirdsiteLive.ActivityPub/Models/ActivityRejectFollow.cs b/src/BirdsiteLive.ActivityPub/Models/ActivityRejectFollow.cs
index da5d613..a03be8f 100644
--- a/src/BirdsiteLive.ActivityPub/Models/ActivityRejectFollow.cs
+++ b/src/BirdsiteLive.ActivityPub/Models/ActivityRejectFollow.cs
@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityRejectFollow : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public ActivityFollow apObject { get; set; }
     }
 }
\ No newline at end of file
diff --git a/src/BirdsiteLive.ActivityPub/Models/ActivityUndo.cs b/src/BirdsiteLive.ActivityPub/Models/ActivityUndo.cs
index 2d98b5d..629bd98 100644
--- a/src/BirdsiteLive.ActivityPub/Models/ActivityUndo.cs
+++ b/src/BirdsiteLive.ActivityPub/Models/ActivityUndo.cs
@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityUndo : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public Activity apObject { get; set; }
     }
 }
\ No newline at end of file
diff --git a/src/BirdsiteLive.ActivityPub/Models/ActivityUndoFollow.cs b/src/BirdsiteLive.ActivityPub/Models/ActivityUndoFollow.cs
index 624988f..054cb23 100644
--- a/src/BirdsiteLive.ActivityPub/Models/ActivityUndoFollow.cs
+++ b/src/BirdsiteLive.ActivityPub/Models/ActivityUndoFollow.cs
@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityUndoFollow : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public ActivityFollow apObject { get; set; }
     }
 }
\ No newline at end of file
diff --git a/src/BirdsiteLive.ActivityPub/Models/Actor.cs b/src/BirdsiteLive.ActivityPub/Models/Actor.cs
index 713ea89..e8cc75b 100644
--- a/src/BirdsiteLive.ActivityPub/Models/Actor.cs
+++ b/src/BirdsiteLive.ActivityPub/Models/Actor.cs
@@ -1,14 +1,12 @@
 using System.Net;
 using BirdsiteLive.ActivityPub.Converters;
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class Actor
     {
-        //[JsonPropertyName("@context")]
-        [JsonProperty("@context")]
-        [JsonConverter(typeof(ContextArrayConverter))]
+        [JsonPropertyName("@context")]
         public string[] context { get; set; } = new[] { "https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1" };
         public string id { get; set; }
         public string type { get; set; }
diff --git a/src/BirdsiteLive.ActivityPub/Models/Followers.cs b/src/BirdsiteLive.ActivityPub/Models/Followers.cs
index 85c44d2..1908086 100644
--- a/src/BirdsiteLive.ActivityPub/Models/Followers.cs
+++ b/src/BirdsiteLive.ActivityPub/Models/Followers.cs
@@ -1,12 +1,11 @@
 using BirdsiteLive.ActivityPub.Converters;
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub.Models
 {
     public class Followers
     {
-        [JsonProperty("@context")]
-        [JsonConverter(typeof(ContextArrayConverter))]
+        [JsonPropertyName("@context")]
         public string context { get; set; } = "https://www.w3.org/ns/activitystreams";
 
         public string id { get; set; }
diff --git a/src/BirdsiteLive.ActivityPub/Models/NestedActivity.cs b/src/BirdsiteLive.ActivityPub/Models/NestedActivity.cs
new file mode 100644
index 0000000..f23022c
--- /dev/null
+++ b/src/BirdsiteLive.ActivityPub/Models/NestedActivity.cs
@@ -0,0 +1,16 @@
+using System.Text.Json.Serialization;
+
+namespace BirdsiteLive.ActivityPub
+{
+    public class NestedActivity
+    {
+        [JsonPropertyName("@context")]
+        public object context { get; set; }
+        public string id { get; set; }
+        public string type { get; set; }
+        public string actor { get; set; }
+
+        [JsonPropertyName("object")]
+        public string apObject { get; set; }
+    }
+}
\ No newline at end of file
diff --git a/src/BirdsiteLive.ActivityPub/Models/Note.cs b/src/BirdsiteLive.ActivityPub/Models/Note.cs
index d7e158d..a6b1a9e 100644
--- a/src/BirdsiteLive.ActivityPub/Models/Note.cs
+++ b/src/BirdsiteLive.ActivityPub/Models/Note.cs
@@ -1,12 +1,10 @@
-using BirdsiteLive.ActivityPub.Converters;
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub.Models
 {
     public class Note
     {
-        [JsonProperty("@context")]
-        [JsonConverter(typeof(ContextArrayConverter))]
+        [JsonPropertyName("@context")]
         public string[] context { get; set; } = new[] { "https://www.w3.org/ns/activitystreams" };
 
         public string id { get; set; }
diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs
index e18193e..914eae6 100644
--- a/src/BirdsiteLive.Domain/ActivityPubService.cs
+++ b/src/BirdsiteLive.Domain/ActivityPubService.cs
@@ -4,13 +4,14 @@ using System.Net;
 using System.Net.Http;
 using System.Security.Cryptography;
 using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
 using System.Threading.Tasks;
 using BirdsiteLive.ActivityPub;
 using BirdsiteLive.ActivityPub.Converters;
 using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.Common.Settings;
 using Microsoft.Extensions.Logging;
-using Newtonsoft.Json;
 
 namespace BirdsiteLive.Domain
 {
@@ -52,7 +53,7 @@ namespace BirdsiteLive.Domain
 
             var content = await result.Content.ReadAsStringAsync();
 
-            var actor = JsonConvert.DeserializeObject<Actor>(content);
+            var actor = JsonSerializer.Deserialize<Actor>(content);
             if (string.IsNullOrWhiteSpace(actor.url)) actor.url = objectId;
             return actor;
         }
@@ -78,7 +79,7 @@ namespace BirdsiteLive.Domain
             if (!string.IsNullOrWhiteSpace(inbox))
                 usedInbox = inbox;
 
-            var json = JsonConvert.SerializeObject(data, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore });
+            var json = JsonSerializer.Serialize(data, new JsonSerializerOptions() { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull });
 
             var date = DateTime.UtcNow.ToUniversalTime();
             var httpDate = date.ToString("r");
diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs
index 8131c7e..ab248e4 100644
--- a/src/BirdsiteLive.Domain/UserService.cs
+++ b/src/BirdsiteLive.Domain/UserService.cs
@@ -188,7 +188,7 @@ namespace BirdsiteLive.Domain
                 id = $"{activity.apObject}#accepts/follows/{Guid.NewGuid()}",
                 type = "Accept",
                 actor = activity.apObject,
-                apObject = new ActivityFollow()
+                apObject = new NestedActivity()
                 {
                     id = activity.id,
                     type = activity.type,
diff --git a/src/BirdsiteLive/Controllers/DebugingController.cs b/src/BirdsiteLive/Controllers/DebugingController.cs
index 00accef..ed32782 100644
--- a/src/BirdsiteLive/Controllers/DebugingController.cs
+++ b/src/BirdsiteLive/Controllers/DebugingController.cs
@@ -10,7 +10,6 @@ using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Domain;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.Mvc.ViewFeatures;
-using Newtonsoft.Json;
 
 namespace BirdsiteLive.Controllers
 {
diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs
index 5ce6179..d6b0bff 100644
--- a/src/BirdsiteLive/Controllers/UsersController.cs
+++ b/src/BirdsiteLive/Controllers/UsersController.cs
@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text.Json;
 using System.Net.Mime;
 using System.Text.RegularExpressions;
 using System.Threading;
@@ -19,7 +20,6 @@ using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Primitives;
-using Newtonsoft.Json;
 
 namespace BirdsiteLive.Controllers
 {
@@ -111,7 +111,7 @@ namespace BirdsiteLive.Controllers
                     if (isSaturated) return new ObjectResult("Too Many Requests") { StatusCode = 429 };
                     if (notFound) return NotFound();
                     var apUser = _userService.GetUser(user);
-                    var jsonApUser = JsonConvert.SerializeObject(apUser);
+                    var jsonApUser = System.Text.Json.JsonSerializer.Serialize(apUser);
                     return Content(jsonApUser, "application/activity+json; charset=utf-8");
                 }
             }
@@ -155,7 +155,7 @@ namespace BirdsiteLive.Controllers
 
                 if (r.Contains("application/activity+json"))
                 {
-                    var jsonApUser = JsonConvert.SerializeObject(status);
+                    var jsonApUser = JsonSerializer.Serialize(status);
                     return Content(jsonApUser, "application/activity+json; charset=utf-8");
                 }
             }
@@ -185,7 +185,7 @@ namespace BirdsiteLive.Controllers
 
             var status = _statusService.GetActivity(id, tweet);
 
-            var jsonApUser = JsonConvert.SerializeObject(status);
+            var jsonApUser = JsonSerializer.Serialize(status);
             return Content(jsonApUser, "application/activity+json; charset=utf-8");
         }
 
@@ -269,7 +269,7 @@ namespace BirdsiteLive.Controllers
             {
                 id = $"https://{_instanceSettings.Domain}/users/{id}/followers"
             };
-            var jsonApUser = JsonConvert.SerializeObject(followers);
+            var jsonApUser = JsonSerializer.Serialize(followers);
             return Content(jsonApUser, "application/activity+json; charset=utf-8");
         }
     }
diff --git a/src/BirdsiteLive/Properties/launchSettings.json b/src/BirdsiteLive/Properties/launchSettings.json
index 80d2896..dedaf88 100644
--- a/src/BirdsiteLive/Properties/launchSettings.json
+++ b/src/BirdsiteLive/Properties/launchSettings.json
@@ -19,7 +19,8 @@
       "commandName": "Project",
       "launchBrowser": true,
       "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development"
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "Instance__ParallelTwitterRequests": "0"
       },
       "applicationUrl": "http://localhost:5000"
     },

From 29ba6baddb00b5858298482002e2e5e8764086df Mon Sep 17 00:00:00 2001
From: Vincent Cloutier <vincent@cloutier.co>
Date: Thu, 16 Mar 2023 10:23:31 -0400
Subject: [PATCH 02/20] fix Accept Follow serialization

---
 .../ApDeserializer.cs                         |  5 --
 .../Models/Activity.cs                        |  2 +-
 .../Models/NestedActivity.cs                  |  1 +
 src/BirdsiteLive.Domain/ActivityPubService.cs | 47 ++++++++---
 src/BirdsiteLive.Domain/UserService.cs        | 15 +---
 .../ActivityTests.cs                          | 38 ++++-----
 .../ApDeserializerTests.cs                    |  7 +-
 .../ActivityServiceTests.cs                   | 80 +++++++++++++++++++
 8 files changed, 141 insertions(+), 54 deletions(-)
 create mode 100644 src/Tests/BirdsiteLive.Domain.Tests/ActivityServiceTests.cs

diff --git a/src/BirdsiteLive.ActivityPub/ApDeserializer.cs b/src/BirdsiteLive.ActivityPub/ApDeserializer.cs
index ba60c29..17fa3a0 100644
--- a/src/BirdsiteLive.ActivityPub/ApDeserializer.cs
+++ b/src/BirdsiteLive.ActivityPub/ApDeserializer.cs
@@ -56,10 +56,5 @@ namespace BirdsiteLive.ActivityPub
             return null;
         }
 
-        private class Ac : Activity
-        {
-            [JsonPropertyName("object")]
-            public Activity apObject { get; set; }
-        }
     }
 }
\ No newline at end of file
diff --git a/src/BirdsiteLive.ActivityPub/Models/Activity.cs b/src/BirdsiteLive.ActivityPub/Models/Activity.cs
index e6826fa..0e21b48 100644
--- a/src/BirdsiteLive.ActivityPub/Models/Activity.cs
+++ b/src/BirdsiteLive.ActivityPub/Models/Activity.cs
@@ -5,7 +5,7 @@ namespace BirdsiteLive.ActivityPub
     public class Activity
     {
         [JsonPropertyName("@context")]
-        public object context { get; set; }
+        public string context { get; set; }
         public string id { get; set; }
         public string type { get; set; }
         public string actor { get; set; }
diff --git a/src/BirdsiteLive.ActivityPub/Models/NestedActivity.cs b/src/BirdsiteLive.ActivityPub/Models/NestedActivity.cs
index f23022c..9117311 100644
--- a/src/BirdsiteLive.ActivityPub/Models/NestedActivity.cs
+++ b/src/BirdsiteLive.ActivityPub/Models/NestedActivity.cs
@@ -5,6 +5,7 @@ namespace BirdsiteLive.ActivityPub
     public class NestedActivity
     {
         [JsonPropertyName("@context")]
+        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
         public object context { get; set; }
         public string id { get; set; }
         public string type { get; set; }
diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs
index 914eae6..3a3137c 100644
--- a/src/BirdsiteLive.Domain/ActivityPubService.cs
+++ b/src/BirdsiteLive.Domain/ActivityPubService.cs
@@ -21,6 +21,8 @@ namespace BirdsiteLive.Domain
         Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null);
         Task PostNewActivity(ActivityCreateNote note, string username, string noteId, string targetHost,
             string targetInbox);
+
+        ActivityAcceptFollow BuildAcceptFollow(ActivityFollow activity);
     }
 
     public class ActivityPubService : IActivityPubService
@@ -73,13 +75,33 @@ namespace BirdsiteLive.Domain
             }
         }
 
-        public async Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null)
+        public ActivityAcceptFollow BuildAcceptFollow(ActivityFollow activity)
+        {
+            var acceptFollow = new ActivityAcceptFollow()
+            {
+                context = "https://www.w3.org/ns/activitystreams",
+                id = $"{activity.apObject}#accepts/follows/{Guid.NewGuid()}",
+                type = "Accept",
+                actor = activity.apObject,
+                apObject = new NestedActivity()
+                {
+                    id = activity.id,
+                    type = activity.type,
+                    actor = activity.actor,
+                    apObject = activity.apObject
+                }
+            };
+            return acceptFollow;
+        }
+        public HttpRequestMessage BuildRequest<T>(T data, string targetHost, string actorUrl,
+            string inbox = null)
         {
             var usedInbox = $"/inbox";
             if (!string.IsNullOrWhiteSpace(inbox))
                 usedInbox = inbox;
 
-            var json = JsonSerializer.Serialize(data, new JsonSerializerOptions() { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull });
+            var json = JsonSerializer.Serialize(data,
+                new JsonSerializerOptions() { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull });
 
             var date = DateTime.UtcNow.ToUniversalTime();
             var httpDate = date.ToString("r");
@@ -88,26 +110,33 @@ namespace BirdsiteLive.Domain
 
             var signature = _cryptoService.SignAndGetSignatureHeader(date, actorUrl, targetHost, digest, usedInbox);
 
-            var client = _httpClientFactory.CreateClient();
-            client.Timeout = TimeSpan.FromSeconds(2);
             var httpRequestMessage = new HttpRequestMessage
             {
                 Method = HttpMethod.Post,
                 RequestUri = new Uri($"https://{targetHost}{usedInbox}"),
                 Headers =
                 {
-                    {"Host", targetHost},
-                    {"Date", httpDate},
-                    {"Signature", signature},
-                    {"Digest", $"SHA-256={digest}"}
+                    { "Host", targetHost },
+                    { "Date", httpDate },
+                    { "Signature", signature },
+                    { "Digest", $"SHA-256={digest}" }
                 },
                 Content = new StringContent(json, Encoding.UTF8, "application/ld+json")
             };
 
+            return httpRequestMessage;
+        }
+
+        public async Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null)
+        {
+            var httpRequestMessage = BuildRequest(data, targetHost, actorUrl, inbox);
+
+            var client = _httpClientFactory.CreateClient();
+            client.Timeout = TimeSpan.FromSeconds(2);
+            
             var response = await client.SendAsync(httpRequestMessage);
             response.EnsureSuccessStatusCode();
             _logger.LogInformation("Sent tweet to " + targetHost);
-            _logger.LogInformation("Tweet content is " + json);
 
             return response.StatusCode;
         }
diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs
index ab248e4..2b38d90 100644
--- a/src/BirdsiteLive.Domain/UserService.cs
+++ b/src/BirdsiteLive.Domain/UserService.cs
@@ -182,20 +182,7 @@ namespace BirdsiteLive.Domain
         
         private async Task<bool> SendAcceptFollowAsync(ActivityFollow activity, string followerHost)
         {
-            var acceptFollow = new ActivityAcceptFollow()
-            {
-                context = "https://www.w3.org/ns/activitystreams",
-                id = $"{activity.apObject}#accepts/follows/{Guid.NewGuid()}",
-                type = "Accept",
-                actor = activity.apObject,
-                apObject = new NestedActivity()
-                {
-                    id = activity.id,
-                    type = activity.type,
-                    actor = activity.actor,
-                    apObject = activity.apObject
-                }
-            };
+            var acceptFollow = _activityPubService.BuildAcceptFollow(activity);
             var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject);
             return result == HttpStatusCode.Accepted ||
                    result == HttpStatusCode.OK; //TODO: revamp this for better error handling
diff --git a/src/Tests/BirdsiteLive.ActivityPub.Tests/ActivityTests.cs b/src/Tests/BirdsiteLive.ActivityPub.Tests/ActivityTests.cs
index 1687ac4..6dba0de 100644
--- a/src/Tests/BirdsiteLive.ActivityPub.Tests/ActivityTests.cs
+++ b/src/Tests/BirdsiteLive.ActivityPub.Tests/ActivityTests.cs
@@ -3,31 +3,21 @@ using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub.Tests
 {
-    //[TestClass]
-    //public class ActivityTests
-    //{
-    //    [TestMethod]
-    //    public void FollowDeserializationTest()
-    //    {
-    //        var json = "{ \"@context\":\"https://www.w3.org/ns/activitystreams\",\"id\":\"https://mastodon.technology/c94567cf-1fda-42ba-82fc-a0f82f63ccbe\",\"type\":\"Follow\",\"actor\":\"https://mastodon.technology/users/testtest\",\"object\":\"https://4a120ca2680e.ngrok.io/users/manu\"}";
+    [TestClass]
+    public class ActivityTests
+    {
+        [TestMethod]
+        public void Serialize()
+        {
+            var obj = new Actor
+            {
+                type = "Person",
+                id = "id"
+            };
 
-    //        var data = JsonConvert.DeserializeObject<Activity>(json);
+            var json = JsonConvert.SerializeObject(obj);
 
-    //        Assert.AreEqual("https://mastodon.technology/c94567cf-1fda-42ba-82fc-a0f82f63ccbe", data.id);
-    //        Assert.AreEqual("Follow", data.type);
-    //        Assert.AreEqual("https://4a120ca2680e.ngrok.io/users/manu", data.apObject);
-    //    }
 
-    //    [TestMethod]
-    //    public void UndoDeserializationTest()
-    //    {
-    //        var json =
-    //            "{\"@context\":\"https://www.w3.org/ns/activitystreams\",\"id\":\"https://mastodon.technology/users/testtest#follows/225982/undo\",\"type\":\"Undo\",\"actor\":\"https://mastodon.technology/users/testtest\",\"object\":{\"id\":\"https://mastodon.technology/c94567cf-1fda-42ba-82fc-a0f82f63ccbe\",\"type\":\"Follow\",\"actor\":\"https://mastodon.technology/users/testtest\",\"object\":\"https://4a120ca2680e.ngrok.io/users/manu\"}}";
-
-    //        var data = JsonConvert.DeserializeObject<Activity>(json);
-    //        Assert.AreEqual("https://mastodon.technology/users/testtest#follows/225982/undo", data.id);
-    //        Assert.AreEqual("Undo", data.type);
-    //        Assert.AreEqual("https://4a120ca2680e.ngrok.io/users/manu", data.apObject);
-    //    }
-    //}
+        }
+    }
 }
diff --git a/src/Tests/BirdsiteLive.ActivityPub.Tests/ApDeserializerTests.cs b/src/Tests/BirdsiteLive.ActivityPub.Tests/ApDeserializerTests.cs
index 3d64e90..f583d7f 100644
--- a/src/Tests/BirdsiteLive.ActivityPub.Tests/ApDeserializerTests.cs
+++ b/src/Tests/BirdsiteLive.ActivityPub.Tests/ApDeserializerTests.cs
@@ -1,6 +1,6 @@
 using BirdsiteLive.ActivityPub.Models;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
-using Newtonsoft.Json;
+using System.Text.Json;
 
 namespace BirdsiteLive.ActivityPub.Tests
 {
@@ -28,9 +28,11 @@ namespace BirdsiteLive.ActivityPub.Tests
             var data = ApDeserializer.ProcessActivity(json) as ActivityUndoFollow;
             Assert.AreEqual("https://mastodon.technology/users/testtest#follows/225982/undo", data.id);
             Assert.AreEqual("Undo", data.type);
+            Assert.AreEqual("https://www.w3.org/ns/activitystreams", data.context);
             Assert.AreEqual("Follow", data.apObject.type);
             Assert.AreEqual("https://mastodon.technology/users/testtest", data.apObject.actor);
             Assert.AreEqual("https://4a120ca2680e.ngrok.io/users/manu", data.apObject.apObject);
+            Assert.AreEqual(null, data.apObject.context);
         }
 
         [TestMethod]
@@ -62,6 +64,9 @@ namespace BirdsiteLive.ActivityPub.Tests
             Assert.AreEqual("https://mastodon.technology/users/deleteduser", data.actor);
             Assert.AreEqual("https://mastodon.technology/users/deleteduser", data.apObject);
         }
+        // {"object":{"object":"https://bird.makeup/users/spectatorindex","id":"https://masto.ai/b89eb86e-c902-48bc-956f-94f081617f18","type":"Follow","actor":"https://masto.ai/users/singha"},"@context":"https://www.w3.org/ns/activitystreams","id":"https://bird.makeup/users/spectatorindex#accepts/follows/27363118-e61e-4710-a41c-75dd5d54912f","type":"Accept","actor":"https://bird.makeup/users/spectatorindex"}
+        // {"object":{"object":"https://bird.makeup/users/moltke","id":"https://universeodon.com/81cddd78-d7d6-4665-aa21-7bcfbea82b6b","type":"Follow","actor":"https://universeodon.com/users/amhrasmussen"},"@context":"https://www.w3.org/ns/activitystreams","id":"https://bird.makeup/users/moltke#accepts/follows/d28146be-e884-4e91-8385-19fa004f35b3","type":"Accept","actor":"https://bird.makeup/users/moltke"}
+
 
         //[TestMethod]
         //public void NoteDeserializationTest()
diff --git a/src/Tests/BirdsiteLive.Domain.Tests/ActivityServiceTests.cs b/src/Tests/BirdsiteLive.Domain.Tests/ActivityServiceTests.cs
new file mode 100644
index 0000000..20525f6
--- /dev/null
+++ b/src/Tests/BirdsiteLive.Domain.Tests/ActivityServiceTests.cs
@@ -0,0 +1,80 @@
+using System.Net.Http;
+using System.Threading.Tasks;
+using BirdsiteLive.ActivityPub;
+using BirdsiteLive.Common.Settings;
+using BirdsiteLive.Domain.Factories;
+using Microsoft.Extensions.Logging;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Moq;
+
+
+namespace BirdsiteLive.Domain.Tests
+{
+    [TestClass]
+    public class ActivityServiceTests
+    {
+        private readonly InstanceSettings _settings;
+
+        #region Ctor
+        public ActivityServiceTests()
+        {
+            _settings = new InstanceSettings
+            {
+                Domain = "domain.name"
+            };
+        }
+        #endregion
+
+        [TestMethod]
+        public async Task ActivityTest()
+        {
+            var logger1 = new Mock<ILogger<ActivityPubService>>();
+            var httpFactory = new Mock<IHttpClientFactory>();
+            var keyFactory = new Mock<MagicKeyFactory>();
+            var cryptoService = new CryptoService(keyFactory.Object);
+            httpFactory.Setup(_ => _.CreateClient(string.Empty)).Returns(new HttpClient());
+            var service = new ActivityPubService(cryptoService, _settings, httpFactory.Object, logger1.Object);
+
+            var activity = new ActivityAcceptFollow()
+            {
+                id = "awef",
+            };
+            var json = "{\"id\":\"awef\"}";
+            #region Validations
+
+            var req = service.BuildRequest(activity, "google.com", "tata", "awef");
+            
+            Assert.AreEqual(await req.Content.ReadAsStringAsync(), json);
+
+            #endregion
+        }
+        [TestMethod]
+        public async Task AcceptFollow()
+        {
+ 
+
+            var logger1 = new Mock<ILogger<ActivityPubService>>();
+            var httpFactory = new Mock<IHttpClientFactory>();
+            var keyFactory = new Mock<MagicKeyFactory>();
+            var cryptoService = new CryptoService(keyFactory.Object);
+            httpFactory.Setup(_ => _.CreateClient(string.Empty)).Returns(new HttpClient());
+            var service = new ActivityPubService(cryptoService, _settings, httpFactory.Object, logger1.Object);
+
+            var json = "{ \"@context\":\"https://www.w3.org/ns/activitystreams\",\"id\":\"https://mastodon.technology/c94567cf-1fda-42ba-82fc-a0f82f63ccbe\",\"type\":\"Follow\",\"actor\":\"https://mastodon.technology/users/testtest\",\"object\":\"https://4a120ca2680e.ngrok.io/users/manu\"}";
+            var activity = ApDeserializer.ProcessActivity(json) as ActivityFollow;
+
+            var jsonres =
+                "{\"object\":{\"id\":\"https://mastodon.technology/c94567cf-1fda-42ba-82fc-a0f82f63ccbe\",\"type\":\"Follow\",\"actor\":\"https://mastodon.technology/users/testtest\",\"object\":\"https://4a120ca2680e.ngrok.io/users/manu\"},\"@context\":\"https://www.w3.org/ns/activitystreams\",\"id\":\"https://4a120ca2680e.ngrok.io/users/manu#accepts/follows/32e5fbda-9159-4ede-8249-9d008092d26f\",\"type\":\"Accept\",\"actor\":\"https://4a120ca2680e.ngrok.io/users/manu\"}";
+            var activityRes = ApDeserializer.ProcessActivity(jsonres) as ActivityAcceptFollow;
+            #region Validations
+
+            var req = service.BuildAcceptFollow(activity);
+            
+            Assert.AreEqual(req.actor, activityRes.actor);
+            Assert.AreEqual(req.context, activityRes.context);
+
+            #endregion
+        }
+
+    }
+}

From 23935635740e31c63e1b9a9cd0edcb19fce73c2f Mon Sep 17 00:00:00 2001
From: Vincent Cloutier <vincent@cloutier.co>
Date: Thu, 16 Mar 2023 11:46:05 -0400
Subject: [PATCH 03/20] conversion to System.Text.Json part 2

---
 src/BirdsiteLive.ActivityPub/ApDeserializer.cs |  5 +++--
 .../Models/ActivityAcceptFollow.cs             |  2 +-
 src/BirdsiteLive.Domain/ActivityPubService.cs  |  5 ++---
 src/BirdsiteLive.Domain/UserService.cs         |  9 +++++----
 .../TwitterTweetsService.cs                    | 13 ++++++-------
 .../ActivityServiceTests.cs                    | 18 +++++++++++++++++-
 .../BirdsiteLive.Domain.Tests.csproj           |  2 +-
 .../TimelineTests.cs                           |  7 ++++++-
 .../BirdsiteLive.Twitter.Tests/TweetTests.cs   |  6 +++++-
 9 files changed, 46 insertions(+), 21 deletions(-)

diff --git a/src/BirdsiteLive.ActivityPub/ApDeserializer.cs b/src/BirdsiteLive.ActivityPub/ApDeserializer.cs
index 17fa3a0..df385ef 100644
--- a/src/BirdsiteLive.ActivityPub/ApDeserializer.cs
+++ b/src/BirdsiteLive.ActivityPub/ApDeserializer.cs
@@ -34,12 +34,13 @@ namespace BirdsiteLive.ActivityPub
                                     id = accept.id,
                                     actor = accept.actor,
                                     context = accept.context,
-                                    apObject = new NestedActivity()
+                                    apObject = new ActivityFollow()
                                     {
+
                                         id = accept.apObject.id,
                                         type = accept.apObject.type,
                                         actor = accept.apObject.actor,
-                                        context = accept.apObject.context,
+                                        context = accept.apObject.context?.ToString(),
                                         apObject = accept.apObject.apObject,
                                     }
                                 };
diff --git a/src/BirdsiteLive.ActivityPub/Models/ActivityAcceptFollow.cs b/src/BirdsiteLive.ActivityPub/Models/ActivityAcceptFollow.cs
index 91ec40e..8cba0cc 100644
--- a/src/BirdsiteLive.ActivityPub/Models/ActivityAcceptFollow.cs
+++ b/src/BirdsiteLive.ActivityPub/Models/ActivityAcceptFollow.cs
@@ -5,6 +5,6 @@ namespace BirdsiteLive.ActivityPub
     public class ActivityAcceptFollow : Activity
     {
         [JsonPropertyName("object")]
-        public NestedActivity apObject { get; set; }
+        public ActivityFollow apObject { get; set; }
     }
 }
\ No newline at end of file
diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs
index 3a3137c..470be70 100644
--- a/src/BirdsiteLive.Domain/ActivityPubService.cs
+++ b/src/BirdsiteLive.Domain/ActivityPubService.cs
@@ -83,7 +83,7 @@ namespace BirdsiteLive.Domain
                 id = $"{activity.apObject}#accepts/follows/{Guid.NewGuid()}",
                 type = "Accept",
                 actor = activity.apObject,
-                apObject = new NestedActivity()
+                apObject = new ActivityFollow()
                 {
                     id = activity.id,
                     type = activity.type,
@@ -100,8 +100,7 @@ namespace BirdsiteLive.Domain
             if (!string.IsNullOrWhiteSpace(inbox))
                 usedInbox = inbox;
 
-            var json = JsonSerializer.Serialize(data,
-                new JsonSerializerOptions() { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull });
+            var json = JsonSerializer.Serialize(data);
 
             var date = DateTime.UtcNow.ToUniversalTime();
             var httpDate = date.ToString("r");
diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs
index 2b38d90..ffd0b81 100644
--- a/src/BirdsiteLive.Domain/UserService.cs
+++ b/src/BirdsiteLive.Domain/UserService.cs
@@ -243,10 +243,11 @@ namespace BirdsiteLive.Domain
                 actor = activity.apObject.apObject,
                 apObject = new ActivityUndoFollow()
                 {
-                    id = activity.id,
-                    type = activity.type,
-                    actor = activity.actor,
-                    apObject = activity.apObject
+                    id = (activity.apObject as dynamic).id?.ToString(),
+                    type = (activity.apObject as dynamic).type?.ToString(),
+                    actor = (activity.apObject as dynamic).actor?.ToString(),
+                    context = (activity.apObject as dynamic).context?.ToString(),
+                    apObject = (activity.apObject as dynamic).@object?.ToString()
                 }
             };
             var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject.apObject);
diff --git a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs
index e450dfa..5811c74 100644
--- a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs
+++ b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs
@@ -55,16 +55,14 @@ namespace BirdsiteLive.Twitter
                 "https://api.twitter.com/graphql/XjlydVWHFIDaAUny86oh2g/TweetDetail?variables=%7B%22focalTweetId%22%3A%22"
                 + statusId +
                 "%22,%22with_rux_injections%22%3Atrue,%22includePromotedContent%22%3Afalse,%22withCommunity%22%3Afalse,%22withQuickPromoteEligibilityTweetFields%22%3Afalse,%22withBirdwatchNotes%22%3Afalse,%22withSuperFollowsUserFields%22%3Afalse,%22withDownvotePerspective%22%3Afalse,%22withReactionsMetadata%22%3Afalse,%22withReactionsPerspective%22%3Afalse,%22withSuperFollowsTweetFields%22%3Afalse,%22withVoice%22%3Atrue,%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue,%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue,%22verified_phone_label_enabled%22%3Afalse,%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue,%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse,%22tweetypie_unmention_optimization_enabled%22%3Atrue,%22vibe_api_enabled%22%3Atrue,%22responsive_web_edit_tweet_api_enabled%22%3Atrue,%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Afalse,%22view_counts_everywhere_api_enabled%22%3Atrue,%22longform_notetweets_consumption_enabled%22%3Atrue,%22tweet_awards_web_tipping_enabled%22%3Afalse,%22freedom_of_speech_not_reach_fetch_enabled%22%3Afalse,%22standardized_nudges_misinfo%22%3Atrue,%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse,%22interactive_text_enabled%22%3Atrue,%22responsive_web_text_conversations_enabled%22%3Afalse,%22longform_notetweets_richtext_consumption_enabled%22%3Afalse,%22responsive_web_enhance_cards_enabled%22%3Atrue%7D";
+            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL);
             try
             {
                 JsonDocument tweet;
-                using (var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL))
-                {
-                    var httpResponse = await client.SendAsync(request);
-                    httpResponse.EnsureSuccessStatusCode();
-                    var c = await httpResponse.Content.ReadAsStringAsync();
-                    tweet = JsonDocument.Parse(c);
-                }
+                var httpResponse = await client.SendAsync(request);
+                httpResponse.EnsureSuccessStatusCode();
+                var c = await httpResponse.Content.ReadAsStringAsync();
+                tweet = JsonDocument.Parse(c);
 
 
                 var timeline = tweet.RootElement.GetProperty("data").GetProperty("threaded_conversation_with_injections_v2")
@@ -77,6 +75,7 @@ namespace BirdsiteLive.Twitter
             catch (Exception e)
             {
                 _logger.LogError(e, "Error retrieving tweet {TweetId}", statusId);
+                await _twitterAuthenticationInitializer.RefreshClient(request);
                 return null;
             }
         }
diff --git a/src/Tests/BirdsiteLive.Domain.Tests/ActivityServiceTests.cs b/src/Tests/BirdsiteLive.Domain.Tests/ActivityServiceTests.cs
index 20525f6..5fed8b0 100644
--- a/src/Tests/BirdsiteLive.Domain.Tests/ActivityServiceTests.cs
+++ b/src/Tests/BirdsiteLive.Domain.Tests/ActivityServiceTests.cs
@@ -6,6 +6,7 @@ using BirdsiteLive.Domain.Factories;
 using Microsoft.Extensions.Logging;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Moq;
+using System.Text.Json;
 
 
 namespace BirdsiteLive.Domain.Tests
@@ -38,8 +39,21 @@ namespace BirdsiteLive.Domain.Tests
             var activity = new ActivityAcceptFollow()
             {
                 id = "awef",
+                context = "https://www.w3.org/ns/activitystreams",
+                type = "Accept",
+                actor = "https://mastodon.technology/users/testtest",
+                apObject = new ActivityFollow()
+                {
+                    context = "https://www.w3.org/ns/activitystreams",
+                    id = "abc",
+                    type = "Follow",
+                    actor = "https://mastodon.technology/users/testtest2",
+                    apObject = "https://mastodon.technology/users/testtest3",
+                }
+                
             };
-            var json = "{\"id\":\"awef\"}";
+            var json =
+                """{"object":{"object":"https://mastodon.technology/users/testtest3","@context":"https://www.w3.org/ns/activitystreams","id":"abc","type":"Follow","actor":"https://mastodon.technology/users/testtest2"},"@context":"https://www.w3.org/ns/activitystreams","id":"awef","type":"Accept","actor":"https://mastodon.technology/users/testtest"}""";
             #region Validations
 
             var req = service.BuildRequest(activity, "google.com", "tata", "awef");
@@ -70,6 +84,8 @@ namespace BirdsiteLive.Domain.Tests
 
             var req = service.BuildAcceptFollow(activity);
             
+            string s = JsonSerializer.Serialize(req);
+            
             Assert.AreEqual(req.actor, activityRes.actor);
             Assert.AreEqual(req.context, activityRes.context);
 
diff --git a/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj b/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj
index e00e2f6..fd8c60d 100644
--- a/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj
+++ b/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>net6</TargetFramework>
-
+    <LangVersion>11</LangVersion>
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
diff --git a/src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs b/src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs
index ddaac8a..ef26e2e 100644
--- a/src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs
+++ b/src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs
@@ -41,6 +41,11 @@ namespace BirdsiteLive.ActivityPub.Tests
             ITwitterUserService user = new TwitterUserService(auth, stats.Object, logger2.Object);
             ICachedTwitterUserService user2 = new CachedTwitterUserService(user, settings);
             _tweetService = new TwitterTweetsService(auth, stats.Object, user2, twitterDal.Object, settings, logger3.Object);
+            
+            await Task.Delay(1000);
+            await auth.MakeHttpClient();
+            await Task.Delay(1000);
+            await auth.MakeHttpClient();
         }
 
         [TestMethod]
@@ -48,7 +53,7 @@ namespace BirdsiteLive.ActivityPub.Tests
         {
             var tweets = await _tweetService.GetTimelineAsync("kobebryant", 1218020971346444288);
             Assert.AreEqual(tweets[0].MessageContent, "Continuing to move the game forward @KingJames. Much respect my brother 💪🏾 #33644");
-            Assert.AreEqual(tweets.Length, 8);
+            Assert.IsTrue(tweets.Length > 5);
         }
 
         [TestMethod]
diff --git a/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs b/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs
index 480ee41..ed245bc 100644
--- a/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs
+++ b/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs
@@ -1,4 +1,6 @@
-using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+using System.IO;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Microsoft.Extensions.Logging;
 using System.Threading.Tasks;
 using BirdsiteLive.Twitter;
@@ -18,6 +20,7 @@ namespace BirdsiteLive.ActivityPub.Tests
         [TestInitialize]
         public async Task TestInit()
         {
+            
             var logger1 = new Mock<ILogger<TwitterAuthenticationInitializer>>(MockBehavior.Strict);
             var logger2 = new Mock<ILogger<TwitterUserService>>(MockBehavior.Strict);
             var logger3 = new Mock<ILogger<TwitterTweetsService>>();
@@ -33,6 +36,7 @@ namespace BirdsiteLive.ActivityPub.Tests
             ITwitterUserService user = new TwitterUserService(auth, stats.Object, logger2.Object);
             ICachedTwitterUserService user2 = new CachedTwitterUserService(user, settings);
             _tweetService = new TwitterTweetsService(auth, stats.Object, user2, twitterDal.Object, settings, logger3.Object);
+
         }
 
         [TestMethod]

From db9477bebc2a7b7de97daf0639f90cc153be76dc Mon Sep 17 00:00:00 2001
From: Vincent Cloutier <vincent@cloutier.co>
Date: Fri, 17 Mar 2023 16:10:37 -0400
Subject: [PATCH 04/20] add actor test

---
 src/BirdsiteLive.ActivityPub/Models/Actor.cs  |  2 +-
 .../ApDeserializerTests.cs                    | 97 +++++++++++++++++--
 .../BirdsiteLive.ActivityPub.Tests.csproj     |  2 +
 3 files changed, 93 insertions(+), 8 deletions(-)

diff --git a/src/BirdsiteLive.ActivityPub/Models/Actor.cs b/src/BirdsiteLive.ActivityPub/Models/Actor.cs
index e8cc75b..6f7e2ac 100644
--- a/src/BirdsiteLive.ActivityPub/Models/Actor.cs
+++ b/src/BirdsiteLive.ActivityPub/Models/Actor.cs
@@ -7,7 +7,7 @@ namespace BirdsiteLive.ActivityPub
     public class Actor
     {
         [JsonPropertyName("@context")]
-        public string[] context { get; set; } = new[] { "https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1" };
+        public object[] context { get; set; } = new string[] { "https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1" };
         public string id { get; set; }
         public string type { get; set; }
         public string followers { get; set; }
diff --git a/src/Tests/BirdsiteLive.ActivityPub.Tests/ApDeserializerTests.cs b/src/Tests/BirdsiteLive.ActivityPub.Tests/ApDeserializerTests.cs
index f583d7f..8081a33 100644
--- a/src/Tests/BirdsiteLive.ActivityPub.Tests/ApDeserializerTests.cs
+++ b/src/Tests/BirdsiteLive.ActivityPub.Tests/ApDeserializerTests.cs
@@ -68,13 +68,96 @@ namespace BirdsiteLive.ActivityPub.Tests
         // {"object":{"object":"https://bird.makeup/users/moltke","id":"https://universeodon.com/81cddd78-d7d6-4665-aa21-7bcfbea82b6b","type":"Follow","actor":"https://universeodon.com/users/amhrasmussen"},"@context":"https://www.w3.org/ns/activitystreams","id":"https://bird.makeup/users/moltke#accepts/follows/d28146be-e884-4e91-8385-19fa004f35b3","type":"Accept","actor":"https://bird.makeup/users/moltke"}
 
 
-        //[TestMethod]
-        //public void NoteDeserializationTest()
-        //{
-        //    var json =
-        //        "{\"@context\":[\"https://www.w3.org/ns/activitystreams\",{\"ostatus\":\"http://ostatus.org#\",\"atomUri\":\"ostatus:atomUri\",\"inReplyToAtomUri\":\"ostatus:inReplyToAtomUri\",\"conversation\":\"ostatus:conversation\",\"sensitive\":\"as:sensitive\",\"toot\":\"http://joinmastodon.org/ns#\",\"votersCount\":\"toot:votersCount\"}],\"id\":\"https://mastodon.technology/users/testtest/statuses/104424839893177182/activity\",\"type\":\"Create\",\"actor\":\"https://mastodon.technology/users/testtest\",\"published\":\"2020-06-29T02:10:04Z\",\"to\":[\"https://mastodon.technology/users/testtest/followers\"],\"cc\":[],\"object\":{\"id\":\"https://mastodon.technology/users/testtest/statuses/104424839893177182\",\"type\":\"Note\",\"summary\":null,\"inReplyTo\":null,\"published\":\"2020-06-29T02:10:04Z\",\"url\":\"https://mastodon.technology/@testtest/104424839893177182\",\"attributedTo\":\"https://mastodon.technology/users/testtest\",\"to\":[\"https://mastodon.technology/users/testtest/followers\"],\"cc\":[],\"sensitive\":false,\"atomUri\":\"https://mastodon.technology/users/testtest/statuses/104424839893177182\",\"inReplyToAtomUri\":null,\"conversation\":\"tag:mastodon.technology,2020-06-29:objectId=34900058:objectType=Conversation\",\"content\":\"<p>test</p>\",\"contentMap\":{\"en\":\"<p>test</p>\"},\"attachment\":[],\"tag\":[],\"replies\":{\"id\":\"https://mastodon.technology/users/testtest/statuses/104424839893177182/replies\",\"type\":\"Collection\",\"first\":{\"type\":\"CollectionPage\",\"next\":\"https://mastodon.technology/users/testtest/statuses/104424839893177182/replies?only_other_accounts=true&page=true\",\"partOf\":\"https://mastodon.technology/users/testtest/statuses/104424839893177182/replies\",\"items\":[]}}}}";
+        [TestMethod]
+        public void ActorDeserializationTest()
+        {
+            var json = """
+					{
+						"@context": [
+							"https://www.w3.org/ns/activitystreams",
+							"https://w3id.org/security/v1",
+							{
+								"manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+								"toot": "http://joinmastodon.org/ns#",
+								"featured": {
+									"@id": "toot:featured",
+									"@type": "@id"
+								},
+								"featuredTags": {
+									"@id": "toot:featuredTags",
+									"@type": "@id"
+								},
+								"alsoKnownAs": {
+									"@id": "as:alsoKnownAs",
+									"@type": "@id"
+								},
+								"movedTo": {
+									"@id": "as:movedTo",
+									"@type": "@id"
+								},
+								"schema": "http://schema.org#",
+								"PropertyValue": "schema:PropertyValue",
+								"value": "schema:value",
+								"discoverable": "toot:discoverable",
+								"Device": "toot:Device",
+								"Ed25519Signature": "toot:Ed25519Signature",
+								"Ed25519Key": "toot:Ed25519Key",
+								"Curve25519Key": "toot:Curve25519Key",
+								"EncryptedMessage": "toot:EncryptedMessage",
+								"publicKeyBase64": "toot:publicKeyBase64",
+								"deviceId": "toot:deviceId",
+								"claim": {
+									"@type": "@id",
+									"@id": "toot:claim"
+								},
+								"fingerprintKey": {
+									"@type": "@id",
+									"@id": "toot:fingerprintKey"
+								},
+								"identityKey": {
+									"@type": "@id",
+									"@id": "toot:identityKey"
+								},
+								"devices": {
+									"@type": "@id",
+									"@id": "toot:devices"
+								},
+								"messageFranking": "toot:messageFranking",
+								"messageType": "toot:messageType",
+								"cipherText": "toot:cipherText",
+								"suspended": "toot:suspended"
+							}
+						],
+						"id": "https://mastodon.online/users/devvincent",
+						"type": "Person",
+						"following": "https://mastodon.online/users/devvincent/following",
+						"followers": "https://mastodon.online/users/devvincent/followers",
+						"inbox": "https://mastodon.online/users/devvincent/inbox",
+						"outbox": "https://mastodon.online/users/devvincent/outbox",
+						"featured": "https://mastodon.online/users/devvincent/collections/featured",
+						"featuredTags": "https://mastodon.online/users/devvincent/collections/tags",
+						"preferredUsername": "devvincent",
+						"name": "",
+						"summary": "",
+						"url": "https://mastodon.online/@devvincent",
+						"manuallyApprovesFollowers": false,
+						"discoverable": false,
+						"published": "2022-05-08T00:00:00Z",
+						"devices": "https://mastodon.online/users/devvincent/collections/devices",
+						"publicKey": {
+							"id": "https://mastodon.online/users/devvincent#main-key",
+							"owner": "https://mastodon.online/users/devvincent",
+							"publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA7U07uS4zu5jeZSBVZ072\naXcTeVQc0baM8BBUvJkpX+mV2vh+V4yfqN44KzFxlkk8XcAoidt8HBAvpQ/5yCwZ\neGS2ySCxC+sqvErIbaYadWVHGJhZjLYPVa0n8wvkqRQ0aUJ8K17/wY+/YYfukgeC\nTGHGoyzDDZZxrR1Z8LTvImSEkYooTvvzaaFaTUnFwCKepxftKLdJAfp4sP4l1Zom\nUZGwaYimuJmN1bfhet/2v0S7M7/XPlmVRpfUluE2vYE0RtJt3BVDZfoWEGJPk9us\nN/JHu6UBUh6UM6ASFy5MlDLh36OxyO9sVx1WgQlNDmu2qcGUIkIgqTKppDCIP3Xk\nVQIDAQAB\n-----END PUBLIC KEY-----\n"
+						},
+						"tag": [],
+						"attachment": [],
+						"endpoints": {
+							"sharedInbox": "https://mastodon.online/inbox"
+						}
+					}
+				""";
 
-        //    var data = ApDeserializer.ProcessActivity(json) as ActivityAcceptFollow;
-        //}
+            var actor = JsonSerializer.Deserialize<Actor>(json);
+        }
     }
 }
\ No newline at end of file
diff --git a/src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj b/src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj
index 970865c..bf7140a 100644
--- a/src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj
+++ b/src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj
@@ -4,6 +4,8 @@
     <TargetFramework>net6</TargetFramework>
 
     <IsPackable>false</IsPackable>
+
+    <LangVersion>11</LangVersion>
   </PropertyGroup>
 
   <ItemGroup>

From 8d6851c6391d420ef7bb1c88887116b45513a0ca Mon Sep 17 00:00:00 2001
From: Vincent Cloutier <vincent@cloutier.co>
Date: Sat, 25 Mar 2023 13:24:11 -0400
Subject: [PATCH 05/20] moved followers retrieval

---
 .../Processors/RetrieveFollowersProcessor.cs  | 24 +++++++++----------
 .../SendTweetsToFollowersProcessor.cs         |  6 ++++-
 .../RetrieveFollowersProcessorTests.cs        |  1 +
 3 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs
index 65357a9..33dc353 100644
--- a/src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs
+++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs
@@ -21,18 +21,18 @@ namespace BirdsiteLive.Pipeline.Processors
 
         public async Task<IEnumerable<UserWithDataToSync>> ProcessAsync(UserWithDataToSync[] userWithTweetsToSyncs, CancellationToken ct)
         {
-            List<Task> todo = new List<Task>();
-            foreach (var user in userWithTweetsToSyncs)
-            {
-                var t = Task.Run( 
-                async() => {
-                    var followers = await _followersDal.GetFollowersAsync(user.User.Id);
-                    user.Followers = followers;
-                });
-                todo.Add(t);
-            }
-            
-            await Task.WhenAll(todo);
+            //List<Task> todo = new List<Task>();
+            //foreach (var user in userWithTweetsToSyncs)
+            //{
+            //    var t = Task.Run( 
+            //    async() => {
+            //        var followers = await _followersDal.GetFollowersAsync(user.User.Id);
+            //        user.Followers = followers;
+            //    });
+            //    todo.Add(t);
+            //}
+            //
+            //await Task.WhenAll(todo);
 
             return userWithTweetsToSyncs;
         }
diff --git a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs
index a4fc7b9..630d158 100644
--- a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs
+++ b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs
@@ -49,6 +49,10 @@ namespace BirdsiteLive.Pipeline.Processors
             
             var t = Task.Run( async () => 
             {
+                if (userWithTweetsToSync.Followers is null || userWithTweetsToSync.Followers.Length == 0)
+                {
+                    userWithTweetsToSync.Followers = await _followersDal.GetFollowersAsync(user.Id);
+                }
                 // Process Shared Inbox
                 var followersWtSharedInbox = userWithTweetsToSync.Followers
                     .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
@@ -60,7 +64,7 @@ namespace BirdsiteLive.Pipeline.Processors
                     .Where(x => string.IsNullOrWhiteSpace(x.SharedInboxRoute))
                     .ToList();
                 await ProcessFollowersWithInboxAsync(userWithTweetsToSync.Tweets, followerWtInbox, user);
-            });
+            }, ct);
             _todo.Add(t);
 
             if (_todo.Count >= _instanceSettings.ParallelFediversePosts)
diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveFollowersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveFollowersProcessorTests.cs
index 4679259..8188540 100644
--- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveFollowersProcessorTests.cs
+++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveFollowersProcessorTests.cs
@@ -14,6 +14,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
     [TestClass]
     public class RetrieveFollowersProcessorTests
     {
+        [Ignore]
         [TestMethod]
         public async Task ProcessAsync_Test()
         {

From 9551c735ea5db6a01e5bc00d4e3a1c0214f69787 Mon Sep 17 00:00:00 2001
From: Vincent Cloutier <vincent@cloutier.co>
Date: Sat, 25 Mar 2023 13:53:07 -0400
Subject: [PATCH 06/20] moved followers retrieval 2

---
 .../RetrieveTwitterUsersProcessor.cs          | 10 ++++-
 .../SendTweetsToFollowersProcessor.cs         |  4 --
 .../RetrieveTwitterUsersProcessorTests.cs     | 43 ++++++++++++++++---
 3 files changed, 45 insertions(+), 12 deletions(-)

diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
index af721d6..31b4ea4 100644
--- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
+++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
@@ -15,15 +15,17 @@ namespace BirdsiteLive.Pipeline.Processors
     public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor
     {
         private readonly ITwitterUserDal _twitterUserDal;
+        private readonly IFollowersDal _followersDal;
         private readonly ILogger<RetrieveTwitterUsersProcessor> _logger;
         private static Random rng = new Random();
         
         public int WaitFactor = 1000 * 60; //1 min
 
         #region Ctor
-        public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, ILogger<RetrieveTwitterUsersProcessor> logger)
+        public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, ILogger<RetrieveTwitterUsersProcessor> logger)
         {
             _twitterUserDal = twitterUserDal;
+            _followersDal = followersDal;
             _logger = logger;
         }
         #endregion
@@ -44,7 +46,11 @@ namespace BirdsiteLive.Pipeline.Processors
                     foreach (var u in splitUsers)
                     {
                         ct.ThrowIfCancellationRequested();
-                        UserWithDataToSync[] toSync = u.Select(x => new UserWithDataToSync { User = x }).ToArray();
+                        UserWithDataToSync[] toSync = await Task.WhenAll(
+                            u.Select(async x => new UserWithDataToSync
+                                { User = x, Followers = await _followersDal.GetFollowersAsync(x.Id) } 
+                            )
+                        );
 
                         await twitterUsersBufferBlock.SendAsync(toSync, ct);
                     }
diff --git a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs
index 630d158..80bb25a 100644
--- a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs
+++ b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs
@@ -49,10 +49,6 @@ namespace BirdsiteLive.Pipeline.Processors
             
             var t = Task.Run( async () => 
             {
-                if (userWithTweetsToSync.Followers is null || userWithTweetsToSync.Followers.Length == 0)
-                {
-                    userWithTweetsToSync.Followers = await _followersDal.GetFollowersAsync(user.Id);
-                }
                 // Process Shared Inbox
                 var followersWtSharedInbox = userWithTweetsToSync.Followers
                     .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs
index 244c85f..a9cddd3 100644
--- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs
+++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs
@@ -38,10 +38,16 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                     It.Is<int>(y => true)))
                 .ReturnsAsync(users);
             
+            var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
+            followersDalMock
+                .Setup(x => x.GetFollowersAsync(It.Is<int>(x => true)))
+                .ReturnsAsync(new Follower[] {});
+            
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
+            
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
             processor.WaitFactor = 10;
             var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -79,10 +85,15 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 .ReturnsAsync(new SyncTwitterUser[0])
                 .ReturnsAsync(new SyncTwitterUser[0]);
 
+            var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
+            followersDalMock
+                .Setup(x => x.GetFollowersAsync(It.Is<int>(x => true)))
+                .ReturnsAsync(new Follower[] {});
+            
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
             processor.WaitFactor = 2;
             var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -120,10 +131,15 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 .ReturnsAsync(new SyncTwitterUser[0])
                 .ReturnsAsync(new SyncTwitterUser[0]);
             
+            var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
+            followersDalMock
+                .Setup(x => x.GetFollowersAsync(It.Is<int>(x => true)))
+                .ReturnsAsync(new Follower[] {});
+            
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
             processor.WaitFactor = 2;
             var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -154,10 +170,15 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                     It.Is<int>(y => true)))
                 .ReturnsAsync(new SyncTwitterUser[0]);
 
+            var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
+            followersDalMock
+                .Setup(x => x.GetFollowersAsync(It.Is<int>(x => true)))
+                .ReturnsAsync(new Follower[] {});
+            
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
             processor.WaitFactor = 1;
             var t =processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -185,10 +206,15 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                     It.Is<int>(y => true)))
                 .Returns(async () => await DelayFaultedTask<SyncTwitterUser[]>(new Exception()));
 
+            var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
+            followersDalMock
+                .Setup(x => x.GetFollowersAsync(It.Is<int>(x => true)))
+                .ReturnsAsync(new Follower[] {});
+            
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
             processor.WaitFactor = 10;
             var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -215,10 +241,15 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             
+            var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
+            followersDalMock
+                .Setup(x => x.GetFollowersAsync(It.Is<int>(x => true)))
+                .ReturnsAsync(new Follower[] {});
+            
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
             processor.WaitFactor = 1;
             await processor.GetTwitterUsersAsync(buffer, canTokenS.Token);
         }

From 46be9552e99d002c80205bbc04f8d3f3e662c54a Mon Sep 17 00:00:00 2001
From: Vincent Cloutier <vincent@cloutier.co>
Date: Sat, 25 Mar 2023 15:26:11 -0400
Subject: [PATCH 07/20] pipeline refactoring

---
 .../ISendTweetsToFollowersProcessor.cs        |  2 +-
 .../SendTweetsToFollowersProcessor.cs         | 51 ++++++++++---------
 .../StatusPublicationPipeline.cs              | 10 ++--
 .../SendTweetsToFollowersProcessorTests.cs    | 24 ++++-----
 4 files changed, 45 insertions(+), 42 deletions(-)

diff --git a/src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs
index c188f55..eddadf5 100644
--- a/src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs
+++ b/src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs
@@ -6,6 +6,6 @@ namespace BirdsiteLive.Pipeline.Contracts
 {
     public interface ISendTweetsToFollowersProcessor
     {
-        Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct);
+        Task ProcessAsync(UserWithDataToSync[] usersWithTweetsToSync, CancellationToken ct);
     }
 }
\ No newline at end of file
diff --git a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs
index 80bb25a..49e3bdb 100644
--- a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs
+++ b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs
@@ -41,34 +41,39 @@ namespace BirdsiteLive.Pipeline.Processors
         }
         #endregion
 
-        public async Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
+        public async Task ProcessAsync(UserWithDataToSync[] usersWithTweetsToSync, CancellationToken ct)
         {
-            var user = userWithTweetsToSync.User;
-
-            _todo = _todo.Where(x => !x.IsCompleted).ToList();
-            
-            var t = Task.Run( async () => 
+            foreach (var userWithTweetsToSync in usersWithTweetsToSync)
             {
-                // Process Shared Inbox
-                var followersWtSharedInbox = userWithTweetsToSync.Followers
-                    .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
-                    .ToList();
-                await ProcessFollowersWithSharedInboxAsync(userWithTweetsToSync.Tweets, followersWtSharedInbox, user);
+                var user = userWithTweetsToSync.User;
 
-                // Process Inbox
-                var followerWtInbox = userWithTweetsToSync.Followers
-                    .Where(x => string.IsNullOrWhiteSpace(x.SharedInboxRoute))
-                    .ToList();
-                await ProcessFollowersWithInboxAsync(userWithTweetsToSync.Tweets, followerWtInbox, user);
-            }, ct);
-            _todo.Add(t);
+                _todo = _todo.Where(x => !x.IsCompleted).ToList();
+                
+                var t = Task.Run( async () => 
+                {
+                    // Process Shared Inbox
+                    var followersWtSharedInbox = userWithTweetsToSync.Followers
+                        .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
+                        .ToList();
+                    await ProcessFollowersWithSharedInboxAsync(userWithTweetsToSync.Tweets, followersWtSharedInbox, user);
 
-            if (_todo.Count >= _instanceSettings.ParallelFediversePosts)
-            {
-                await Task.WhenAny(_todo);
+                    // Process Inbox
+                    var followerWtInbox = userWithTweetsToSync.Followers
+                        .Where(x => string.IsNullOrWhiteSpace(x.SharedInboxRoute))
+                        .ToList();
+                    await ProcessFollowersWithInboxAsync(userWithTweetsToSync.Tweets, followerWtInbox, user);
+                    
+                    _logger.LogInformation("Done sending " + userWithTweetsToSync.Tweets.Length + "tweets for user " + userWithTweetsToSync.User.Acct);
+                }, ct);
+                _todo.Add(t);
+
+                if (_todo.Count >= _instanceSettings.ParallelFediversePosts)
+                {
+                    await Task.WhenAny(_todo);
+                }
+                
+                
             }
-            
-            _logger.LogInformation("Done sending " + userWithTweetsToSync.Followers.Length + "tweets for user " + userWithTweetsToSync.User.Acct);
 
         }
 
diff --git a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs
index 97c4bf6..47ee70d 100644
--- a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs
+++ b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs
@@ -45,16 +45,14 @@ namespace BirdsiteLive.Pipeline
                 { BoundedCapacity = 1, CancellationToken = ct });
             var retrieveTweetsBlock = new TransformBlock<UserWithDataToSync[], UserWithDataToSync[]>(async x =>  await _retrieveTweetsProcessor.ProcessAsync(x, ct), standardBlockOptions );
             var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
-            var retrieveFollowersBlock = new TransformManyBlock<UserWithDataToSync[], UserWithDataToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { BoundedCapacity = 1 } );
-            var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 500, CancellationToken = ct });
-            var sendTweetsToFollowersBlock = new ActionBlock<UserWithDataToSync>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), standardBlockOptions);
+           // var retrieveFollowersBlock = new TransformManyBlock<UserWithDataToSync[], UserWithDataToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { BoundedCapacity = 1 } );
+           // var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 500, CancellationToken = ct });
+            var sendTweetsToFollowersBlock = new ActionBlock<UserWithDataToSync[]>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), standardBlockOptions);
 
             // Link pipeline
             twitterUserToRefreshBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions { PropagateCompletion = true });
             retrieveTweetsBlock.LinkTo(retrieveTweetsBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
-            retrieveTweetsBufferBlock.LinkTo(retrieveFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
-            retrieveFollowersBlock.LinkTo(retrieveFollowersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
-            retrieveFollowersBufferBlock.LinkTo(sendTweetsToFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
+            retrieveTweetsBufferBlock.LinkTo(sendTweetsToFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
 
             // Launch twitter user retriever after a little delay
             // to give time for the Tweet cache to fill
diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs
index 06d8b67..2121831 100644
--- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs
+++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs
@@ -88,7 +88,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new[] {userWithTweets}, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();
@@ -170,7 +170,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new[] {userWithTweets}, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();
@@ -261,7 +261,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new[] {userWithTweets}, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();
@@ -353,7 +353,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();
@@ -450,7 +450,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();
@@ -530,7 +530,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();
@@ -611,7 +611,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();
@@ -700,7 +700,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();
@@ -790,7 +790,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();
@@ -880,7 +880,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();
@@ -970,7 +970,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new [] {userWithTweets}, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();
@@ -1065,7 +1065,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             #endregion
 
             var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object);
-            await processor.ProcessAsync(userWithTweets, CancellationToken.None);
+            await processor.ProcessAsync(new [] { userWithTweets }, CancellationToken.None);
 
             #region Validations
             sendTweetsToInboxTaskMock.VerifyAll();

From f631e922bc271f3f674fc90af16f8a33bf9f6b1a Mon Sep 17 00:00:00 2001
From: Vincent Cloutier <vincent@cloutier.co>
Date: Sat, 25 Mar 2023 15:44:42 -0400
Subject: [PATCH 08/20] tweak logging

---
 .../Processors/SendTweetsToFollowersProcessor.cs               | 3 ++-
 .../Processors/SubTasks/SendTweetsToSharedInboxTask.cs         | 1 -
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs
index 49e3bdb..9cfac79 100644
--- a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs
+++ b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs
@@ -63,7 +63,8 @@ namespace BirdsiteLive.Pipeline.Processors
                         .ToList();
                     await ProcessFollowersWithInboxAsync(userWithTweetsToSync.Tweets, followerWtInbox, user);
                     
-                    _logger.LogInformation("Done sending " + userWithTweetsToSync.Tweets.Length + "tweets for user " + userWithTweetsToSync.User.Acct);
+                    _logger.LogInformation("Done sending " + userWithTweetsToSync.Tweets.Length + " tweets for "
+                        + userWithTweetsToSync.Followers.Length + "followers for user " + userWithTweetsToSync.User.Acct);
                 }, ct);
                 _todo.Add(t);
 
diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs
index f66eeab..87966c4 100644
--- a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs
+++ b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs
@@ -43,7 +43,6 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
             var tweetsToSend = tweets
                 .OrderBy(x => x.Id)
                 .ToList();
-            _logger.LogInformation("After filtering, there were " + tweetsToSend.Count() + " tweets left to send");
 
             foreach (var tweet in tweetsToSend)
             {

From fe1dce630098f3dcb7e6ecac5f37cdd9e3f3747b Mon Sep 17 00:00:00 2001
From: Vincent Cloutier <vincent@cloutier.co>
Date: Mon, 27 Mar 2023 19:12:24 -0400
Subject: [PATCH 09/20] twitter auth tweaks

---
 .../Tools/TwitterAuthenticationInitializer.cs            | 9 +++++----
 src/BirdsiteLive.Twitter/TwitterTweetsService.cs         | 6 +++---
 src/BirdsiteLive.Twitter/TwitterUserService.cs           | 2 +-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs
index bad4a3e..c8b7a57 100644
--- a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs
+++ b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs
@@ -14,7 +14,7 @@ namespace BirdsiteLive.Twitter.Tools
     public interface ITwitterAuthenticationInitializer
     {
         Task<HttpClient> MakeHttpClient();
-        HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint);
+        HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint, bool addToken);
         Task RefreshClient(HttpRequestMessage client);
     }
 
@@ -27,7 +27,7 @@ namespace BirdsiteLive.Twitter.Tools
         private List<HttpClient> _twitterClients = new List<HttpClient>();
         private List<String> _tokens = new List<string>();
         static Random rnd = new Random();
-        private const int _targetClients = 20;
+        private const int _targetClients = 3;
         public String BearerToken { 
             get { return "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"; }
         }
@@ -132,12 +132,13 @@ namespace BirdsiteLive.Twitter.Tools
             int r = rnd.Next(_twitterClients.Count);
             return _twitterClients[r];
         }
-        public HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint)
+        public HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint, bool addToken)
         {
             var request = new HttpRequestMessage(m, endpoint);
             int r = rnd.Next(_twitterClients.Count);
             request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + BearerToken); 
-            request.Headers.TryAddWithoutValidation("x-guest-token", _tokens[r]);
+            if (addToken)
+                request.Headers.TryAddWithoutValidation("x-guest-token", _tokens[r]);
             //request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");
             //request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes");
             return request;
diff --git a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs
index 5811c74..88fd585 100644
--- a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs
+++ b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs
@@ -55,7 +55,7 @@ namespace BirdsiteLive.Twitter
                 "https://api.twitter.com/graphql/XjlydVWHFIDaAUny86oh2g/TweetDetail?variables=%7B%22focalTweetId%22%3A%22"
                 + statusId +
                 "%22,%22with_rux_injections%22%3Atrue,%22includePromotedContent%22%3Afalse,%22withCommunity%22%3Afalse,%22withQuickPromoteEligibilityTweetFields%22%3Afalse,%22withBirdwatchNotes%22%3Afalse,%22withSuperFollowsUserFields%22%3Afalse,%22withDownvotePerspective%22%3Afalse,%22withReactionsMetadata%22%3Afalse,%22withReactionsPerspective%22%3Afalse,%22withSuperFollowsTweetFields%22%3Afalse,%22withVoice%22%3Atrue,%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue,%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue,%22verified_phone_label_enabled%22%3Afalse,%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue,%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse,%22tweetypie_unmention_optimization_enabled%22%3Atrue,%22vibe_api_enabled%22%3Atrue,%22responsive_web_edit_tweet_api_enabled%22%3Atrue,%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Afalse,%22view_counts_everywhere_api_enabled%22%3Atrue,%22longform_notetweets_consumption_enabled%22%3Atrue,%22tweet_awards_web_tipping_enabled%22%3Afalse,%22freedom_of_speech_not_reach_fetch_enabled%22%3Afalse,%22standardized_nudges_misinfo%22%3Atrue,%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse,%22interactive_text_enabled%22%3Atrue,%22responsive_web_text_conversations_enabled%22%3Afalse,%22longform_notetweets_richtext_consumption_enabled%22%3Afalse,%22responsive_web_enhance_cards_enabled%22%3Atrue%7D";
-            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL);
+            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL, true);
             try
             {
                 JsonDocument tweet;
@@ -104,7 +104,7 @@ namespace BirdsiteLive.Twitter
                 userId + "%22,%22count%22%3A40,%22includePromotedContent%22%3Atrue,%22withCommunity%22%3Atrue,%22withSuperFollowsUserFields%22%3Atrue,%22withDownvotePerspective%22%3Afalse,%22withReactionsMetadata%22%3Afalse,%22withReactionsPerspective%22%3Afalse,%22withSuperFollowsTweetFields%22%3Atrue,%22withVoice%22%3Atrue,%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue,%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue,%22verified_phone_label_enabled%22%3Afalse,%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue,%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse,%22tweetypie_unmention_optimization_enabled%22%3Atrue,%22vibe_api_enabled%22%3Atrue,%22responsive_web_edit_tweet_api_enabled%22%3Atrue,%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue,%22view_counts_everywhere_api_enabled%22%3Atrue,%22longform_notetweets_consumption_enabled%22%3Atrue,%22tweet_awards_web_tipping_enabled%22%3Afalse,%22freedom_of_speech_not_reach_fetch_enabled%22%3Afalse,%22standardized_nudges_misinfo%22%3Atrue,%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse,%22interactive_text_enabled%22%3Atrue,%22responsive_web_text_conversations_enabled%22%3Afalse,%22longform_notetweets_richtext_consumption_enabled%22%3Afalse,%22responsive_web_enhance_cards_enabled%22%3Afalse%7D";
             JsonDocument results;
             List<ExtractedTweet> extractedTweets = new List<ExtractedTweet>();
-            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL);
+            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL, false);
             try
             {
 
@@ -118,7 +118,7 @@ namespace BirdsiteLive.Twitter
             catch (HttpRequestException e)
             {
                 _logger.LogError(e, "Error retrieving timeline of {Username}; refreshing client", username);
-                await _twitterAuthenticationInitializer.RefreshClient(request);
+                //await _twitterAuthenticationInitializer.RefreshClient(request);
                 return null;
             }
             catch (Exception e)
diff --git a/src/BirdsiteLive.Twitter/TwitterUserService.cs b/src/BirdsiteLive.Twitter/TwitterUserService.cs
index cdbc708..ea1c3b5 100644
--- a/src/BirdsiteLive.Twitter/TwitterUserService.cs
+++ b/src/BirdsiteLive.Twitter/TwitterUserService.cs
@@ -40,7 +40,7 @@ namespace BirdsiteLive.Twitter
 
             JsonDocument res;
             var client = await _twitterAuthenticationInitializer.MakeHttpClient();
-            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), endpoint.Replace("elonmusk", username));
+            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), endpoint.Replace("elonmusk", username), true);
             try
             {
 

From f554269cba01f6dd04a158cacc6d189fab5dcf7a Mon Sep 17 00:00:00 2001
From: Vincent Cloutier <vincent@cloutier.co>
Date: Mon, 27 Mar 2023 19:48:35 -0400
Subject: [PATCH 10/20] little rate limiting for twitter auth

---
 .../BirdsiteLive.Twitter.csproj               |  1 +
 .../Tools/TwitterAuthenticationInitializer.cs | 45 ++++++++++++-------
 2 files changed, 30 insertions(+), 16 deletions(-)

diff --git a/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj b/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj
index caf7b52..6d1746d 100644
--- a/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj
+++ b/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj
@@ -7,6 +7,7 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Http" Version="5.0.0" />
+    <PackageReference Include="System.Threading.RateLimiting" Version="7.0.0" />
   </ItemGroup>
 
   <ItemGroup>
diff --git a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs
index c8b7a57..99cd4f7 100644
--- a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs
+++ b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs
@@ -8,6 +8,7 @@ using Microsoft.Extensions.Logging;
 using System.Net.Http;
 using System.Net;
 using System.Text.Json;
+using System.Threading.RateLimiting;
 
 namespace BirdsiteLive.Twitter.Tools
 {
@@ -27,6 +28,8 @@ namespace BirdsiteLive.Twitter.Tools
         private List<HttpClient> _twitterClients = new List<HttpClient>();
         private List<String> _tokens = new List<string>();
         static Random rnd = new Random();
+        private RateLimiter _rateLimiter;
+        static SemaphoreSlim semaphoreSlim = new SemaphoreSlim(1, 1);
         private const int _targetClients = 3;
         public String BearerToken { 
             get { return "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"; }
@@ -41,7 +44,7 @@ namespace BirdsiteLive.Twitter.Tools
             aTimer = new System.Timers.Timer();
             aTimer.Interval = 20 * 1000; 
             aTimer.Elapsed += async (sender, e) => await RefreshCred();
-            
+
             aTimer.Start();
         }
         #endregion
@@ -69,22 +72,32 @@ namespace BirdsiteLive.Twitter.Tools
 
         private async Task RefreshCred()
         {
-            (string bearer, string guest) = await GetCred();
-
-            HttpClient client = _httpClientFactory.CreateClient();
-            //HttpClient client = new HttpClient();
-            client.DefaultRequestHeaders.TryAddWithoutValidation("Authorization", $"Bearer " + bearer); 
-            client.DefaultRequestHeaders.TryAddWithoutValidation("x-guest-token", guest);
-            client.DefaultRequestHeaders.TryAddWithoutValidation("Referer", "https://twitter.com/");
-            client.DefaultRequestHeaders.TryAddWithoutValidation("x-twitter-active-user", "yes");
-
-            _twitterClients.Add(client);
-            _tokens.Add(guest);
-
-            if (_twitterClients.Count > _targetClients)
+            
+            await semaphoreSlim.WaitAsync();
+            try
             {
-                _twitterClients.RemoveAt(0);
-                _tokens.RemoveAt(0);
+                (string bearer, string guest) = await GetCred();
+
+                HttpClient client = _httpClientFactory.CreateClient();
+                //HttpClient client = new HttpClient();
+                client.DefaultRequestHeaders.TryAddWithoutValidation("Authorization", $"Bearer " + bearer);
+                client.DefaultRequestHeaders.TryAddWithoutValidation("x-guest-token", guest);
+                client.DefaultRequestHeaders.TryAddWithoutValidation("Referer", "https://twitter.com/");
+                client.DefaultRequestHeaders.TryAddWithoutValidation("x-twitter-active-user", "yes");
+
+                _twitterClients.Add(client);
+                _tokens.Add(guest);
+
+                if (_twitterClients.Count > _targetClients)
+                {
+                    _twitterClients.RemoveAt(0);
+                    _tokens.RemoveAt(0);
+                }
+
+            }
+            finally
+            {
+                semaphoreSlim.Release();
             }
         }
 

From 2a15a3cae660ebbb5d2fc678a57a7fa4790b2cdf Mon Sep 17 00:00:00 2001
From: Vincent Cloutier <vincent@cloutier.co>
Date: Wed, 29 Mar 2023 19:03:22 -0400
Subject: [PATCH 11/20] twitter cache and auth tweaks

---
 .builds/arch.yml                                     |  8 ++++++--
 src/BirdsiteLive.Common/Settings/InstanceSettings.cs |  2 +-
 .../StatusPublicationPipeline.cs                     |  2 +-
 src/BirdsiteLive.Twitter/CachedTwitterService.cs     |  5 +++++
 .../Tools/TwitterAuthenticationInitializer.cs        |  7 -------
 src/BirdsiteLive/Controllers/UsersController.cs      | 12 ++++++++++--
 6 files changed, 23 insertions(+), 13 deletions(-)

diff --git a/.builds/arch.yml b/.builds/arch.yml
index fd6962c..817d9d3 100644
--- a/.builds/arch.yml
+++ b/.builds/arch.yml
@@ -13,8 +13,12 @@ tasks:
       sudo docker run -d -p 5432:5432 -e POSTGRES_PASSWORD=birdsitelive -e POSTGRES_USER=birdsitelive -e POSTGRES_DB=birdsitelive postgres:15
       cd bird.makeup/src
       dotnet test
-
-  - publish: |
+  - publish-arm: |
+      cd bird.makeup/src/BirdsiteLive
+      dotnet publish --os linux --arch arm64 /t:PublishContainer -c Release
+      docker tag cloutier/bird.makeup:1.0 cloutier/bird.makeup:latest-arm
+      docker push cloutier/bird.makeup:latest-arm
+  - publish-x64: |
       cd bird.makeup/src/BirdsiteLive
       dotnet publish --os linux --arch x64 /t:PublishContainer -c Release
       docker tag cloutier/bird.makeup:1.0 cloutier/bird.makeup:latest
diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs
index 4cec952..93c25a2 100644
--- a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs
+++ b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs
@@ -14,7 +14,7 @@
         public int FailingTwitterUserCleanUpThreshold { get; set; }
         public int FailingFollowerCleanUpThreshold { get; set; } = -1;
 
-        public int UserCacheCapacity { get; set; } = 20_000;
+        public int UserCacheCapacity { get; set; } = 40_000;
         public int TweetCacheCapacity { get; set; } = 20_000;
         public int ParallelTwitterRequests { get; set; } = 10;
         public int ParallelFediversePosts { get; set; } = 10;
diff --git a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs
index 47ee70d..cbdd169 100644
--- a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs
+++ b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs
@@ -44,7 +44,7 @@ namespace BirdsiteLive.Pipeline
             var twitterUserToRefreshBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions
                 { BoundedCapacity = 1, CancellationToken = ct });
             var retrieveTweetsBlock = new TransformBlock<UserWithDataToSync[], UserWithDataToSync[]>(async x =>  await _retrieveTweetsProcessor.ProcessAsync(x, ct), standardBlockOptions );
-            var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
+            var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 2, CancellationToken = ct });
            // var retrieveFollowersBlock = new TransformManyBlock<UserWithDataToSync[], UserWithDataToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { BoundedCapacity = 1 } );
            // var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 500, CancellationToken = ct });
             var sendTweetsToFollowersBlock = new ActionBlock<UserWithDataToSync[]>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), standardBlockOptions);
diff --git a/src/BirdsiteLive.Twitter/CachedTwitterService.cs b/src/BirdsiteLive.Twitter/CachedTwitterService.cs
index 92ccf52..91f46c9 100644
--- a/src/BirdsiteLive.Twitter/CachedTwitterService.cs
+++ b/src/BirdsiteLive.Twitter/CachedTwitterService.cs
@@ -11,6 +11,7 @@ namespace BirdsiteLive.Twitter
     {
         void PurgeUser(string username);
         void AddUser(TwitterUser user);
+        bool UserIsCached(string username);
     }
 
     public class CachedTwitterUserService : ICachedTwitterUserService
@@ -39,6 +40,10 @@ namespace BirdsiteLive.Twitter
         }
         #endregion
 
+        public bool UserIsCached(string username)
+        {
+            return _userCache.TryGetValue(username, out _);
+        }
         public async Task<TwitterUser> GetUserAsync(string username)
         {
             if (!_userCache.TryGetValue(username, out Task<TwitterUser> user))
diff --git a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs
index 99cd4f7..6602213 100644
--- a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs
+++ b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs
@@ -23,7 +23,6 @@ namespace BirdsiteLive.Twitter.Tools
     {
         private readonly ILogger<TwitterAuthenticationInitializer> _logger;
         private static bool _initialized;
-        private static System.Timers.Timer aTimer;
         private readonly IHttpClientFactory _httpClientFactory;
         private List<HttpClient> _twitterClients = new List<HttpClient>();
         private List<String> _tokens = new List<string>();
@@ -40,12 +39,6 @@ namespace BirdsiteLive.Twitter.Tools
         {
             _logger = logger;
             _httpClientFactory = httpClientFactory;
-
-            aTimer = new System.Timers.Timer();
-            aTimer.Interval = 20 * 1000; 
-            aTimer.Elapsed += async (sender, e) => await RefreshCred();
-
-            aTimer.Start();
         }
         #endregion
 
diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs
index d6b0bff..9e3e714 100644
--- a/src/BirdsiteLive/Controllers/UsersController.cs
+++ b/src/BirdsiteLive/Controllers/UsersController.cs
@@ -25,7 +25,7 @@ namespace BirdsiteLive.Controllers
 {
     public class UsersController : Controller
     {
-        private readonly ITwitterUserService _twitterUserService;
+        private readonly ICachedTwitterUserService _twitterUserService;
         private readonly ICachedTwitterTweetsService _twitterTweetService;
         private readonly IUserService _userService;
         private readonly IStatusService _statusService;
@@ -33,7 +33,7 @@ namespace BirdsiteLive.Controllers
         private readonly ILogger<UsersController> _logger;
 
         #region Ctor
-        public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ICachedTwitterTweetsService twitterTweetService, ILogger<UsersController> logger)
+        public UsersController(ICachedTwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ICachedTwitterTweetsService twitterTweetService, ILogger<UsersController> logger)
         {
             _twitterUserService = twitterUserService;
             _userService = userService;
@@ -75,6 +75,14 @@ namespace BirdsiteLive.Controllers
             {
                 try
                 {
+                    if (!_twitterUserService.UserIsCached(id))
+                    {
+                        try
+                        {
+                            await _twitterTweetService.GetTimelineAsync(id);
+                        }
+                        catch (Exception e) { }
+                    }
                     user = await _twitterUserService.GetUserAsync(id);
                 }
                 catch (UserNotFoundException)

From 348c46eb8f0e79d0e53fffdb083b4bead5dd89aa Mon Sep 17 00:00:00 2001
From: Vincent Cloutier <vincent@cloutier.co>
Date: Thu, 30 Mar 2023 19:19:16 -0400
Subject: [PATCH 12/20] twitter auth tweaks

---
 .../Settings/InstanceSettings.cs              |  2 +
 .../Tools/TwitterAuthenticationInitializer.cs | 43 +++++++++++++++++--
 .../TimelineTests.cs                          |  2 +-
 .../BirdsiteLive.Twitter.Tests/TweetTests.cs  |  2 +-
 .../BirdsiteLive.Twitter.Tests/UsersTest.cs   |  7 ++-
 5 files changed, 50 insertions(+), 6 deletions(-)

diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs
index 93c25a2..56ae0b6 100644
--- a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs
+++ b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs
@@ -16,6 +16,8 @@
 
         public int UserCacheCapacity { get; set; } = 40_000;
         public int TweetCacheCapacity { get; set; } = 20_000;
+        // "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
+        public string TwitterBearerToken { get; set; } = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA";
         public int ParallelTwitterRequests { get; set; } = 10;
         public int ParallelFediversePosts { get; set; } = 10;
     }
diff --git a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs
index 6602213..7ffbb42 100644
--- a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs
+++ b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs
@@ -7,6 +7,8 @@ using BirdsiteLive.Common.Settings;
 using Microsoft.Extensions.Logging;
 using System.Net.Http;
 using System.Net;
+using System.Net.Http.Headers;
+using System.Text;
 using System.Text.Json;
 using System.Threading.RateLimiting;
 
@@ -30,18 +32,51 @@ namespace BirdsiteLive.Twitter.Tools
         private RateLimiter _rateLimiter;
         static SemaphoreSlim semaphoreSlim = new SemaphoreSlim(1, 1);
         private const int _targetClients = 3;
+        private InstanceSettings _instanceSettings;
+        private readonly (string, string)[] _apiKeys = new[]
+        {
+            ("IQKbtAYlXLripLGPWd0HUA", "GgDYlkSvaPxGxC4X8liwpUoqKwwr3lCADbz8A7ADU"), // iPhone
+            ("3nVuSoBZnx6U4vzUxf5w", "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"), // Android
+            ("CjulERsDeqhhjSme66ECg", "IQWdVyqFxghAtURHGeGiWAsmCAGmdW3WmbEx6Hck"), // iPad
+            ("3rJOl1ODzm9yZy63FACdg", "5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8"), // Mac
+        };
         public String BearerToken { 
-            get { return "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"; }
+            //get { return "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"; }
+            get
+            {
+                return _instanceSettings.TwitterBearerToken;
+            }
         }
 
         #region Ctor
-        public TwitterAuthenticationInitializer(IHttpClientFactory httpClientFactory, ILogger<TwitterAuthenticationInitializer> logger)
+        public TwitterAuthenticationInitializer(IHttpClientFactory httpClientFactory, InstanceSettings settings, ILogger<TwitterAuthenticationInitializer> logger)
         {
             _logger = logger;
+            _instanceSettings = settings;
             _httpClientFactory = httpClientFactory;
         }
         #endregion
 
+        private async Task GenerateBearerToken()
+        {
+            var httpClient = _httpClientFactory.CreateClient();
+            using (var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.twitter.com/oauth2/token?grant_type=client_credentials"))
+            {
+                int r = rnd.Next(_apiKeys.Length);
+                var (login, password) = _apiKeys[r];
+                var authValue = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{login}:{password}")));
+                request.Headers.TryAddWithoutValidation("Basic", $"Bearer " + authValue); 
+
+                var httpResponse = await httpClient.SendAsync(request);
+
+                var c = await httpResponse.Content.ReadAsStringAsync();
+                httpResponse.EnsureSuccessStatusCode();
+                var doc = JsonDocument.Parse(c);
+                var token = doc.RootElement.GetProperty("access_token").GetString();
+            }
+            
+        }
+
 
         public async Task RefreshClient(HttpRequestMessage req)
         {
@@ -49,7 +84,7 @@ namespace BirdsiteLive.Twitter.Tools
 
             var i = _tokens.IndexOf(token);
 
-            // this is prabably not thread save but yolo
+            // this is prabably not thread safe but yolo
             try
             {
                 _twitterClients.RemoveAt(i);
@@ -143,6 +178,8 @@ namespace BirdsiteLive.Twitter.Tools
             var request = new HttpRequestMessage(m, endpoint);
             int r = rnd.Next(_twitterClients.Count);
             request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + BearerToken); 
+            request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");
+            request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes");
             if (addToken)
                 request.Headers.TryAddWithoutValidation("x-guest-token", _tokens[r]);
             //request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");
diff --git a/src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs b/src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs
index ef26e2e..f09f28f 100644
--- a/src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs
+++ b/src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs
@@ -37,7 +37,7 @@ namespace BirdsiteLive.ActivityPub.Tests
                 ))
                 .ReturnsAsync(new SyncTwitterUser { TwitterUserId = default });
 
-            ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(httpFactory.Object, logger1.Object);
+            ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(httpFactory.Object, settings, logger1.Object);
             ITwitterUserService user = new TwitterUserService(auth, stats.Object, logger2.Object);
             ICachedTwitterUserService user2 = new CachedTwitterUserService(user, settings);
             _tweetService = new TwitterTweetsService(auth, stats.Object, user2, twitterDal.Object, settings, logger3.Object);
diff --git a/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs b/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs
index ed245bc..e25ab59 100644
--- a/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs
+++ b/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs
@@ -32,7 +32,7 @@ namespace BirdsiteLive.ActivityPub.Tests
             {
                 Domain = "domain.name"
             };
-            ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(httpFactory.Object, logger1.Object);
+            ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(httpFactory.Object, settings, logger1.Object);
             ITwitterUserService user = new TwitterUserService(auth, stats.Object, logger2.Object);
             ICachedTwitterUserService user2 = new CachedTwitterUserService(user, settings);
             _tweetService = new TwitterTweetsService(auth, stats.Object, user2, twitterDal.Object, settings, logger3.Object);
diff --git a/src/Tests/BirdsiteLive.Twitter.Tests/UsersTest.cs b/src/Tests/BirdsiteLive.Twitter.Tests/UsersTest.cs
index a3ec0e6..06e248a 100644
--- a/src/Tests/BirdsiteLive.Twitter.Tests/UsersTest.cs
+++ b/src/Tests/BirdsiteLive.Twitter.Tests/UsersTest.cs
@@ -6,6 +6,7 @@ using BirdsiteLive.Twitter.Tools;
 using BirdsiteLive.Statistics.Domain;
 using Moq;
 using System.Net.Http;
+using BirdsiteLive.Common.Settings;
 
 namespace BirdsiteLive.ActivityPub.Tests
 {
@@ -21,8 +22,12 @@ namespace BirdsiteLive.ActivityPub.Tests
             var logger3 = new Mock<ILogger<TwitterUserService>>();
             var stats = new Mock<ITwitterStatisticsHandler>();
             var httpFactory = new Mock<IHttpClientFactory>();
+            var settings = new InstanceSettings
+            {
+                Domain = "domain.name"
+            };
             httpFactory.Setup(_ => _.CreateClient(string.Empty)).Returns(new HttpClient());
-            ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(httpFactory.Object, logger1.Object);
+            ITwitterAuthenticationInitializer auth = new TwitterAuthenticationInitializer(httpFactory.Object, settings, logger1.Object);
             _tweetService = new TwitterUserService(auth, stats.Object, logger3.Object);
         }
 

From c21f0bac5bc616d7bf4bbb3f277e4e1f01704c39 Mon Sep 17 00:00:00 2001
From: Vincent Cloutier <vincent@cloutier.co>
Date: Thu, 30 Mar 2023 20:47:53 -0400
Subject: [PATCH 13/20] auth changes

---
 .../Tools/TwitterAuthenticationInitializer.cs | 77 +++++++------------
 .../TimelineTests.cs                          |  6 +-
 .../BirdsiteLive.Twitter.Tests/TweetTests.cs  |  1 +
 3 files changed, 29 insertions(+), 55 deletions(-)

diff --git a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs
index 7ffbb42..ffc87f6 100644
--- a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs
+++ b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs
@@ -27,7 +27,7 @@ namespace BirdsiteLive.Twitter.Tools
         private static bool _initialized;
         private readonly IHttpClientFactory _httpClientFactory;
         private List<HttpClient> _twitterClients = new List<HttpClient>();
-        private List<String> _tokens = new List<string>();
+        private List<(String, String)> _tokens = new List<(string,string)>();
         static Random rnd = new Random();
         private RateLimiter _rateLimiter;
         static SemaphoreSlim semaphoreSlim = new SemaphoreSlim(1, 1);
@@ -57,7 +57,7 @@ namespace BirdsiteLive.Twitter.Tools
         }
         #endregion
 
-        private async Task GenerateBearerToken()
+        private async Task<string> GenerateBearerToken()
         {
             var httpClient = _httpClientFactory.CreateClient();
             using (var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.twitter.com/oauth2/token?grant_type=client_credentials"))
@@ -65,7 +65,7 @@ namespace BirdsiteLive.Twitter.Tools
                 int r = rnd.Next(_apiKeys.Length);
                 var (login, password) = _apiKeys[r];
                 var authValue = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{login}:{password}")));
-                request.Headers.TryAddWithoutValidation("Basic", $"Bearer " + authValue); 
+                request.Headers.Authorization = authValue;
 
                 var httpResponse = await httpClient.SendAsync(request);
 
@@ -73,6 +73,7 @@ namespace BirdsiteLive.Twitter.Tools
                 httpResponse.EnsureSuccessStatusCode();
                 var doc = JsonDocument.Parse(c);
                 var token = doc.RootElement.GetProperty("access_token").GetString();
+                return token;
             }
             
         }
@@ -81,8 +82,9 @@ namespace BirdsiteLive.Twitter.Tools
         public async Task RefreshClient(HttpRequestMessage req)
         {
             string token = req.Headers.GetValues("x-guest-token").First();
+            string bearer = req.Headers.GetValues("Authorization").First().Replace("Bearer ", "");
 
-            var i = _tokens.IndexOf(token);
+            var i = _tokens.IndexOf((bearer, token));
 
             // this is prabably not thread safe but yolo
             try
@@ -96,46 +98,37 @@ namespace BirdsiteLive.Twitter.Tools
             }
 
             await RefreshCred();
+            await Task.Delay(1000);
+            await RefreshCred();
         }
 
         private async Task RefreshCred()
         {
             
-            await semaphoreSlim.WaitAsync();
-            try
+            (string bearer, string guest) = await GetCred();
+
+            HttpClient client = _httpClientFactory.CreateClient();
+            //HttpClient client = new HttpClient();
+
+            _twitterClients.Add(client);
+            _tokens.Add((bearer,guest));
+
+            if (_twitterClients.Count > _targetClients)
             {
-                (string bearer, string guest) = await GetCred();
-
-                HttpClient client = _httpClientFactory.CreateClient();
-                //HttpClient client = new HttpClient();
-                client.DefaultRequestHeaders.TryAddWithoutValidation("Authorization", $"Bearer " + bearer);
-                client.DefaultRequestHeaders.TryAddWithoutValidation("x-guest-token", guest);
-                client.DefaultRequestHeaders.TryAddWithoutValidation("Referer", "https://twitter.com/");
-                client.DefaultRequestHeaders.TryAddWithoutValidation("x-twitter-active-user", "yes");
-
-                _twitterClients.Add(client);
-                _tokens.Add(guest);
-
-                if (_twitterClients.Count > _targetClients)
-                {
-                    _twitterClients.RemoveAt(0);
-                    _tokens.RemoveAt(0);
-                }
-
-            }
-            finally
-            {
-                semaphoreSlim.Release();
+                _twitterClients.RemoveAt(0);
+                _tokens.RemoveAt(0);
             }
+
         }
 
         private async Task<(string, string)> GetCred()
         {
             string token;
             var httpClient = _httpClientFactory.CreateClient();
+            string bearer = await GenerateBearerToken();
             using (var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.twitter.com/1.1/guest/activate.json"))
             {
-                request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + BearerToken); 
+                request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer); 
 
                 var httpResponse = await httpClient.SendAsync(request);
 
@@ -145,30 +138,13 @@ namespace BirdsiteLive.Twitter.Tools
                 token = doc.RootElement.GetProperty("guest_token").GetString();
             }
 
-            return (BearerToken, token);
+            return (bearer, token);
 
         }
-        private async Task InitTwitterCredentials()
-        {
-            for (;;)
-            {
-                try
-                {
-                    await RefreshCred();
-                    _initialized = true;
-                    return;
-                }
-                catch (Exception e)
-                {
-                    _logger.LogError(e, "Twitter Authentication Failed");
-                    await Task.Delay(3600*1000);
-                }
-            }
-        }
 
         public async Task<HttpClient> MakeHttpClient()
         {
-            if (_twitterClients.Count < _targetClients)
+            if (_twitterClients.Count < 2)
                 await RefreshCred();
             int r = rnd.Next(_twitterClients.Count);
             return _twitterClients[r];
@@ -177,11 +153,12 @@ namespace BirdsiteLive.Twitter.Tools
         {
             var request = new HttpRequestMessage(m, endpoint);
             int r = rnd.Next(_twitterClients.Count);
-            request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + BearerToken); 
+            (string bearer, string token) = _tokens[r];
+            request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer); 
             request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");
             request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes");
             if (addToken)
-                request.Headers.TryAddWithoutValidation("x-guest-token", _tokens[r]);
+                request.Headers.TryAddWithoutValidation("x-guest-token", token);
             //request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");
             //request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes");
             return request;
diff --git a/src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs b/src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs
index f09f28f..80b69b1 100644
--- a/src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs
+++ b/src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs
@@ -41,11 +41,7 @@ namespace BirdsiteLive.ActivityPub.Tests
             ITwitterUserService user = new TwitterUserService(auth, stats.Object, logger2.Object);
             ICachedTwitterUserService user2 = new CachedTwitterUserService(user, settings);
             _tweetService = new TwitterTweetsService(auth, stats.Object, user2, twitterDal.Object, settings, logger3.Object);
-            
-            await Task.Delay(1000);
-            await auth.MakeHttpClient();
-            await Task.Delay(1000);
-            await auth.MakeHttpClient();
+
         }
 
         [TestMethod]
diff --git a/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs b/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs
index e25ab59..b40674b 100644
--- a/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs
+++ b/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs
@@ -78,6 +78,7 @@ namespace BirdsiteLive.ActivityPub.Tests
             Assert.IsTrue(tweet.Media[0].Url.StartsWith("https://video.twimg.com/"));
         }
 
+        [Ignore]
         [TestMethod]
         public async Task GifAndQT()
         {

From 6ed607f3fc524d47a3b3bb88219f1e82d8188f5a Mon Sep 17 00:00:00 2001
From: Vincent Cloutier <vincent@cloutier.co>
Date: Thu, 30 Mar 2023 21:02:14 -0400
Subject: [PATCH 14/20] user controller tweak

---
 src/BirdsiteLive/Controllers/UsersController.cs | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs
index 9e3e714..32b1fa4 100644
--- a/src/BirdsiteLive/Controllers/UsersController.cs
+++ b/src/BirdsiteLive/Controllers/UsersController.cs
@@ -75,14 +75,6 @@ namespace BirdsiteLive.Controllers
             {
                 try
                 {
-                    if (!_twitterUserService.UserIsCached(id))
-                    {
-                        try
-                        {
-                            await _twitterTweetService.GetTimelineAsync(id);
-                        }
-                        catch (Exception e) { }
-                    }
                     user = await _twitterUserService.GetUserAsync(id);
                 }
                 catch (UserNotFoundException)

From bc90bc293ec10c860722bc608b37df56be990645 Mon Sep 17 00:00:00 2001
From: Vincent Cloutier <vincent@cloutier.co>
Date: Fri, 31 Mar 2023 13:02:09 -0400
Subject: [PATCH 15/20] added back timeline fetching token

---
 src/BirdsiteLive.Twitter/TwitterTweetsService.cs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs
index 88fd585..931acc0 100644
--- a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs
+++ b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs
@@ -104,7 +104,7 @@ namespace BirdsiteLive.Twitter
                 userId + "%22,%22count%22%3A40,%22includePromotedContent%22%3Atrue,%22withCommunity%22%3Atrue,%22withSuperFollowsUserFields%22%3Atrue,%22withDownvotePerspective%22%3Afalse,%22withReactionsMetadata%22%3Afalse,%22withReactionsPerspective%22%3Afalse,%22withSuperFollowsTweetFields%22%3Atrue,%22withVoice%22%3Atrue,%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue,%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue,%22verified_phone_label_enabled%22%3Afalse,%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue,%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse,%22tweetypie_unmention_optimization_enabled%22%3Atrue,%22vibe_api_enabled%22%3Atrue,%22responsive_web_edit_tweet_api_enabled%22%3Atrue,%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue,%22view_counts_everywhere_api_enabled%22%3Atrue,%22longform_notetweets_consumption_enabled%22%3Atrue,%22tweet_awards_web_tipping_enabled%22%3Afalse,%22freedom_of_speech_not_reach_fetch_enabled%22%3Afalse,%22standardized_nudges_misinfo%22%3Atrue,%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse,%22interactive_text_enabled%22%3Atrue,%22responsive_web_text_conversations_enabled%22%3Afalse,%22longform_notetweets_richtext_consumption_enabled%22%3Afalse,%22responsive_web_enhance_cards_enabled%22%3Afalse%7D";
             JsonDocument results;
             List<ExtractedTweet> extractedTweets = new List<ExtractedTweet>();
-            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL, false);
+            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL, true);
             try
             {
 

From 3346b7b5e83352de15852e103e8ed2b9456fda74 Mon Sep 17 00:00:00 2001
From: Vincent Cloutier <vincent@cloutier.co>
Date: Sat, 1 Apr 2023 19:55:20 -0400
Subject: [PATCH 16/20] sharding support

---
 .../Settings/InstanceSettings.cs              |  3 ++
 .../RetrieveTwitterUsersProcessor.cs          |  6 ++-
 .../TwitterUserPostgresDal.cs                 | 12 +++--
 .../Contracts/ITwitterUserDal.cs              |  2 +-
 .../RetrieveTwitterUsersProcessorTests.cs     | 51 ++++++++++++++++---
 5 files changed, 62 insertions(+), 12 deletions(-)

diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs
index 56ae0b6..1505793 100644
--- a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs
+++ b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs
@@ -18,6 +18,9 @@
         public int TweetCacheCapacity { get; set; } = 20_000;
         // "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
         public string TwitterBearerToken { get; set; } = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA";
+        public int m { get; set; } = 1;
+        public int n_start { get; set; } = 0;
+        public int n_end { get; set; } = 1;
         public int ParallelTwitterRequests { get; set; } = 10;
         public int ParallelFediversePosts { get; set; } = 10;
     }
diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
index 31b4ea4..39a6400 100644
--- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
+++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
@@ -16,16 +16,18 @@ namespace BirdsiteLive.Pipeline.Processors
     {
         private readonly ITwitterUserDal _twitterUserDal;
         private readonly IFollowersDal _followersDal;
+        private readonly InstanceSettings _instanceSettings;
         private readonly ILogger<RetrieveTwitterUsersProcessor> _logger;
         private static Random rng = new Random();
         
         public int WaitFactor = 1000 * 60; //1 min
 
         #region Ctor
-        public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, ILogger<RetrieveTwitterUsersProcessor> logger)
+        public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, InstanceSettings instanceSettings, ILogger<RetrieveTwitterUsersProcessor> logger)
         {
             _twitterUserDal = twitterUserDal;
             _followersDal = followersDal;
+            _instanceSettings = instanceSettings;
             _logger = logger;
         }
         #endregion
@@ -38,7 +40,7 @@ namespace BirdsiteLive.Pipeline.Processors
 
                 try
                 {
-                    var users = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000);
+                    var users = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000, _instanceSettings.n_start, _instanceSettings.n_end, _instanceSettings.m);
 
                     var userCount = users.Any() ? Math.Min(users.Length, 200) : 1;
                     var splitUsers = users.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList();
diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs
index 8b06e85..b8f2047 100644
--- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs
+++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs
@@ -118,14 +118,20 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
             }
         }
 
-        public async Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber)
+        public async Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber, int nStart, int nEnd, int m)
         {
-            var query = "SELECT * FROM (SELECT unnest(followings) as follow FROM followers GROUP BY follow) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id  ORDER BY lastSync ASC NULLS FIRST LIMIT $1";
+            var query = "SELECT * FROM (SELECT unnest(followings) as follow FROM followers GROUP BY follow) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id WHERE mod(id, $2) >= $3 AND mod(id, $2) <= $4 ORDER BY lastSync ASC NULLS FIRST LIMIT $1";
 
             await using var connection = DataSource.CreateConnection();
             await connection.OpenAsync();
             await using var command = new NpgsqlCommand(query, connection) {
-                Parameters = { new() { Value = maxNumber}}
+                Parameters =
+                {
+                    new() { Value = maxNumber},
+                    new() { Value = m},
+                    new() { Value = nStart},
+                    new() { Value = nEnd}
+                }
             };
             var reader = await command.ExecuteReaderAsync();
             var results = new List<SyncTwitterUser>();
diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs
index 7126b98..9e82b74 100644
--- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs
+++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs
@@ -9,7 +9,7 @@ namespace BirdsiteLive.DAL.Contracts
         Task CreateTwitterUserAsync(string acct, long lastTweetPostedId);
         Task<SyncTwitterUser> GetTwitterUserAsync(string acct);
         Task<SyncTwitterUser> GetTwitterUserAsync(int id);
-        Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber);
+        Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber, int nStart, int nEnd, int m);
         Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber);
         Task<SyncTwitterUser[]> GetAllTwitterUsersAsync();
         Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync);
diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs
index a9cddd3..bcd954d 100644
--- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs
+++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs
@@ -29,12 +29,19 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 new SyncTwitterUser(),
             };
             var maxUsers = 1000;
+            var instanceSettings = new InstanceSettings()
+            {
+                n_start = 1,
+            };
             #endregion
 
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .Setup(x => x.GetAllTwitterUsersWithFollowersAsync(
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
                     It.Is<int>(y => true)))
                 .ReturnsAsync(users);
             
@@ -47,7 +54,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
             processor.WaitFactor = 10;
             var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -72,12 +79,19 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 users.Add(new SyncTwitterUser());
 
             var maxUsers = 1000;
+            var instanceSettings = new InstanceSettings()
+            {
+                n_start = 1,
+            };
             #endregion
 
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .SetupSequence(x => x.GetAllTwitterUsersWithFollowersAsync(
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
                     It.Is<int>(y => true)))
                 .ReturnsAsync(users.ToArray())
                 .ReturnsAsync(new SyncTwitterUser[0])
@@ -93,7 +107,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
             processor.WaitFactor = 2;
             var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -118,12 +132,19 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 users.Add(new SyncTwitterUser());
 
             var maxUsers = 1000;
+            var instanceSettings = new InstanceSettings()
+            {
+                n_start = 1,
+            };
             #endregion
 
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .SetupSequence(x => x.GetAllTwitterUsersWithFollowersAsync(
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
                     It.Is<int>(y => true)))
                 .ReturnsAsync(users.ToArray())
                 .ReturnsAsync(new SyncTwitterUser[0])
@@ -139,7 +160,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
             processor.WaitFactor = 2;
             var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -160,6 +181,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var buffer = new BufferBlock<UserWithDataToSync[]>();
 
             var maxUsers = 1000;
+            var instanceSettings = new InstanceSettings()
+            {
+                n_start = 1,
+            };
             #endregion
 
             #region Mocks
@@ -167,6 +192,9 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .Setup(x => x.GetAllTwitterUsersWithFollowersAsync(
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
                     It.Is<int>(y => true)))
                 .ReturnsAsync(new SyncTwitterUser[0]);
 
@@ -178,7 +206,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
             processor.WaitFactor = 1;
             var t =processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -197,12 +225,19 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var buffer = new BufferBlock<UserWithDataToSync[]>();
 
             var maxUsers = 1000;
+            var instanceSettings = new InstanceSettings()
+            {
+                n_start = 1,
+            };
             #endregion
 
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .Setup(x => x.GetAllTwitterUsersWithFollowersAsync(
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
                     It.Is<int>(y => true)))
                 .Returns(async () => await DelayFaultedTask<SyncTwitterUser[]>(new Exception()));
 
@@ -214,7 +249,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
             processor.WaitFactor = 10;
             var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -236,6 +271,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             canTokenS.Cancel();
 
             var maxUsers = 1000;
+            var instanceSettings = new InstanceSettings()
+            {
+                n_start = 1,
+            };
             #endregion
 
             #region Mocks
@@ -249,7 +288,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
             processor.WaitFactor = 1;
             await processor.GetTwitterUsersAsync(buffer, canTokenS.Token);
         }

From 46f7594e43cb6a27f10a5ca5594f4300be9ee1de Mon Sep 17 00:00:00 2001
From: Vincent Cloutier <vincent@cloutier.co>
Date: Sun, 2 Apr 2023 10:23:45 -0400
Subject: [PATCH 17/20] RetrieveTwitterUsersProcessor tweaks

---
 .../RetrieveTwitterUsersProcessor.cs          | 42 +++++++++----------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
index 39a6400..977dafa 100644
--- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
+++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
@@ -38,31 +38,31 @@ namespace BirdsiteLive.Pipeline.Processors
             {
                 ct.ThrowIfCancellationRequested();
 
-                try
+                if (_instanceSettings.ParallelTwitterRequests == 0)
                 {
-                    var users = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000, _instanceSettings.n_start, _instanceSettings.n_end, _instanceSettings.m);
-
-                    var userCount = users.Any() ? Math.Min(users.Length, 200) : 1;
-                    var splitUsers = users.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList();
-
-                    foreach (var u in splitUsers)
-                    {
-                        ct.ThrowIfCancellationRequested();
-                        UserWithDataToSync[] toSync = await Task.WhenAll(
-                            u.Select(async x => new UserWithDataToSync
-                                { User = x, Followers = await _followersDal.GetFollowersAsync(x.Id) } 
-                            )
-                        );
-
-                        await twitterUsersBufferBlock.SendAsync(toSync, ct);
-                    }
-
-                    await Task.Delay(10, ct); // this is somehow necessary
+                    while (true)
+                        await Task.Delay(10000);
                 }
-                catch (Exception e)
+                
+                var users = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000, _instanceSettings.n_start, _instanceSettings.n_end, _instanceSettings.m);
+
+                var userCount = users.Any() ? Math.Min(users.Length, 200) : 1;
+                var splitUsers = users.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList();
+
+                foreach (var u in splitUsers)
                 {
-                    _logger.LogError(e, "Failing retrieving Twitter Users.");
+                    ct.ThrowIfCancellationRequested();
+                    UserWithDataToSync[] toSync = await Task.WhenAll(
+                        u.Select(async x => new UserWithDataToSync
+                            { User = x, Followers = await _followersDal.GetFollowersAsync(x.Id) } 
+                        )
+                    );
+
+                    await twitterUsersBufferBlock.SendAsync(toSync, ct);
+
                 }
+                
+                await Task.Delay(10, ct); // this is somehow necessary
             }
         }
     }

From 000214043cfefeed255b08d85fef714ff8a0c1e8 Mon Sep 17 00:00:00 2001
From: Vincent Cloutier <vincent@cloutier.co>
Date: Sun, 2 Apr 2023 11:10:14 -0400
Subject: [PATCH 18/20] RetrieveTwitterUsersProcessor tweaks 2

---
 .../RetrieveTwitterUsersProcessor.cs          | 27 ++++++++++++-------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
index 977dafa..f9acd30 100644
--- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
+++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -44,21 +45,27 @@ namespace BirdsiteLive.Pipeline.Processors
                         await Task.Delay(10000);
                 }
                 
-                var users = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000, _instanceSettings.n_start, _instanceSettings.n_end, _instanceSettings.m);
+                var usersDal = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000, _instanceSettings.n_start, _instanceSettings.n_end, _instanceSettings.m);
 
-                var userCount = users.Any() ? Math.Min(users.Length, 200) : 1;
-                var splitUsers = users.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList();
+                var userCount = usersDal.Any() ? Math.Min(usersDal.Length, 200) : 1;
+                var splitUsers = usersDal.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList();
 
-                foreach (var u in splitUsers)
+                foreach (var users in splitUsers)
                 {
                     ct.ThrowIfCancellationRequested();
-                    UserWithDataToSync[] toSync = await Task.WhenAll(
-                        u.Select(async x => new UserWithDataToSync
-                            { User = x, Followers = await _followersDal.GetFollowersAsync(x.Id) } 
-                        )
-                    );
+                    List<UserWithDataToSync> toSync = new List<UserWithDataToSync>();
+                    foreach (var u in users)
+                    {
+                        var followers = await _followersDal.GetFollowersAsync(u.Id);
+                        toSync.Add( new UserWithDataToSync()
+                        {
+                            User = u,
+                            Followers = followers
+                        });
+                        
+                    }
 
-                    await twitterUsersBufferBlock.SendAsync(toSync, ct);
+                    await twitterUsersBufferBlock.SendAsync(toSync.ToArray(), ct);
 
                 }
                 

From f3ea6b58a7a8b473beea7c8a4953468b93bef4b3 Mon Sep 17 00:00:00 2001
From: Vincent Cloutier <vincent@cloutier.co>
Date: Sun, 2 Apr 2023 11:29:14 -0400
Subject: [PATCH 19/20] made stats more efficient

---
 .../Services/CachedStatisticsService.cs       | 38 +++++++++++--------
 1 file changed, 23 insertions(+), 15 deletions(-)

diff --git a/src/BirdsiteLive/Services/CachedStatisticsService.cs b/src/BirdsiteLive/Services/CachedStatisticsService.cs
index d6567c6..22abf41 100644
--- a/src/BirdsiteLive/Services/CachedStatisticsService.cs
+++ b/src/BirdsiteLive/Services/CachedStatisticsService.cs
@@ -15,7 +15,7 @@ namespace BirdsiteLive.Services
         private readonly ITwitterUserDal _twitterUserDal;
         private readonly IFollowersDal _followersDal;
 
-        private static CachedStatistics _cachedStatistics;
+        private static Task<CachedStatistics> _cachedStatistics;
         private readonly InstanceSettings _instanceSettings;
 
         #region Ctor
@@ -24,28 +24,36 @@ namespace BirdsiteLive.Services
             _twitterUserDal = twitterUserDal;
             _instanceSettings = instanceSettings;
             _followersDal = followersDal;
+            _cachedStatistics = CreateStats();
         }
         #endregion
 
         public async Task<CachedStatistics> GetStatisticsAsync()
         {
-            if (_cachedStatistics == null ||
-                (DateTime.UtcNow - _cachedStatistics.RefreshedTime).TotalMinutes > 15)
+            var stats = await _cachedStatistics;
+            if ((DateTime.UtcNow - stats.RefreshedTime).TotalMinutes > 5)
             {
-                var twitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync();
-                var twitterSyncLag = await _twitterUserDal.GetTwitterSyncLag();
-                var fediverseUsers = await _followersDal.GetFollowersCountAsync();
-
-                _cachedStatistics = new CachedStatistics
-                {
-                    RefreshedTime = DateTime.UtcNow,
-                    SyncLag = twitterSyncLag,
-                    TwitterUsers = twitterUserCount,
-                    FediverseUsers = fediverseUsers
-                };
+                _cachedStatistics = CreateStats();
             }
 
-            return _cachedStatistics;
+            return stats;
+        }
+
+        private async Task<CachedStatistics> CreateStats()
+        {
+            var twitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync();
+            var twitterSyncLag = await _twitterUserDal.GetTwitterSyncLag();
+            var fediverseUsers = await _followersDal.GetFollowersCountAsync();
+
+            var stats = new CachedStatistics
+            {
+                RefreshedTime = DateTime.UtcNow,
+                SyncLag = twitterSyncLag,
+                TwitterUsers = twitterUserCount,
+                FediverseUsers = fediverseUsers
+            };
+        
+            return stats;
         }
     }
 

From 2dacf466fd5d49ff4c40d47caefbcadb246abc27 Mon Sep 17 00:00:00 2001
From: Vincent Cloutier <vincent@cloutier.co>
Date: Sun, 2 Apr 2023 11:38:56 -0400
Subject: [PATCH 20/20] optimizations

---
 .../Tools/TwitterAuthenticationInitializer.cs | 44 ++++---------------
 src/BirdsiteLive/Startup.cs                   |  3 ++
 2 files changed, 11 insertions(+), 36 deletions(-)

diff --git a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs
index ffc87f6..56b3d94 100644
--- a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs
+++ b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs
@@ -1,6 +1,7 @@
 using System;
 using System.Threading;
 using System.Collections.Generic;
+using System.Collections.Concurrent;
 using System.Linq;
 using System.Threading.Tasks;
 using BirdsiteLive.Common.Settings;
@@ -26,11 +27,9 @@ namespace BirdsiteLive.Twitter.Tools
         private readonly ILogger<TwitterAuthenticationInitializer> _logger;
         private static bool _initialized;
         private readonly IHttpClientFactory _httpClientFactory;
-        private List<HttpClient> _twitterClients = new List<HttpClient>();
-        private List<(String, String)> _tokens = new List<(string,string)>();
+        private ConcurrentDictionary<String, String> _token2 = new ConcurrentDictionary<string, string>();
         static Random rnd = new Random();
         private RateLimiter _rateLimiter;
-        static SemaphoreSlim semaphoreSlim = new SemaphoreSlim(1, 1);
         private const int _targetClients = 3;
         private InstanceSettings _instanceSettings;
         private readonly (string, string)[] _apiKeys = new[]
@@ -41,7 +40,6 @@ namespace BirdsiteLive.Twitter.Tools
             ("3rJOl1ODzm9yZy63FACdg", "5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8"), // Mac
         };
         public String BearerToken { 
-            //get { return "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"; }
             get
             {
                 return _instanceSettings.TwitterBearerToken;
@@ -82,20 +80,8 @@ namespace BirdsiteLive.Twitter.Tools
         public async Task RefreshClient(HttpRequestMessage req)
         {
             string token = req.Headers.GetValues("x-guest-token").First();
-            string bearer = req.Headers.GetValues("Authorization").First().Replace("Bearer ", "");
 
-            var i = _tokens.IndexOf((bearer, token));
-
-            // this is prabably not thread safe but yolo
-            try
-            {
-                _twitterClients.RemoveAt(i);
-                _tokens.RemoveAt(i);
-            }
-            catch (IndexOutOfRangeException _)
-            {
-                _logger.LogError("Error refreshing twitter token");
-            }
+            _token2.TryRemove(token, out _);
 
             await RefreshCred();
             await Task.Delay(1000);
@@ -104,21 +90,8 @@ namespace BirdsiteLive.Twitter.Tools
 
         private async Task RefreshCred()
         {
-            
             (string bearer, string guest) = await GetCred();
-
-            HttpClient client = _httpClientFactory.CreateClient();
-            //HttpClient client = new HttpClient();
-
-            _twitterClients.Add(client);
-            _tokens.Add((bearer,guest));
-
-            if (_twitterClients.Count > _targetClients)
-            {
-                _twitterClients.RemoveAt(0);
-                _tokens.RemoveAt(0);
-            }
-
+            _token2.TryAdd(guest, bearer);
         }
 
         private async Task<(string, string)> GetCred()
@@ -144,16 +117,15 @@ namespace BirdsiteLive.Twitter.Tools
 
         public async Task<HttpClient> MakeHttpClient()
         {
-            if (_twitterClients.Count < 2)
+            if (_token2.Count < _targetClients)
                 await RefreshCred();
-            int r = rnd.Next(_twitterClients.Count);
-            return _twitterClients[r];
+            return _httpClientFactory.CreateClient();
         }
         public HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint, bool addToken)
         {
             var request = new HttpRequestMessage(m, endpoint);
-            int r = rnd.Next(_twitterClients.Count);
-            (string bearer, string token) = _tokens[r];
+            //(string bearer, string token) = _tokens[r];
+            (string token, string bearer) = _token2.MaxBy(x => rnd.Next());
             request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer); 
             request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");
             request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes");
diff --git a/src/BirdsiteLive/Startup.cs b/src/BirdsiteLive/Startup.cs
index 40571dd..43b581d 100644
--- a/src/BirdsiteLive/Startup.cs
+++ b/src/BirdsiteLive/Startup.cs
@@ -9,6 +9,7 @@ using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Postgres.DataAccessLayers;
 using BirdsiteLive.DAL.Postgres.Settings;
 using BirdsiteLive.Models;
+using BirdsiteLive.Services;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Tools;
 using Lamar;
@@ -89,6 +90,8 @@ namespace BirdsiteLive
             services.For<ITwitterUserService>().Use<TwitterUserService>().Singleton();
 
             services.For<ITwitterAuthenticationInitializer>().Use<TwitterAuthenticationInitializer>().Singleton();
+            
+            services.For<ICachedStatisticsService>().Use<CachedStatisticsService>().Singleton();
 
             services.Scan(_ =>
             {