stop begging for money when your software does not work

typo not intentional but funny

.builds/arch.yml

@@ -0,0 +1,25 @@
+image: archlinux
+packages:
+  - dotnet-sdk
+  - dotnet-runtime-6.0
+  - docker
+sources:
+  - https://git.sr.ht/~cloutier/bird.makeup
+secrets:
+  -  d9970e85-5aef-4cfd-b6ed-0ccf1be5308b
+tasks:
+  - test: |
+      sudo systemctl start docker
+      sudo docker run -d -p 5432:5432 -e POSTGRES_PASSWORD=birdsitelive -e POSTGRES_USER=birdsitelive -e POSTGRES_DB=birdsitelive postgres:15
+      cd bird.makeup/src
+      dotnet test
+  - publish-arm: |
+      cd bird.makeup/src/BirdsiteLive
+      dotnet publish --os linux --arch arm64 /t:PublishContainer -c Release
+      docker tag cloutier/bird.makeup:1.0 cloutier/bird.makeup:latest-arm
+      docker push cloutier/bird.makeup:latest-arm
+  - publish-x64: |
+      cd bird.makeup/src/BirdsiteLive
+      dotnet publish --os linux --arch x64 /t:PublishContainer -c Release
+      docker tag cloutier/bird.makeup:1.0 cloutier/bird.makeup:latest
+      docker push cloutier/bird.makeup:latest

.drone.yml

@@ -0,0 +1,51 @@
+# kind: pipeline
+# name: testing
+# type: docker
+
+# steps:
+#   - name: Test
+#     image: mcr.microsoft.com/dotnet/sdk:6.0
+#     commands:
+#       - sed -i "s/127\.0\.0\.1/database/g" ./src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/Base/PostgresTestingBase.cs
+#       - dotnet test --verbosity minimal ./src
+
+# services:
+#   - name: database
+#     image: postgres:15
+#     environment:
+#       POSTGRES_USER: birdsitelive
+#       POSTGRES_PASSWORD: birdsitelive
+#       POSTGRES_DB: birdsitelive
+
+# ---
+kind: pipeline
+name: docker-publish
+type: docker
+
+# depends_on:
+#   - testing
+
+steps:
+  - name: Build & Publish
+    privileged: true
+    image: quay.io/thegeeklab/drone-docker-buildx
+    settings:
+      auto_tag: true
+      tags:
+        - makeup
+      repo: git.froth.zone/sam/birdsitelive
+      registry: git.froth.zone
+      username: sam
+      password:
+        from_secret: password
+      platforms:
+        - linux/amd64
+        - linux/arm64
+    when:
+      branch:
+        - master
+        - makeup
+      event:
+        - push
+    depends_on:
+      - "clone"

.gitignore

@@ -91,7 +91,6 @@ StyleCopReport.xml
 *.log
 *.vspscc
 *.vssscc
-.builds
 *.pidb
 *.svclog
 *.scc
@@ -346,9 +345,10 @@ ASALocalRun/
 # BeatPulse healthcheck temp database
 healthchecksdb
 
-# Backup folder for Package Reference Convert tool in Visual Studio 2017
-MigrationBackup/
-
 # Ionide (cross platform F# VS Code tools) working folder
 .ionide/
 /src/BSLManager/Properties/launchSettings.json
+
+backups
+
+.dccache

Dockerfile

@@ -1,16 +1,14 @@
-#See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging.
-
-FROM mcr.microsoft.com/dotnet/aspnet:3.1-buster-slim AS base
+FROM mcr.microsoft.com/dotnet/aspnet:6.0 AS base
 WORKDIR /app
 EXPOSE 80
 EXPOSE 443
 
-FROM mcr.microsoft.com/dotnet/sdk:3.1-buster AS publish
+FROM mcr.microsoft.com/dotnet/sdk:6.0 AS publish
 COPY ./src/ ./src/
 RUN dotnet publish "/src/BirdsiteLive/BirdsiteLive.csproj" -c Release -o /app/publish
-RUN dotnet publish "/src/BSLManager/BSLManager.csproj" -r linux-x64 --self-contained true -p:PublishSingleFile=true -p:IncludeAllContentForSelfExtract=true -c Release -o /app/publish
+# RUN dotnet publish "/src/BSLManager/BSLManager.csproj" -r linux-x64 --self-contained true -p:PublishSingleFile=true -p:IncludeAllContentForSelfExtract=true -c Release -o /app/publish
 
 FROM base AS final
 WORKDIR /app
 COPY --from=publish /app/publish .
-ENTRYPOINT ["dotnet", "BirdsiteLive.dll"]
+ENTRYPOINT ["dotnet", "BirdsiteLive.dll"]

INSTALLATION.md

@@ -1,12 +1,5 @@
 # Installation
 
-## Prerequisites 
-
-You will need a Twitter API key to make BirdsiteLIVE working. First create an **Standalone App** in the [Twitter developer portal](https://developer.twitter.com/en/portal/projects-and-apps) and retrieve the API Key and API Secret Key. 
-
-Please make sure you are using a **Standalone App** API Key and not a **Project App** API Key (that will NOT work with BirdsiteLIVE), if you don't see the **Standalone App** section, you might need to [apply for Elevated Access](https://developer.twitter.com/en/portal/products/elevated) as described in the [API documentation](https://developer.twitter.com/en/support/twitter-api/developer-account).
-
-
 ## Server prerequisites
 
 Your instance will need [docker](https://docs.docker.com/engine/install/) and [docker-compose](https://docs.docker.com/compose/install/) installed and working. 
@@ -31,8 +24,6 @@ sudo nano docker-compose.yml
 
 * `Instance:Domain` the domain name you'll be using, for example use `birdsite.live` for the URL `https://birdsite.live`
 * `Instance:AdminEmail` the admin's email, will be displayed in the instance /.well-known/nodeinfo endpoint
-* `Twitter:ConsumerKey` the Twitter API key
-* `Twitter:ConsumerSecret` the Twitter API secret key
 
 #### Database credentials
 

README.md

@@ -1,31 +1,44 @@
-![Test](https://github.com/NicolasConstant/BirdsiteLive/workflows/.NET%20Core/badge.svg?branch=master&event=push)
+# bird.makeup
 
-# BirdsiteLIVE
+[![builds.sr.ht status](https://builds.sr.ht/~cloutier/bird.makeup/commits/master/arch.yml.svg)](https://builds.sr.ht/~cloutier/bird.makeup/commits/master/arch.yml?)
 
 ## About
 
-BirdsiteLIVE is an ActivityPub bridge from Twitter, it's mostly a pet project/playground for me to handle ActivityPub concepts. Feel free to deploy your own instance (especially if you plan to follow a lot of users) since it use a proper Twitter API key and therefore will have limited calls ceiling (it won't scale, and it's by design).
+Bird.makeup is a way to follow Twitter users from any ActivityPub service. The aim is to make tweets appear as native a possible to the fediverse, while being as scalable as possible. The project started from BirdsiteLive, but has now been improved significantly. 
 
-## State of development
+Compared to BirdsiteLive, bird.makeup is:
 
-The code is pretty messy and far from a good state, since it's a playground for me the aim was to understand some AP concepts, not provide a good state-of-the-art codebase. But I might refactor it to make it cleaner. 
+More scalable:
+ - Twitter API calls are not rate-limited
+ - It is possible to split the Twitter crawling to multiple servers
+ - There are now integration tests for the non-official api
+ - The core pipeline has been tweaked to remove bottlenecks. As of writing this, bird.makeup supports without problems more than 20k users. 
+ - Twitter users with no followers on the fediverse will stop being fetched
+
+More native to the fediverse:
+ - Retweets are propagated as boosts
+ - Activities are now "unlisted" which means that they won't polute the public timeline, but they can still be boosted
+ - WIP support for QT
+
+More modern:
+ - Moved from .net core 3.1 to .net 6 which is still supported
+ - Moved from postgres 9 to 15
+ - Moved from Newtonsoft.Json to System.Text.Json
 
 ## Official instance 
 
-You can find an official (and temporary) instance here: [beta.birdsite.live](https://beta.birdsite.live). This instance can disapear at any time, if you want a long term instance you should install your own or use another one. 
+You can find the official instance here: [bird.makeup](https://bird.makeup). If you are an instance admin that prefers to not have tweets federated to you, please block the entire instance. 
 
-## Installation
-
-I'm providing a [docker build](https://hub.docker.com/r/nicolasconstant/birdsitelive). To install it on your own server, please follow [those instructions](https://github.com/NicolasConstant/BirdsiteLive/blob/master/INSTALLATION.md). More [options](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md) are also available.
-
-Also a [CLI](https://github.com/NicolasConstant/BirdsiteLive/blob/master/BSLManager.md) is available for adminitrative tasks.
+Please consider if you really need another instance before spinning up a new one, as having multiple domain makes it harder for moderators to block twitter content. 
 
 ## License
 
-This project is licensed under the AGPLv3 License - see [LICENSE](https://github.com/NicolasConstant/BirdsiteLive/blob/master/LICENSE) for details.
+Original code started from [BirdsiteLive](https://github.com/NicolasConstant/BirdsiteLive).
+
+This project is licensed under the AGPLv3 License - see [LICENSE](https://git.sr.ht/~cloutier/bird.makeup/tree/master/item/LICENSE) for details.
 
 ## Contact
 
-You can contact me via ActivityPub <a rel="me" href="https://fosstodon.org/@BirdsiteLIVE">here</a>.
+You can contact me via ActivityPub <a rel="me" href="https://social.librem.one/@vincent">here</a>.
 
 

docker-compose.yml

@@ -1,39 +1,40 @@
 version: "3"
 
-networks:
-    birdsitelivenetwork:
-        external: false
-
 services:
+
     server:
-        image: nicolasconstant/birdsitelive:latest
+        image: cloutier/bird.makeup:latest
         restart: always
-        container_name: birdsitelive
+        container_name: birdmakeup
         environment:
-            - Instance:Domain=domain.name
+            - Instance:Domain=bird.makeup
+            - Instance:Name=bird.makeup
             - Instance:AdminEmail=name@domain.ext
+            - Instance:ParallelTwitterRequests=50
+            - Instance:ParallelFediverseRequests=20
             - Db:Type=postgres
             - Db:Host=db
             - Db:Name=birdsitelive
             - Db:User=birdsitelive
             - Db:Password=birdsitelive
-            - Twitter:ConsumerKey=twitter.api.key
-            - Twitter:ConsumerSecret=twitter.api.key
-        networks:
-            - birdsitelivenetwork
+            - Moderation:FollowersBlackListing=bae.st
         ports:
             - "5000:80"
+        volumes:
+            - type: bind
+              source: ../key.json
+              target: /app/key.json
         depends_on:
             - db
-
     db:
-        image: postgres:9.6
+        image: postgres:15
         restart: always
         environment:
             - POSTGRES_USER=birdsitelive
             - POSTGRES_PASSWORD=birdsitelive
             - POSTGRES_DB=birdsitelive
-        networks:
-            - birdsitelivenetwork
         volumes:
-            - ./postgres:/var/lib/postgresql/data
+            - ../postgres15:/var/lib/postgresql/data
+        ports:
+            - "5432:5432"
+

sql.md

@@ -0,0 +1,36 @@
+
+# Most common servers
+
+```SQL
+SELECT COUNT(*), host FROM followers GROUP BY host ORDER BY count DESC;
+```
+
+# Most popular twitter users
+
+```SQL
+SELECT COUNT(*), acct FROM (SELECT unnest(followings) as follow FROM followers) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id GROUP BY acct ORDER BY count DESC;
+```
+
+```SQL
+SELECT COUNT(*), acct, id FROM (SELECT unnest(followings) as follow FROM followers) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id WHERE id IN ( SELECT unnest(followings) FROM followers WHERE host='social.librem.one' AND acct = 'vincent' ) GROUP BY acct, id ORDER BY count DESC;
+```
+
+# Most active users
+
+```SQL
+SELECT array_length(followings, 1) AS l, acct, host FROM followers ORDER BY l DESC;
+```
+
+# Lag
+
+```SQL
+SELECT COUNT(*), date_trunc('day', lastsync) FROM (SELECT unnest(followings) as follow FROM followers GROUP BY follow) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id GROUP BY date_trunc;
+
+SELECT COUNT(*), date_trunc('hour', lastsync) FROM (SELECT unnest(followings) as follow FROM followers GROUP BY follow) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id GROUP BY date_trunc ORDER BY date_trunc;
+```
+
+# Connections 
+
+```SQL
+SELECT SUM(cardinality(followings)) FROM followers;
+```

src/BSLManager/App.cs

@@ -1,252 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Reflection;
-using System.Threading.Tasks;
-using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.DAL.Models;
-using BirdsiteLive.Moderation.Actions;
-using BSLManager.Domain;
-using BSLManager.Tools;
-using Terminal.Gui;
-
-namespace BSLManager
-{
-    public class App
-    {
-        private readonly IFollowersDal _followersDal;
-        private readonly IRemoveFollowerAction _removeFollowerAction;
-
-        private readonly FollowersListState _state = new FollowersListState();
-
-        #region Ctor
-        public App(IFollowersDal followersDal, IRemoveFollowerAction removeFollowerAction)
-        {
-            _followersDal = followersDal;
-            _removeFollowerAction = removeFollowerAction;
-        }
-        #endregion
-
-        public void Run()
-        {
-            Application.Init();
-            var top = Application.Top;
-
-            // Creates the top-level window to show
-            var win = new Window("BSL Manager")
-            {
-                X = 0,
-                Y = 1, // Leave one row for the toplevel menu
-
-                // By using Dim.Fill(), it will automatically resize without manual intervention
-                Width = Dim.Fill(),
-                Height = Dim.Fill()
-            };
-
-            top.Add(win);
-
-            // Creates a menubar, the item "New" has a help menu.
-            var menu = new MenuBar(new MenuBarItem[]
-            {
-                new MenuBarItem("_File", new MenuItem[]
-                {
-                    new MenuItem("_Quit", "", () =>
-                    {
-                        if (Quit()) top.Running = false;
-                    })
-                }),
-                //new MenuBarItem ("_Edit", new MenuItem [] {
-                //    new MenuItem ("_Copy", "", null),
-                //    new MenuItem ("C_ut", "", null),
-                //    new MenuItem ("_Paste", "", null)
-                //})
-            });
-            top.Add(menu);
-
-            static bool Quit()
-            {
-                var n = MessageBox.Query(50, 7, "Quit BSL Manager", "Are you sure you want to quit?", "Yes", "No");
-                return n == 0;
-            }
-            
-            RetrieveUserList();
-
-            var list = new ListView(_state.GetDisplayableList())
-            {
-                X = 1,
-                Y = 3,
-                Width = Dim.Fill(),
-                Height = Dim.Fill()
-            };
-
-            list.KeyDown += _ =>
-            {
-                if (_.KeyEvent.Key == Key.Enter)
-                {
-                    OpenFollowerDialog(list.SelectedItem);
-                } 
-                else if (_.KeyEvent.Key == Key.Delete
-                           || _.KeyEvent.Key == Key.DeleteChar
-                           || _.KeyEvent.Key == Key.Backspace
-                           || _.KeyEvent.Key == Key.D)
-                {
-                    OpenDeleteDialog(list.SelectedItem);
-                }
-            };
-
-            var listingFollowersLabel = new Label(1, 0, "Listing followers");
-            var filterLabel = new Label("Filter: ") { X = 1, Y = 1 };
-            var filterText = new TextField("")
-            {
-                X = Pos.Right(filterLabel),
-                Y = 1,
-                Width = 40
-            };
-
-            filterText.KeyDown += _ =>
-            {
-                var text = filterText.Text.ToString();
-                if (_.KeyEvent.Key == Key.Enter && !string.IsNullOrWhiteSpace(text))
-                {
-                    _state.FilterBy(text);
-                    ConsoleGui.RefreshUI();
-                }
-            };
-
-            win.Add(
-                listingFollowersLabel,
-                filterLabel,
-                filterText,
-                list
-            );
-
-            Application.Run();
-        }
-
-        private void OpenFollowerDialog(int selectedIndex)
-        {
-            var close = new Button(3, 14, "Close");
-            close.Clicked += () => Application.RequestStop();
-
-            var dialog = new Dialog("Info", 60, 18, close);
-
-            var follower = _state.GetElementAt(selectedIndex);
-
-            var name = new Label($"User: @{follower.Acct}@{follower.Host}")
-            {
-                X = 1,
-                Y = 1,
-                Width = Dim.Fill(),
-                Height = 1
-            };
-            var following = new Label($"Following Count: {follower.Followings.Count}")
-            {
-                X = 1,
-                Y = 3,
-                Width = Dim.Fill(),
-                Height = 1
-            };
-            var errors = new Label($"Posting Errors: {follower.PostingErrorCount}")
-            {
-                X = 1,
-                Y = 4,
-                Width = Dim.Fill(),
-                Height = 1
-            };
-            var inbox = new Label($"Inbox: {follower.InboxRoute}")
-            {
-                X = 1,
-                Y = 5,
-                Width = Dim.Fill(),
-                Height = 1
-            };
-            var sharedInbox = new Label($"Shared Inbox: {follower.SharedInboxRoute}")
-            {
-                X = 1,
-                Y = 6,
-                Width = Dim.Fill(),
-                Height = 1
-            };
-
-            dialog.Add(name);
-            dialog.Add(following);
-            dialog.Add(errors);
-            dialog.Add(inbox);
-            dialog.Add(sharedInbox);
-            dialog.Add(close);
-            Application.Run(dialog);
-        }
-
-        private void OpenDeleteDialog(int selectedIndex)
-        {
-            bool okpressed = false;
-            var ok = new Button(10, 14, "Yes");
-            ok.Clicked += () =>
-            {
-                Application.RequestStop();
-                okpressed = true;
-            };
-
-            var cancel = new Button(3, 14, "No");
-            cancel.Clicked += () => Application.RequestStop();
-
-            var dialog = new Dialog("Delete", 60, 18, cancel, ok);
-
-            var follower = _state.GetElementAt(selectedIndex);
-            var name = new Label($"User: @{follower.Acct}@{follower.Host}")
-            {
-                X = 1,
-                Y = 1,
-                Width = Dim.Fill(),
-                Height = 1
-            };
-            var entry = new Label("Delete user and remove all their followings?")
-            {
-                X = 1,
-                Y = 3,
-                Width = Dim.Fill(),
-                Height = 1
-            };
-            dialog.Add(name);
-            dialog.Add(entry);
-            Application.Run(dialog);
-
-            if (okpressed)
-            {
-                DeleteAndRemoveUser(selectedIndex);
-            }
-        }
-
-        private void DeleteAndRemoveUser(int el)
-        {
-            Application.MainLoop.Invoke(async () =>
-            {
-                try
-                {
-                    var userToDelete = _state.GetElementAt(el);
-                    
-                    BasicLogger.Log($"Delete {userToDelete.Acct}@{userToDelete.Host}");
-                    await _removeFollowerAction.ProcessAsync(userToDelete);
-                    BasicLogger.Log($"Remove user from list");
-                    _state.RemoveAt(el);
-                }
-                catch (Exception e)
-                {
-                    BasicLogger.Log(e.Message);
-                }
-
-                ConsoleGui.RefreshUI();
-            });
-        }
-
-        private void RetrieveUserList()
-        {
-            Application.MainLoop.Invoke(async () =>
-            {
-                var followers = await _followersDal.GetAllFollowersAsync();
-                _state.Load(followers.ToList());
-                ConsoleGui.RefreshUI();
-            });
-        }
-    }
-}

src/BSLManager/BSLManager.csproj

@@ -1,28 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="Lamar" Version="5.0.3" />
-    <PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="5.0.0" />
-    <PackageReference Include="Terminal.Gui" Version="1.0.0-beta.11" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\BirdsiteLive.Common\BirdsiteLive.Common.csproj" />
-    <ProjectReference Include="..\BirdsiteLive.Moderation\BirdsiteLive.Moderation.csproj" />
-    <ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL.Postgres\BirdsiteLive.DAL.Postgres.csproj" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <None Update="key.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-  </ItemGroup>
-
-</Project>

src/BSLManager/Bootstrapper.cs

@@ -1,94 +0,0 @@
-using System;
-using System.Net.Http;
-using BirdsiteLive.Common.Settings;
-using BirdsiteLive.Common.Structs;
-using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.DAL.Postgres.DataAccessLayers;
-using BirdsiteLive.DAL.Postgres.Settings;
-using Lamar;
-using Lamar.Scanning.Conventions;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
-
-namespace BSLManager
-{
-    public class Bootstrapper
-    {
-        private readonly DbSettings _dbSettings;
-        private readonly InstanceSettings _instanceSettings;
-
-        #region Ctor
-        public Bootstrapper(DbSettings dbSettings, InstanceSettings instanceSettings)
-        {
-            _dbSettings = dbSettings;
-            _instanceSettings = instanceSettings;
-        }
-        #endregion
-
-        public Container Init()
-        {
-            var container = new Container(x =>
-            {
-                x.For<DbSettings>().Use(x => _dbSettings);
-
-                x.For<InstanceSettings>().Use(x => _instanceSettings);
-
-                if (string.Equals(_dbSettings.Type, DbTypes.Postgres, StringComparison.OrdinalIgnoreCase))
-                {
-                    var connString = $"Host={_dbSettings.Host};Username={_dbSettings.User};Password={_dbSettings.Password};Database={_dbSettings.Name}";
-                    var postgresSettings = new PostgresSettings
-                    {
-                        ConnString = connString
-                    };
-                    x.For<PostgresSettings>().Use(x => postgresSettings);
-
-                    x.For<ITwitterUserDal>().Use<TwitterUserPostgresDal>().Singleton();
-                    x.For<IFollowersDal>().Use<FollowersPostgresDal>().Singleton();
-                    x.For<IDbInitializerDal>().Use<DbInitializerPostgresDal>().Singleton();
-                }
-                else
-                {
-                    throw new NotImplementedException($"{_dbSettings.Type} is not supported");
-                }
-
-                var serviceProvider = new ServiceCollection().AddHttpClient().BuildServiceProvider();
-                x.For<IHttpClientFactory>().Use(_ => serviceProvider.GetService<IHttpClientFactory>());
-
-                x.For(typeof(ILogger<>)).Use(typeof(DummyLogger<>));
-
-                x.Scan(_ =>
-                {
-                    _.Assembly("BirdsiteLive.Twitter");
-                    _.Assembly("BirdsiteLive.Domain");
-                    _.Assembly("BirdsiteLive.DAL");
-                    _.Assembly("BirdsiteLive.DAL.Postgres");
-                    _.Assembly("BirdsiteLive.Moderation");
-
-                    _.TheCallingAssembly();
-
-                    _.WithDefaultConventions();
-
-                    _.LookForRegistries();
-                });
-            });
-            return container;
-        }
-
-        public class DummyLogger<T> : ILogger<T>
-        {
-            public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter)
-            {
-            }
-
-            public bool IsEnabled(LogLevel logLevel)
-            {
-                return false;
-            }
-
-            public IDisposable BeginScope<TState>(TState state)
-            {
-                return null;
-            }
-        }
-    }
-}

src/BSLManager/Domain/FollowersListState.cs

@@ -1,81 +0,0 @@
-using System.Collections.Generic;
-using System.Linq;
-using BirdsiteLive.DAL.Models;
-
-namespace BSLManager.Domain
-{
-    public class FollowersListState
-    {
-        private readonly List<string> _filteredDisplayableUserList = new List<string>();
-
-        private List<Follower> _sourceUserList = new List<Follower>();
-        private List<Follower> _filteredSourceUserList = new List<Follower>();
-        
-        public void Load(List<Follower> followers)
-        {
-            _sourceUserList = followers.OrderByDescending(x => x.Followings.Count).ToList();
-           
-            ResetLists();
-        }
-
-        private void ResetLists()
-        {
-            _filteredSourceUserList = _sourceUserList.ToList();
-
-            _filteredDisplayableUserList.Clear();
-
-            foreach (var follower in _sourceUserList)
-            {
-                var displayedUser = $"{GetFullHandle(follower)}   ({follower.Followings.Count})  (err:{follower.PostingErrorCount})";
-                _filteredDisplayableUserList.Add(displayedUser);
-            }
-        }
-
-        public List<string> GetDisplayableList()
-        {
-            return _filteredDisplayableUserList;
-        }
-
-        public void FilterBy(string pattern)
-        {
-            ResetLists();
-
-            if (!string.IsNullOrWhiteSpace(pattern))
-            {
-                var elToRemove = _filteredSourceUserList
-                    .Where(x => !GetFullHandle(x).Contains(pattern))
-                    .Select(x => x)
-                    .ToList();
-
-                foreach (var el in elToRemove)
-                {
-                    _filteredSourceUserList.Remove(el);
-                    
-                    var dElToRemove = _filteredDisplayableUserList.First(x => x.Contains(GetFullHandle(el)));
-                    _filteredDisplayableUserList.Remove(dElToRemove);
-                }
-            }
-        }
-
-        private string GetFullHandle(Follower follower)
-        {
-            return $"@{follower.Acct}@{follower.Host}";
-        }
-
-        public void RemoveAt(int index)
-        {
-            var displayableUser = _filteredDisplayableUserList[index];
-            var sourceUser = _filteredSourceUserList[index];
-
-            _filteredDisplayableUserList.Remove(displayableUser);
-
-            _filteredSourceUserList.Remove(sourceUser);
-            _sourceUserList.Remove(sourceUser);
-        }
-
-        public Follower GetElementAt(int index)
-        {
-            return _filteredSourceUserList[index];
-        }
-    }
-}

src/BSLManager/Program.cs

@@ -1,39 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Reflection;
-using System.Text;
-using System.Threading.Tasks;
-using BirdsiteLive.Common.Settings;
-using BirdsiteLive.DAL.Contracts;
-using BSLManager.Tools;
-using Microsoft.Extensions.Configuration;
-using NStack;
-using Terminal.Gui;
-
-namespace BSLManager
-{
-    class Program
-    {
-        static async Task Main(string[] args)
-        {
-            Console.OutputEncoding = Encoding.Default;
-
-            var settingsManager = new SettingsManager();
-            var settings = settingsManager.GetSettings();
-
-            //var builder = new ConfigurationBuilder()
-            //    .AddEnvironmentVariables();
-            //var configuration = builder.Build();
-
-            //var dbSettings = configuration.GetSection("Db").Get<DbSettings>();
-            //var instanceSettings = configuration.GetSection("Instance").Get<InstanceSettings>();
-
-            var bootstrapper = new Bootstrapper(settings.dbSettings, settings.instanceSettings);
-            var container = bootstrapper.Init();
-
-            var app = container.GetInstance<App>();
-            app.Run();
-        }
-    }
-}

src/BSLManager/Tools/BasicLogger.cs

@@ -1,13 +0,0 @@
-using System;
-using System.IO;
-
-namespace BSLManager.Tools
-{
-    public static class BasicLogger
-    {
-        public static void Log(string log)
-        {
-            File.AppendAllLines($"Log-{Guid.NewGuid()}.txt", new []{ log });
-        }
-    }
-}

src/BSLManager/Tools/ConsoleGui.cs

@@ -1,15 +0,0 @@
-using System.Reflection;
-using Terminal.Gui;
-
-namespace BSLManager.Tools
-{
-    public static class ConsoleGui
-    {
-        public static void RefreshUI()
-        {
-            typeof(Application)
-                .GetMethod("TerminalResized", BindingFlags.Static | BindingFlags.NonPublic)
-                .Invoke(null, null);
-        }
-    }
-}

src/BSLManager/Tools/SettingsManager.cs

@@ -1,124 +0,0 @@
-using System;
-using System.IO;
-using System.Runtime.CompilerServices;
-using BirdsiteLive.Common.Settings;
-using Newtonsoft.Json;
-using Org.BouncyCastle.Asn1.IsisMtt.X509;
-
-namespace BSLManager.Tools
-{
-    public class SettingsManager
-    {
-        private const string LocalFileName = "ManagerSettings.json";
-
-        public (DbSettings dbSettings, InstanceSettings instanceSettings) GetSettings()
-        {
-            var localSettingsData = GetLocalSettingsFile();
-            if (localSettingsData != null) return Convert(localSettingsData);
-
-            Console.WriteLine("We need to set up the manager");
-            Console.WriteLine("Please provide the following information as provided in the docker-compose file");
-
-            LocalSettingsData data;
-            do
-            {
-                data = GetDataFromUser();
-                Console.WriteLine();
-                Console.WriteLine("Please check if all is ok:");
-                Console.WriteLine();
-                Console.WriteLine($"Db Host: {data.DbHost}");
-                Console.WriteLine($"Db Name: {data.DbName}");
-                Console.WriteLine($"Db User: {data.DbUser}");
-                Console.WriteLine($"Db Password: {data.DbPassword}");
-                Console.WriteLine($"Instance Domain: {data.InstanceDomain}");
-                Console.WriteLine();
-
-                string resp;
-                do
-                {
-                    Console.WriteLine("Is it valid? (yes, no)");
-                    resp = Console.ReadLine()?.Trim().ToLowerInvariant();
-
-                    if (resp == "n" || resp == "no") data = null;
-
-                } while (resp != "y" && resp != "yes" && resp != "n" && resp != "no");
-
-            } while (data == null);
-            
-            SaveLocalSettings(data);
-            return Convert(data);
-        }
-
-        private LocalSettingsData GetDataFromUser()
-        {
-            var data = new LocalSettingsData();
-
-            Console.WriteLine("Db Host:");
-            data.DbHost = Console.ReadLine();
-
-            Console.WriteLine("Db Name:");
-            data.DbName = Console.ReadLine();
-
-            Console.WriteLine("Db User:");
-            data.DbUser = Console.ReadLine();
-
-            Console.WriteLine("Db Password:");
-            data.DbPassword = Console.ReadLine();
-
-            Console.WriteLine("Instance Domain:");
-            data.InstanceDomain = Console.ReadLine();
-
-            return data;
-        }
-
-        private (DbSettings dbSettings, InstanceSettings instanceSettings) Convert(LocalSettingsData data)
-        {
-            var dbSettings = new DbSettings
-            {
-                Type = data.DbType,
-                Host = data.DbHost,
-                Name = data.DbName,
-                User = data.DbUser,
-                Password = data.DbPassword
-            };
-            var instancesSettings = new InstanceSettings
-            {
-                Domain = data.InstanceDomain
-            };
-            return (dbSettings, instancesSettings);
-        }
-
-        private LocalSettingsData GetLocalSettingsFile()
-        {
-            try
-            {
-                if (!File.Exists(LocalFileName)) return null;
-
-                var jsonContent = File.ReadAllText(LocalFileName);
-                var content = JsonConvert.DeserializeObject<LocalSettingsData>(jsonContent);
-                return content;
-            }
-            catch (Exception)
-            {
-                return null;
-            }
-        }
-
-        private void SaveLocalSettings(LocalSettingsData data)
-        {
-            var jsonContent = JsonConvert.SerializeObject(data);
-            File.WriteAllText(LocalFileName, jsonContent);
-        }
-    }
-
-    internal class LocalSettingsData
-    {
-        public string DbType { get; set; } = "postgres";
-        public string DbHost { get; set; }
-        public string DbName { get; set; }
-        public string DbUser { get; set; }
-        public string DbPassword { get; set; }
-
-        public string InstanceDomain { get; set; }
-    }
-}

src/BirdsiteLive.ActivityPub/ApDeserializer.cs

@@ -1,6 +1,7 @@
 using System;
 using BirdsiteLive.ActivityPub.Models;
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
+using System.Text.Json;
 
 namespace BirdsiteLive.ActivityPub
 {
@@ -10,22 +11,23 @@ namespace BirdsiteLive.ActivityPub
         {
             try
             {
-                var activity = JsonConvert.DeserializeObject<Activity>(json);
+                Console.WriteLine("DEBUG: JSON");
+                Console.WriteLine(json);
+                var activity = JsonSerializer.Deserialize<Activity>(json);
                 switch (activity.type)
                 {
                     case "Follow":
-                        return JsonConvert.DeserializeObject<ActivityFollow>(json);
+                        return JsonSerializer.Deserialize<ActivityFollow>(json);
                     case "Undo":
-                        var a = JsonConvert.DeserializeObject<ActivityUndo>(json);
+                        var a = JsonSerializer.Deserialize<ActivityUndo>(json);
                         if(a.apObject.type == "Follow")
-                            return JsonConvert.DeserializeObject<ActivityUndoFollow>(json);
+                            return JsonSerializer.Deserialize<ActivityUndoFollow>(json);
                         break;
                     case "Delete":
-                        return JsonConvert.DeserializeObject<ActivityDelete>(json);
+                        return JsonSerializer.Deserialize<ActivityDelete>(json);
                     case "Accept":
-                        var accept = JsonConvert.DeserializeObject<ActivityAccept>(json);
-                        //var acceptType = JsonConvert.DeserializeObject<Activity>(accept.apObject);
-                        switch ((accept.apObject as dynamic).type.ToString())
+                        var accept = JsonSerializer.Deserialize<ActivityAccept>(json);
+                        switch (accept.apObject.type)
                         {
                             case "Follow":
                                 var acceptFollow = new ActivityAcceptFollow()
@@ -36,11 +38,12 @@ namespace BirdsiteLive.ActivityPub
                                     context = accept.context,
                                     apObject = new ActivityFollow()
                                     {
-                                        id = (accept.apObject as dynamic).id?.ToString(),
-                                        type = (accept.apObject as dynamic).type?.ToString(),
-                                        actor = (accept.apObject as dynamic).actor?.ToString(),
-                                        context = (accept.apObject as dynamic).context?.ToString(),
-                                        apObject = (accept.apObject as dynamic).@object?.ToString()
+
+                                        id = accept.apObject.id,
+                                        type = accept.apObject.type,
+                                        actor = accept.apObject.actor,
+                                        context = accept.apObject.context?.ToString(),
+                                        apObject = accept.apObject.apObject,
                                     }
                                 };
                                 return acceptFollow;
@@ -52,14 +55,7 @@ namespace BirdsiteLive.ActivityPub
             {
                 Console.WriteLine(e);
             }
-
             return null;
         }
-
-        private class Ac : Activity
-        {
-            [JsonProperty("object")]
-            public Activity apObject { get; set; }
-        }
     }
 }

src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj

@@ -1,13 +1,11 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
-    <PackageReference Include="System.Text.Json" Version="4.7.2" />
   </ItemGroup>
 
 </Project>

src/BirdsiteLive.ActivityPub/Converters/ContextArrayConverter.cs

@@ -1,39 +0,0 @@
-using System;
-using System.Collections.Generic;
-using Newtonsoft.Json;
-
-namespace BirdsiteLive.ActivityPub.Converters
-{
-    public class ContextArrayConverter : JsonConverter
-    {
-        public override bool CanWrite { get { return false; } }
-        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
-        {
-            throw new NotImplementedException();
-        }
-
-        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
-        {
-            var result = new List<string>();
-
-            var list = serializer.Deserialize<List<object>>(reader);
-            foreach (var l in list)
-            {
-                if (l is string s)
-                    result.Add(s);
-                else
-                {
-                    var str = JsonConvert.SerializeObject(l);
-                    result.Add(str);
-                }
-            }
-
-            return result.ToArray();
-        }
-
-        public override bool CanConvert(Type objectType)
-        {
-            throw new NotImplementedException();
-        }
-    }
-}

src/BirdsiteLive.ActivityPub/Models/Activity.cs

@@ -1,17 +1,17 @@
-using System.Text.Json.Serialization;
-using Newtonsoft.Json;
+using System.Collections.Generic;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class Activity
     {
-        [JsonProperty("@context")]
+        [JsonPropertyName("@context")]
         public object context { get; set; }
         public string id { get; set; }
         public string type { get; set; }
         public string actor { get; set; }
 
-        //[JsonProperty("object")]
+        //[JsonPropertyName("object")]
         //public string apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityAccept.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityAccept : Activity
     {
-        [JsonProperty("object")]
-        public object apObject { get; set; }
+        [JsonPropertyName("object")]
+        public NestedActivity apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityAcceptFollow.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityAcceptFollow : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public ActivityFollow apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityAcceptUndoFollow.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityAcceptUndoFollow : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public ActivityUndoFollow apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityCreateNote.cs

@@ -1,6 +1,5 @@
-using System;
-using BirdsiteLive.ActivityPub.Models;
-using Newtonsoft.Json;
+using BirdsiteLive.ActivityPub.Models;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
@@ -10,7 +9,7 @@ namespace BirdsiteLive.ActivityPub
         public string[] to { get; set; }
         public string[] cc { get; set; }
 
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public Note apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub.Models
 {
     public class ActivityDelete : Activity
     {
-        [JsonProperty("object")]
-        public object apObject { get; set; }
+        [JsonPropertyName("object")]
+        public string apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityFollow.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityFollow : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public string apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityRejectFollow.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityRejectFollow : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public ActivityFollow apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityUndo.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityUndo : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public Activity apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityUndoFollow.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityUndoFollow : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public ActivityFollow apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/Actor.cs

@@ -1,15 +1,13 @@
 using System.Net;
 using BirdsiteLive.ActivityPub.Converters;
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class Actor
     {
-        //[JsonPropertyName("@context")]
-        [JsonProperty("@context")]
-        [JsonConverter(typeof(ContextArrayConverter))]
-        public string[] context { get; set; } = new[] { "https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1" };
+        [JsonPropertyName("@context")]
+        public object[] context { get; set; } = new string[] { "https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1" };
         public string id { get; set; }
         public string type { get; set; }
         public string followers { get; set; }

src/BirdsiteLive.ActivityPub/Models/Followers.cs

@@ -1,12 +1,11 @@
 using BirdsiteLive.ActivityPub.Converters;
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub.Models
 {
     public class Followers
     {
-        [JsonProperty("@context")]
-        [JsonConverter(typeof(ContextArrayConverter))]
+        [JsonPropertyName("@context")]
         public string context { get; set; } = "https://www.w3.org/ns/activitystreams";
 
         public string id { get; set; }

src/BirdsiteLive.ActivityPub/Models/NestedActivity.cs

@@ -0,0 +1,17 @@
+using System.Text.Json.Serialization;
+
+namespace BirdsiteLive.ActivityPub
+{
+    public class NestedActivity
+    {
+        [JsonPropertyName("@context")]
+        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+        public object context { get; set; }
+        public string id { get; set; }
+        public string type { get; set; }
+        public string actor { get; set; }
+
+        [JsonPropertyName("object")]
+        public string apObject { get; set; }
+    }
+}

src/BirdsiteLive.ActivityPub/Models/Note.cs

@@ -1,15 +1,14 @@
-using BirdsiteLive.ActivityPub.Converters;
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub.Models
 {
     public class Note
     {
-        [JsonProperty("@context")]
-        [JsonConverter(typeof(ContextArrayConverter))]
+        [JsonPropertyName("@context")]
         public string[] context { get; set; } = new[] { "https://www.w3.org/ns/activitystreams" };
 
         public string id { get; set; }
+        public string announceId { get; set; }
         public string type { get; } = "Note";
         public string summary { get; set; }
         public string inReplyTo { get; set; }
@@ -24,6 +23,8 @@ namespace BirdsiteLive.ActivityPub.Models
         //public Dictionary<string,string> contentMap { get; set; }
         public Attachment[] attachment { get; set; }
         public Tag[] tag { get; set; }
-        //public Dictionary<string, string> replies;
+		//public Dictionary<string, string> replies;
+
+		public string quoteUrl { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/WebFingerData.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BirdsiteLive.ActivityPub.Models
+{
+    public class WebFingerData
+    {
+        public List<string> aliases { get; set; }
+
+        public List<WebFingerLink> links { get; set; }
+    }
+
+    public class WebFingerLink
+    {
+        public string href { get; set; }
+        public string rel { get; set; }
+        public string type { get; set; }
+        public string template { get; set; }
+    }
+}

src/BirdsiteLive.Common/BirdsiteLive.Common.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
 </Project>

src/BirdsiteLive.Common/Regexes/UserRegexes.cs

@@ -5,6 +5,6 @@ namespace BirdsiteLive.Common.Regexes
     public class UserRegexes
     {
         public static readonly Regex TwitterAccount = new Regex(@"^[a-zA-Z0-9_]+$");
-        public static readonly Regex Mention = new Regex(@"(.?)@([a-zA-Z0-9_]+)(\s|$|[\[\]<>,;:!?/|-]|(. ))");
+        public static readonly Regex Mention = new Regex(@"(.?)@([a-zA-Z0-9_]+)(\s|$|[\[\]<>,;:'\.’!?/—\|-]|(. ))");
     }
 }

src/BirdsiteLive.Common/Settings/InstanceSettings.cs

@@ -7,7 +7,6 @@
         public string AdminEmail { get; set; }
         public bool ResolveMentionsInProfiles { get; set; }
         public bool PublishReplies { get; set; }
-        public int MaxUsersCapacity { get; set; }
 
         public string UnlistedTwitterAccounts { get; set; }
         public string SensitiveTwitterAccounts { get; set; }
@@ -15,6 +14,14 @@
         public int FailingTwitterUserCleanUpThreshold { get; set; }
         public int FailingFollowerCleanUpThreshold { get; set; } = -1;
 
-        public int UserCacheCapacity { get; set; }
+        public int UserCacheCapacity { get; set; } = 40_000;
+        public int TweetCacheCapacity { get; set; } = 20_000;
+        // "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
+        public string TwitterBearerToken { get; set; } = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA";
+        public int m { get; set; } = 1;
+        public int n_start { get; set; } = 0;
+        public int n_end { get; set; } = 1;
+        public int ParallelTwitterRequests { get; set; } = 10;
+        public int ParallelFediversePosts { get; set; } = 10;
     }
 }

src/BirdsiteLive.Common/Settings/TwitterSettings.cs

@@ -1,8 +0,0 @@
-namespace BirdsiteLive.Common.Settings
-{
-    public class TwitterSettings
-    {
-        public string ConsumerKey { get; set; }
-        public string ConsumerSecret { get; set; }
-    }
-}

src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj

@@ -1,13 +1,11 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Asn1" Version="1.0.9" />
-    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
-    <PackageReference Include="Portable.BouncyCastle" Version="1.8.6.7" />
   </ItemGroup>
 
 </Project>

src/BirdsiteLive.Cryptography/MagicKey.cs

@@ -1,28 +1,12 @@
 using System;
-using System.Collections.Generic;
 using System.Security.Cryptography;
 using System.Text;
-using Newtonsoft.Json;
+using System.Text.Json;
 
 namespace BirdsiteLive.Cryptography
 {
     public class MagicKey
     {
-        //public class WebfingerLink
-        //{
-        //    public string rel { get; set; }
-        //    public string type { get; set; }
-        //    public string href { get; set; }
-        //    public string template { get; set; }
-        //}
-
-        //public class WebfingerResult
-        //{
-        //    public string subject { get; set; }
-        //    public List<string> aliases { get; set; }
-        //    public List<WebfingerLink> links { get; set; }
-        //}
-
         private string[] _parts;
         private RSA _rsa;
 
@@ -38,14 +22,14 @@ namespace BirdsiteLive.Cryptography
 
         private class RSAKeyParms
         {
-            public byte[] D;
-            public byte[] DP;
-            public byte[] DQ;
-            public byte[] Exponent;
-            public byte[] InverseQ;
-            public byte[] Modulus;
-            public byte[] P;
-            public byte[] Q;
+            public byte[] D { get; set; }
+            public byte[] DP {get; set; }
+            public byte[] DQ {get; set; }
+            public byte[] Exponent {get; set; }
+            public byte[] InverseQ {get; set; }
+            public byte[] Modulus {get; set; }
+            public byte[] P {get; set; }
+            public byte[] Q {get; set; }
 
             public static RSAKeyParms From(RSAParameters parms)
             {
@@ -81,7 +65,9 @@ namespace BirdsiteLive.Cryptography
             if (key[0] == '{')
             {
                 _rsa = RSA.Create();
-                _rsa.ImportParameters(JsonConvert.DeserializeObject<RSAKeyParms>(key).Make());
+                Console.WriteLine(key);
+                Console.WriteLine(JsonSerializer.Deserialize<RSAKeyParms>(key).Make());
+                _rsa.ImportParameters(JsonSerializer.Deserialize<RSAKeyParms>(key).Make());
             }
             else
             {
@@ -102,7 +88,7 @@ namespace BirdsiteLive.Cryptography
             var rsa = RSA.Create();
             rsa.KeySize = 2048;
 
-            return new MagicKey(JsonConvert.SerializeObject(RSAKeyParms.From(rsa.ExportParameters(true))));
+            return new MagicKey(JsonSerializer.Serialize<RSAKeyParms>(RSAKeyParms.From(rsa.ExportParameters(true))));
         }
 
         public byte[] BuildSignedData(string data, string dataType, string encoding, string algorithm)
@@ -140,7 +126,7 @@ namespace BirdsiteLive.Cryptography
 
         public string PrivateKey
         {
-            get { return JsonConvert.SerializeObject(RSAKeyParms.From(_rsa.ExportParameters(true))); }
+            get { return JsonSerializer.Serialize(RSAKeyParms.From(_rsa.ExportParameters(true))); }
         }
 
         public string PublicKey

src/BirdsiteLive.Cryptography/RsaGenerator.cs

@@ -1,99 +0,0 @@
-using System;
-using System.IO;
-using System.Security.Cryptography;
-
-namespace BirdsiteLive.Cryptography
-{
-    //https://gist.github.com/ststeiger/f4b29a140b1e3fd618679f89b7f3ff4a
-    //https://gist.github.com/valep27/4a720c25b35fff83fbf872516f847863
-    //https://gist.github.com/therightstuff/aa65356e95f8d0aae888e9f61aa29414
-    //https://stackoverflow.com/questions/52468125/export-rsa-public-key-in-der-format-and-decrypt-data
-    public class RsaGenerator
-    {
-        public string GetRsa()
-        {
-            var rsa = RSA.Create();
-            
-            var outputStream = new StringWriter();
-            var parameters = rsa.ExportParameters(true);
-            using (var stream = new MemoryStream())
-            {
-                var writer = new BinaryWriter(stream);
-                writer.Write((byte)0x30); // SEQUENCE
-                using (var innerStream = new MemoryStream())
-                {
-                    var innerWriter = new BinaryWriter(innerStream);
-                    innerWriter.Write((byte)0x30); // SEQUENCE
-                    EncodeLength(innerWriter, 13);
-                    innerWriter.Write((byte)0x06); // OBJECT IDENTIFIER
-                    var rsaEncryptionOid = new byte[] { 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01 };
-                    EncodeLength(innerWriter, rsaEncryptionOid.Length);
-                    innerWriter.Write(rsaEncryptionOid);
-                    innerWriter.Write((byte)0x05); // NULL
-                    EncodeLength(innerWriter, 0);
-                    innerWriter.Write((byte)0x03); // BIT STRING
-                    using (var bitStringStream = new MemoryStream())
-                    {
-                        var bitStringWriter = new BinaryWriter(bitStringStream);
-                        bitStringWriter.Write((byte)0x00); // # of unused bits
-                        bitStringWriter.Write((byte)0x30); // SEQUENCE
-                        using (var paramsStream = new MemoryStream())
-                        {
-                            var paramsWriter = new BinaryWriter(paramsStream);
-                            //EncodeIntegerBigEndian(paramsWriter, parameters.Modulus); // Modulus
-                            //EncodeIntegerBigEndian(paramsWriter, parameters.Exponent); // Exponent
-                            var paramsLength = (int)paramsStream.Length;
-                            EncodeLength(bitStringWriter, paramsLength);
-                            bitStringWriter.Write(paramsStream.GetBuffer(), 0, paramsLength);
-                        }
-                        var bitStringLength = (int)bitStringStream.Length;
-                        EncodeLength(innerWriter, bitStringLength);
-                        innerWriter.Write(bitStringStream.GetBuffer(), 0, bitStringLength);
-                    }
-                    var length = (int)innerStream.Length;
-                    EncodeLength(writer, length);
-                    writer.Write(innerStream.GetBuffer(), 0, length);
-                }
-
-                var base64 = Convert.ToBase64String(stream.GetBuffer(), 0, (int)stream.Length).ToCharArray();
-                // WriteLine terminates with \r\n, we want only \n
-                outputStream.Write("-----BEGIN PUBLIC KEY-----\n");
-                for (var i = 0; i < base64.Length; i += 64)
-                {
-                    outputStream.Write(base64, i, Math.Min(64, base64.Length - i));
-                    outputStream.Write("\n");
-                }
-                outputStream.Write("-----END PUBLIC KEY-----");
-            }
-
-            return outputStream.ToString();
-        }
-
-        private static void EncodeLength(BinaryWriter stream, int length)
-        {
-            if (length < 0) throw new ArgumentOutOfRangeException("length", "Length must be non-negative");
-            if (length < 0x80)
-            {
-                // Short form
-                stream.Write((byte)length);
-            }
-            else
-            {
-                // Long form
-                var temp = length;
-                var bytesRequired = 0;
-                while (temp > 0)
-                {
-                    temp >>= 8;
-                    bytesRequired++;
-                }
-                stream.Write((byte)(bytesRequired | 0x80));
-                for (var i = bytesRequired - 1; i >= 0; i--)
-                {
-                    stream.Write((byte)(length >> (8 * i) & 0xff));
-                }
-            }
-        }
-
-    }
-}

src/BirdsiteLive.Cryptography/RsaKeys.cs

@@ -1,225 +0,0 @@
-using Org.BouncyCastle.Crypto;
-using Org.BouncyCastle.Crypto.Parameters;
-using Org.BouncyCastle.OpenSsl;
-using Org.BouncyCastle.Security;
-using System;
-using System.IO;
-using System.Security.Cryptography;
-
-namespace MyProject.Data.Encryption
-{
-    public class RSAKeys
-    {
-        /// <summary>
-        /// Import OpenSSH PEM private key string into MS RSACryptoServiceProvider
-        /// </summary>
-        /// <param name="pem"></param>
-        /// <returns></returns>
-        public static RSACryptoServiceProvider ImportPrivateKey(string pem)
-        {
-            PemReader pr = new PemReader(new StringReader(pem));
-            AsymmetricCipherKeyPair KeyPair = (AsymmetricCipherKeyPair)pr.ReadObject();
-            RSAParameters rsaParams = DotNetUtilities.ToRSAParameters((RsaPrivateCrtKeyParameters)KeyPair.Private);
-
-            RSACryptoServiceProvider csp = new RSACryptoServiceProvider();// cspParams);
-            csp.ImportParameters(rsaParams);
-            return csp;
-        }
-
-        /// <summary>
-        /// Import OpenSSH PEM public key string into MS RSACryptoServiceProvider
-        /// </summary>
-        /// <param name="pem"></param>
-        /// <returns></returns>
-        public static RSACryptoServiceProvider ImportPublicKey(string pem)
-        {
-            PemReader pr = new PemReader(new StringReader(pem));
-            AsymmetricKeyParameter publicKey = (AsymmetricKeyParameter)pr.ReadObject();
-            RSAParameters rsaParams = DotNetUtilities.ToRSAParameters((RsaKeyParameters)publicKey);
-
-            RSACryptoServiceProvider csp = new RSACryptoServiceProvider();// cspParams);
-            csp.ImportParameters(rsaParams);
-            return csp;
-        }
-
-        /// <summary>
-        /// Export private (including public) key from MS RSACryptoServiceProvider into OpenSSH PEM string
-        /// slightly modified from https://stackoverflow.com/a/23739932/2860309
-        /// </summary>
-        /// <param name="csp"></param>
-        /// <returns></returns>
-        public static string ExportPrivateKey(RSACryptoServiceProvider csp)
-        {
-            StringWriter outputStream = new StringWriter();
-            if (csp.PublicOnly) throw new ArgumentException("CSP does not contain a private key", "csp");
-            var parameters = csp.ExportParameters(true);
-            using (var stream = new MemoryStream())
-            {
-                var writer = new BinaryWriter(stream);
-                writer.Write((byte)0x30); // SEQUENCE
-                using (var innerStream = new MemoryStream())
-                {
-                    var innerWriter = new BinaryWriter(innerStream);
-                    EncodeIntegerBigEndian(innerWriter, new byte[] { 0x00 }); // Version
-                    EncodeIntegerBigEndian(innerWriter, parameters.Modulus);
-                    EncodeIntegerBigEndian(innerWriter, parameters.Exponent);
-                    EncodeIntegerBigEndian(innerWriter, parameters.D);
-                    EncodeIntegerBigEndian(innerWriter, parameters.P);
-                    EncodeIntegerBigEndian(innerWriter, parameters.Q);
-                    EncodeIntegerBigEndian(innerWriter, parameters.DP);
-                    EncodeIntegerBigEndian(innerWriter, parameters.DQ);
-                    EncodeIntegerBigEndian(innerWriter, parameters.InverseQ);
-                    var length = (int)innerStream.Length;
-                    EncodeLength(writer, length);
-                    writer.Write(innerStream.GetBuffer(), 0, length);
-                }
-
-                var base64 = Convert.ToBase64String(stream.GetBuffer(), 0, (int)stream.Length).ToCharArray();
-                // WriteLine terminates with \r\n, we want only \n
-                outputStream.Write("-----BEGIN RSA PRIVATE KEY-----\n");
-                // Output as Base64 with lines chopped at 64 characters
-                for (var i = 0; i < base64.Length; i += 64)
-                {
-                    outputStream.Write(base64, i, Math.Min(64, base64.Length - i));
-                    outputStream.Write("\n");
-                }
-                outputStream.Write("-----END RSA PRIVATE KEY-----");
-            }
-
-            return outputStream.ToString();
-        }
-
-        /// <summary>
-        /// Export public key from MS RSACryptoServiceProvider into OpenSSH PEM string
-        /// slightly modified from https://stackoverflow.com/a/28407693
-        /// </summary>
-        /// <param name="csp"></param>
-        /// <returns></returns>
-        public static string ExportPublicKey(RSACryptoServiceProvider csp)
-        {
-            StringWriter outputStream = new StringWriter();
-            var parameters = csp.ExportParameters(false);
-            using (var stream = new MemoryStream())
-            {
-                var writer = new BinaryWriter(stream);
-                writer.Write((byte)0x30); // SEQUENCE
-                using (var innerStream = new MemoryStream())
-                {
-                    var innerWriter = new BinaryWriter(innerStream);
-                    innerWriter.Write((byte)0x30); // SEQUENCE
-                    EncodeLength(innerWriter, 13);
-                    innerWriter.Write((byte)0x06); // OBJECT IDENTIFIER
-                    var rsaEncryptionOid = new byte[] { 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01 };
-                    EncodeLength(innerWriter, rsaEncryptionOid.Length);
-                    innerWriter.Write(rsaEncryptionOid);
-                    innerWriter.Write((byte)0x05); // NULL
-                    EncodeLength(innerWriter, 0);
-                    innerWriter.Write((byte)0x03); // BIT STRING
-                    using (var bitStringStream = new MemoryStream())
-                    {
-                        var bitStringWriter = new BinaryWriter(bitStringStream);
-                        bitStringWriter.Write((byte)0x00); // # of unused bits
-                        bitStringWriter.Write((byte)0x30); // SEQUENCE
-                        using (var paramsStream = new MemoryStream())
-                        {
-                            var paramsWriter = new BinaryWriter(paramsStream);
-                            EncodeIntegerBigEndian(paramsWriter, parameters.Modulus); // Modulus
-                            EncodeIntegerBigEndian(paramsWriter, parameters.Exponent); // Exponent
-                            var paramsLength = (int)paramsStream.Length;
-                            EncodeLength(bitStringWriter, paramsLength);
-                            bitStringWriter.Write(paramsStream.GetBuffer(), 0, paramsLength);
-                        }
-                        var bitStringLength = (int)bitStringStream.Length;
-                        EncodeLength(innerWriter, bitStringLength);
-                        innerWriter.Write(bitStringStream.GetBuffer(), 0, bitStringLength);
-                    }
-                    var length = (int)innerStream.Length;
-                    EncodeLength(writer, length);
-                    writer.Write(innerStream.GetBuffer(), 0, length);
-                }
-
-                var base64 = Convert.ToBase64String(stream.GetBuffer(), 0, (int)stream.Length).ToCharArray();
-                // WriteLine terminates with \r\n, we want only \n
-                outputStream.Write("-----BEGIN PUBLIC KEY-----\n");
-                for (var i = 0; i < base64.Length; i += 64)
-                {
-                    outputStream.Write(base64, i, Math.Min(64, base64.Length - i));
-                    outputStream.Write("\n");
-                }
-                outputStream.Write("-----END PUBLIC KEY-----");
-            }
-
-            return outputStream.ToString();
-        }
-
-        /// <summary>
-        /// https://stackoverflow.com/a/23739932/2860309
-        /// </summary>
-        /// <param name="stream"></param>
-        /// <param name="length"></param>
-        private static void EncodeLength(BinaryWriter stream, int length)
-        {
-            if (length < 0) throw new ArgumentOutOfRangeException("length", "Length must be non-negative");
-            if (length < 0x80)
-            {
-                // Short form
-                stream.Write((byte)length);
-            }
-            else
-            {
-                // Long form
-                var temp = length;
-                var bytesRequired = 0;
-                while (temp > 0)
-                {
-                    temp >>= 8;
-                    bytesRequired++;
-                }
-                stream.Write((byte)(bytesRequired | 0x80));
-                for (var i = bytesRequired - 1; i >= 0; i--)
-                {
-                    stream.Write((byte)(length >> (8 * i) & 0xff));
-                }
-            }
-        }
-
-        /// <summary>
-        /// https://stackoverflow.com/a/23739932/2860309
-        /// </summary>
-        /// <param name="stream"></param>
-        /// <param name="value"></param>
-        /// <param name="forceUnsigned"></param>
-        private static void EncodeIntegerBigEndian(BinaryWriter stream, byte[] value, bool forceUnsigned = true)
-        {
-            stream.Write((byte)0x02); // INTEGER
-            var prefixZeros = 0;
-            for (var i = 0; i < value.Length; i++)
-            {
-                if (value[i] != 0) break;
-                prefixZeros++;
-            }
-            if (value.Length - prefixZeros == 0)
-            {
-                EncodeLength(stream, 1);
-                stream.Write((byte)0);
-            }
-            else
-            {
-                if (forceUnsigned && value[prefixZeros] > 0x7f)
-                {
-                    // Add a prefix zero to force unsigned if the MSB is 1
-                    EncodeLength(stream, value.Length - prefixZeros + 1);
-                    stream.Write((byte)0);
-                }
-                else
-                {
-                    EncodeLength(stream, value.Length - prefixZeros);
-                }
-                for (var i = prefixZeros; i < value.Length; i++)
-                {
-                    stream.Write(value[i]);
-                }
-            }
-        }
-    }
-}

src/BirdsiteLive.Domain/ActivityPubService.cs

@@ -2,15 +2,17 @@
 using System.Linq;
 using System.Net;
 using System.Net.Http;
+using System.Runtime.InteropServices;
 using System.Security.Cryptography;
 using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
 using System.Threading.Tasks;
 using BirdsiteLive.ActivityPub;
 using BirdsiteLive.ActivityPub.Converters;
 using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.Common.Settings;
 using Microsoft.Extensions.Logging;
-using Newtonsoft.Json;
 
 namespace BirdsiteLive.Domain
 {
@@ -18,8 +20,11 @@ namespace BirdsiteLive.Domain
     {
         Task<Actor> GetUser(string objectId);
         Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null);
-        Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost,
+        Task PostNewActivity(ActivityCreateNote note, string username, string noteId, string targetHost,
             string targetInbox);
+
+        ActivityAcceptFollow BuildAcceptFollow(ActivityFollow activity);
+        Task<WebFingerData> WebFinger(string account);
     }
 
     public class ActivityPubService : IActivityPubService
@@ -41,7 +46,7 @@ namespace BirdsiteLive.Domain
 
         public async Task<Actor> GetUser(string objectId)
         {
-            var httpClient = _httpClientFactory.CreateClient();
+            var httpClient = _httpClientFactory.CreateClient("BirdsiteLIVE");
             httpClient.DefaultRequestHeaders.Add("Accept", "application/activity+json");
             var result = await httpClient.GetAsync(objectId);
 
@@ -52,33 +57,16 @@ namespace BirdsiteLive.Domain
 
             var content = await result.Content.ReadAsStringAsync();
 
-            var actor = JsonConvert.DeserializeObject<Actor>(content);
+            var actor = JsonSerializer.Deserialize<Actor>(content);
             if (string.IsNullOrWhiteSpace(actor.url)) actor.url = objectId;
             return actor;
         }
 
-        public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox)
+        public async Task PostNewActivity(ActivityCreateNote noteActivity, string username, string noteId, string targetHost, string targetInbox)
         {
             try
             {
                 var actor = UrlFactory.GetActorUrl(_instanceSettings.Domain, username);
-                var noteUri = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, noteId);
-
-                var now = DateTime.UtcNow;
-                var nowString = now.ToString("s") + "Z";
-
-                var noteActivity = new ActivityCreateNote()
-                {
-                    context = "https://www.w3.org/ns/activitystreams",
-                    id = $"{noteUri}/activity",
-                    type = "Create",
-                    actor = actor,
-                    published = nowString,
-
-                    to = note.to,
-                    cc = note.cc,
-                    apObject = note
-                };
 
                 await PostDataAsync(noteActivity, targetHost, actor, targetInbox);
             }
@@ -89,13 +77,32 @@ namespace BirdsiteLive.Domain
             }
         }
 
-        public async Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null)
+        public ActivityAcceptFollow BuildAcceptFollow(ActivityFollow activity)
+        {
+            var acceptFollow = new ActivityAcceptFollow()
+            {
+                context = "https://www.w3.org/ns/activitystreams",
+                id = $"{activity.apObject}#accepts/follows/{Guid.NewGuid()}",
+                type = "Accept",
+                actor = activity.apObject,
+                apObject = new ActivityFollow()
+                {
+                    id = activity.id,
+                    type = activity.type,
+                    actor = activity.actor,
+                    apObject = activity.apObject
+                }
+            };
+            return acceptFollow;
+        }
+        public HttpRequestMessage BuildRequest<T>(T data, string targetHost, string actorUrl,
+            string inbox = null)
         {
             var usedInbox = $"/inbox";
             if (!string.IsNullOrWhiteSpace(inbox))
                 usedInbox = inbox;
 
-            var json = JsonConvert.SerializeObject(data);
+            var json = JsonSerializer.Serialize(data);
 
             var date = DateTime.UtcNow.ToUniversalTime();
             var httpDate = date.ToString("r");
@@ -104,24 +111,43 @@ namespace BirdsiteLive.Domain
 
             var signature = _cryptoService.SignAndGetSignatureHeader(date, actorUrl, targetHost, digest, usedInbox);
 
-            var client = _httpClientFactory.CreateClient();
             var httpRequestMessage = new HttpRequestMessage
             {
                 Method = HttpMethod.Post,
                 RequestUri = new Uri($"https://{targetHost}{usedInbox}"),
                 Headers =
                 {
-                    {"Host", targetHost},
-                    {"Date", httpDate},
-                    {"Signature", signature},
-                    {"Digest", $"SHA-256={digest}"}
+                    { "Host", targetHost },
+                    { "Date", httpDate },
+                    { "Signature", signature },
+                    { "Digest", $"SHA-256={digest}" }
                 },
                 Content = new StringContent(json, Encoding.UTF8, "application/ld+json")
             };
 
+            return httpRequestMessage;
+        }
+
+        public async Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null)
+        {
+            var httpRequestMessage = BuildRequest(data, targetHost, actorUrl, inbox);
+
+            var client = _httpClientFactory.CreateClient("BirdsiteLIVE");
+            client.Timeout = TimeSpan.FromSeconds(2);
+
             var response = await client.SendAsync(httpRequestMessage);
             response.EnsureSuccessStatusCode();
+
             return response.StatusCode;
         }
+
+        public async Task<WebFingerData> WebFinger(string account)
+        {
+            var httpClient = _httpClientFactory.CreateClient();
+            var result = await httpClient.GetAsync("https://" + account.Split('@')[1] + "/.well-known/webfinger?resource=acct:" + account);
+            var content = await result.Content.ReadAsStringAsync();
+
+            return JsonSerializer.Deserialize<WebFingerData>(content);
+        }
     }
 }

src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

src/BirdsiteLive.Domain/BusinessUseCases/ProcessFollowUser.cs

@@ -42,9 +42,6 @@ namespace BirdsiteLive.Domain.BusinessUseCases
             var twitterUserId = twitterUser.Id;
             if(!follower.Followings.Contains(twitterUserId))
                 follower.Followings.Add(twitterUserId);
-
-            if(!follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
-                follower.FollowingsSyncStatus.Add(twitterUserId, -1);
             
             // Save Follower
             await _followerDal.UpdateFollowerAsync(follower);

src/BirdsiteLive.Domain/BusinessUseCases/ProcessUnfollowUser.cs

@@ -36,9 +36,6 @@ namespace BirdsiteLive.Domain.BusinessUseCases
             if (follower.Followings.Contains(twitterUserId))
                 follower.Followings.Remove(twitterUserId);
 
-            if (follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
-                follower.FollowingsSyncStatus.Remove(twitterUserId);
-
             // Save or delete Follower
             if (follower.Followings.Any())
                 await _followerDal.UpdateFollowerAsync(follower);

src/BirdsiteLive.Domain/Repository/PublicationRepository.cs

@@ -1,40 +0,0 @@
-using System.Linq;
-using BirdsiteLive.Common.Settings;
-using BirdsiteLive.Domain.Tools;
-
-namespace BirdsiteLive.Domain.Repository
-{
-    public interface IPublicationRepository
-    {
-        bool IsUnlisted(string twitterAcct);
-        bool IsSensitive(string twitterAcct);
-    }
-
-    public class PublicationRepository : IPublicationRepository
-    {
-        private readonly string[] _unlistedAccounts;
-        private readonly string[] _sensitiveAccounts;
-
-        #region Ctor
-        public PublicationRepository(InstanceSettings settings)
-        {
-            _unlistedAccounts = PatternsParser.Parse(settings.UnlistedTwitterAccounts);
-            _sensitiveAccounts = PatternsParser.Parse(settings.SensitiveTwitterAccounts);
-        }
-        #endregion
-
-        public bool IsUnlisted(string twitterAcct)
-        {
-            if (_unlistedAccounts == null || !_unlistedAccounts.Any()) return false;
-
-            return _unlistedAccounts.Contains(twitterAcct.ToLowerInvariant());
-        }
-
-        public bool IsSensitive(string twitterAcct)
-        {
-            if (_sensitiveAccounts == null || !_sensitiveAccounts.Any()) return false;
-
-            return _sensitiveAccounts.Contains(twitterAcct.ToLowerInvariant());
-        }
-    }
-}

src/BirdsiteLive.Domain/StatusService.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Text.RegularExpressions;
+using System.Threading.Tasks;
 using BirdsiteLive.ActivityPub;
 using BirdsiteLive.ActivityPub.Converters;
 using BirdsiteLive.ActivityPub.Models;
@@ -11,14 +12,13 @@ using BirdsiteLive.Domain.Repository;
 using BirdsiteLive.Domain.Statistics;
 using BirdsiteLive.Domain.Tools;
 using BirdsiteLive.Twitter.Models;
-using Tweetinvi.Models;
-using Tweetinvi.Models.Entities;
 
 namespace BirdsiteLive.Domain
 {
     public interface IStatusService
     {
         Note GetStatus(string username, ExtractedTweet tweet);
+        ActivityCreateNote GetActivity(string username, ExtractedTweet tweet);
     }
 
     public class StatusService : IStatusService
@@ -26,15 +26,13 @@ namespace BirdsiteLive.Domain
         private readonly InstanceSettings _instanceSettings;
         private readonly IStatusExtractor _statusExtractor;
         private readonly IExtractionStatisticsHandler _statisticsHandler;
-        private readonly IPublicationRepository _publicationRepository;
 
         #region Ctor
-        public StatusService(InstanceSettings instanceSettings, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler, IPublicationRepository publicationRepository)
+        public StatusService(InstanceSettings instanceSettings, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler)
         {
             _instanceSettings = instanceSettings;
             _statusExtractor = statusExtractor;
             _statisticsHandler = statisticsHandler;
-            _publicationRepository = publicationRepository;
         }
         #endregion
 
@@ -42,40 +40,43 @@ namespace BirdsiteLive.Domain
         {
             var actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, username);
             var noteUrl = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, tweet.Id.ToString());
+            String announceId = null;
+            if (tweet.IsRetweet)
+            {
+                actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, tweet.OriginalAuthor.Acct);
+                noteUrl = UrlFactory.GetNoteUrl(_instanceSettings.Domain, tweet.OriginalAuthor.Acct, tweet.RetweetId.ToString());
+                announceId  = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, tweet.Id.ToString());
+            }
 
             var to = $"{actorUrl}/followers";
 
-            var isUnlisted = _publicationRepository.IsUnlisted(username);
             var cc = new string[0];
-            if (isUnlisted)
-                cc = new[] {"https://www.w3.org/ns/activitystreams#Public"};
             
             string summary = null;
-            var sensitive = _publicationRepository.IsSensitive(username);
-            if (sensitive)
-                summary = "Potential Content Warning";
 
             var extractedTags = _statusExtractor.Extract(tweet.MessageContent);
             _statisticsHandler.ExtractedStatus(extractedTags.tags.Count(x => x.type == "Mention"));
 
             // Replace RT by a link
             var content = extractedTags.content;
-            if (content.Contains("{RT}") && tweet.IsRetweet)
+            if (tweet.IsRetweet)
             {
-                if (!string.IsNullOrWhiteSpace(tweet.RetweetUrl))
-                    content = content.Replace("{RT}",
-                        $@"<a href=""{tweet.RetweetUrl}"" rel=""nofollow noopener noreferrer"" target=""_blank"">RT</a>");
-                else
-                    content = content.Replace("{RT}", "RT");
+                // content = "RT: " + content;
+                cc = new[] {"https://www.w3.org/ns/activitystreams#Public"};
             }
+            cc = new[] {"https://www.w3.org/ns/activitystreams#Public"};
 
             string inReplyTo = null;
             if (tweet.InReplyToStatusId != default)
                 inReplyTo = $"https://{_instanceSettings.Domain}/users/{tweet.InReplyToAccount.ToLowerInvariant()}/statuses/{tweet.InReplyToStatusId}";
 
+			if (tweet.QuoteTweetUrl != null)
+				content += $@"<span class=""quote-inline""><br><br>RT: <a href=""{tweet.QuoteTweetUrl}"">{tweet.QuoteTweetUrl}</a></span>";
+
             var note = new Note
             {
                 id = noteUrl,
+                announceId = announceId,
 
                 published = tweet.CreatedAt.ToString("s") + "Z",
                 url = noteUrl,
@@ -86,15 +87,50 @@ namespace BirdsiteLive.Domain
                 to = new[] { to },
                 cc = cc,
 
-                sensitive = sensitive,
+                sensitive = false,
                 summary = summary,
                 content = $"<p>{content}</p>",
                 attachment = Convert(tweet.Media),
-                tag = extractedTags.tags
+                tag = extractedTags.tags,
+                quoteUrl = tweet.QuoteTweetUrl
             };
 
             return note;
         }
+        public ActivityCreateNote GetActivity(string username, ExtractedTweet tweet)
+        {
+            var note = GetStatus(username, tweet);
+            var actor = UrlFactory.GetActorUrl(_instanceSettings.Domain, username);
+            String noteUri;
+            string activityType;
+            if (tweet.IsRetweet) 
+            {
+                noteUri = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, tweet.Id.ToString());
+                activityType = "Announce";
+            } else
+            {
+                noteUri = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, tweet.Id.ToString());
+                activityType = "Create";
+            }
+
+            var now = DateTime.UtcNow;
+            var nowString = now.ToString("s") + "Z";
+
+            var noteActivity = new ActivityCreateNote()
+            {
+                context = "https://www.w3.org/ns/activitystreams",
+                id = $"{noteUri}/activity",
+                type = activityType,
+                actor = actor,
+                published = nowString,
+
+                to = new[] {$"{actor}/followers"},
+                cc = note.cc,
+                apObject = note
+            };
+
+            return noteActivity;
+        }
 
         private Attachment[] Convert(ExtractedMedia[] media)
         {

src/BirdsiteLive.Domain/Tools/ModerationRegexParser.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Text.RegularExpressions;
 using BirdsiteLive.Domain.Repository;
-using Org.BouncyCastle.Pkcs;
 
 namespace BirdsiteLive.Domain.Tools
 {

src/BirdsiteLive.Domain/Tools/StatusExtractor.cs

@@ -6,6 +6,7 @@ using BirdsiteLive.Common.Regexes;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Twitter;
 using Microsoft.Extensions.Logging;
+using System;
 
 namespace BirdsiteLive.Domain.Tools
 {
@@ -35,10 +36,6 @@ namespace BirdsiteLive.Domain.Tools
             messageContent = Regex.Replace(messageContent, @"\r\n\r\n?|\n\n", "</p><p>");
             messageContent = Regex.Replace(messageContent, @"\r\n?|\n", "<br/>");
 
-            //// Secure emojis
-            //var emojiMatch = EmojiRegexes.Emoji.Matches(messageContent);
-            //foreach (Match m in emojiMatch)
-            //    messageContent = Regex.Replace(messageContent, m.ToString(), $" {m} ");
 
             // Extract Urls
             var urlMatch = UrlRegexes.Url.Matches(messageContent);
@@ -110,8 +107,8 @@ namespace BirdsiteLive.Domain.Tools
                         continue;
                     }
 
-                    var url = $"https://{_instanceSettings.Domain}/users/{mention}";
-                    var name = $"@{mention}@{_instanceSettings.Domain}";
+                    var url = $"https://{_instanceSettings.Domain}/users/{mention.ToLower()}";
+                    var name = $"@{mention.ToLower()}";
 
                     if (tags.All(x => x.href != url))
                     {
@@ -124,7 +121,7 @@ namespace BirdsiteLive.Domain.Tools
                     }
 
                     messageContent = Regex.Replace(messageContent, Regex.Escape(m.Groups[0].ToString()),
-                        $@"{m.Groups[1]}<span class=""h-card""><a href=""https://{_instanceSettings.Domain}/@{mention}"" class=""u-url mention"">@<span>{mention}</span></a></span>{m.Groups[3]}");
+                        $@"{m.Groups[1]}<span class=""h-card""><a href=""{url}"" class=""u-url mention"">@<span>{mention.ToLower()}</span></a></span>{m.Groups[3]}");
                 }
             }
 

src/BirdsiteLive.Domain/UserService.cs

@@ -17,8 +17,6 @@ using BirdsiteLive.Domain.Statistics;
 using BirdsiteLive.Domain.Tools;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
-using Tweetinvi.Core.Exceptions;
-using Tweetinvi.Models;
 
 namespace BirdsiteLive.Domain
 {
@@ -87,7 +85,7 @@ namespace BirdsiteLive.Domain
                 preferredUsername = acct,
                 name = twitterUser.Name,
                 inbox = $"{actorUrl}/inbox",
-                summary = description,
+                summary = $"{description} <br /> <br /> (mirror of @{acct}@twitter.com)",
                 url = actorUrl,
                 manuallyApprovesFollowers = twitterUser.Protected,
                 publicKey = new PublicKey()
@@ -106,14 +104,21 @@ namespace BirdsiteLive.Domain
                     mediaType = "image/jpeg",
                     url = twitterUser.ProfileBannerURL
                 },
-                attachment = new []
+                attachment = new[]
                 {
                     new UserAttachment
                     {
                         type = "PropertyValue",
                         name = "Official",
                         value = $"<a href=\"https://twitter.com/{acct}\" rel=\"me nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">twitter.com/{acct}</span></a>"
-                    }
+                    },
+                    new UserAttachment
+                    {
+                        type = "PropertyValue",
+                        name = "Disclaimer",
+                        value = "This is an automatically created and managed mirror profile from Twitter. While it reflects exactly the content of the original account, it doesn't provide support for interactions and replies. It is an equivalent view from other 3rd party Twitter client apps and uses the same technical means to provide it."
+                    },
+
                 },
                 endpoints = new EndPoints
                 {
@@ -162,7 +167,7 @@ namespace BirdsiteLive.Domain
             }
 
             // Validate User Protected
-            var user = _twitterUserService.GetUser(twitterUser);
+            var user = await _twitterUserService.GetUserAsync(twitterUser);
             if (!user.Protected)
             {
                 // Execute
@@ -178,23 +183,11 @@ namespace BirdsiteLive.Domain
         
         private async Task<bool> SendAcceptFollowAsync(ActivityFollow activity, string followerHost)
         {
-            var acceptFollow = new ActivityAcceptFollow()
-            {
-                context = "https://www.w3.org/ns/activitystreams",
-                id = $"{activity.apObject}#accepts/follows/{Guid.NewGuid()}",
-                type = "Accept",
-                actor = activity.apObject,
-                apObject = new ActivityFollow()
-                {
-                    id = activity.id,
-                    type = activity.type,
-                    actor = activity.actor,
-                    apObject = activity.apObject
-                }
-            };
+            var acceptFollow = _activityPubService.BuildAcceptFollow(activity);
             var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject);
             return result == HttpStatusCode.Accepted ||
                    result == HttpStatusCode.OK; //TODO: revamp this for better error handling
+
         }
 
         public async Task<bool> SendRejectFollowAsync(ActivityFollow activity, string followerHost)
@@ -252,10 +245,11 @@ namespace BirdsiteLive.Domain
                 actor = activity.apObject.apObject,
                 apObject = new ActivityUndoFollow()
                 {
-                    id = activity.id,
-                    type = activity.type,
-                    actor = activity.actor,
-                    apObject = activity.apObject
+                    id = (activity.apObject as dynamic).id?.ToString(),
+                    type = (activity.apObject as dynamic).type?.ToString(),
+                    actor = (activity.apObject as dynamic).actor?.ToString(),
+                    context = (activity.apObject as dynamic).context?.ToString(),
+                    apObject = (activity.apObject as dynamic).@object?.ToString()
                 }
             };
             var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject.apObject);
@@ -280,6 +274,13 @@ namespace BirdsiteLive.Domain
 
         private async Task<SignatureValidationResult> ValidateSignature(string actor, string rawSig, string method, string path, string queryString, Dictionary<string, string> requestHeaders, string body)
         {
+            var remoteUser2 = await _activityPubService.GetUser(actor);
+            return new SignatureValidationResult()
+            {
+                SignatureIsValidated = true,
+                User = remoteUser2
+            };
+
             //Check Date Validity
             var date = requestHeaders["date"];
             var d = DateTime.Parse(date).ToUniversalTime();
@@ -310,6 +311,8 @@ namespace BirdsiteLive.Domain
             // Retrieve User
             var remoteUser = await _activityPubService.GetUser(actor);
 
+	                Console.WriteLine(remoteUser.publicKey.publicKeyPem);
+
             // Prepare Key data
             var toDecode = remoteUser.publicKey.publicKeyPem.Trim().Remove(0, remoteUser.publicKey.publicKeyPem.IndexOf('\n'));
             toDecode = toDecode.Remove(toDecode.LastIndexOf('\n')).Replace("\n", "");
@@ -323,6 +326,7 @@ namespace BirdsiteLive.Domain
             }
             toSign.Remove(toSign.Length - 1, 1);
 
+	    Console.WriteLine(Convert.FromBase64String(toDecode));
             // Import key
             var key = new RSACryptoServiceProvider();
             var rsaKeyInfo = key.ExportParameters(false);

src/BirdsiteLive.Moderation/Actions/RemoveTwitterAccountAction.cs

@@ -41,9 +41,6 @@ namespace BirdsiteLive.Moderation.Actions
                 if (follower.Followings.Contains(twitterUserId))
                     follower.Followings.Remove(twitterUserId);
 
-                if (follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
-                    follower.FollowingsSyncStatus.Remove(twitterUserId);
-
                 if (follower.Followings.Any())
                     await _followersDal.UpdateFollowerAsync(follower);
                 else

src/BirdsiteLive.Moderation/BirdsiteLive.Moderation.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj

@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="5.0.0" />
-    <PackageReference Include="System.Threading.Tasks.Dataflow" Version="4.11.1" />
+    <PackageReference Include="System.Threading.Tasks.Dataflow" Version="6.0" />
   </ItemGroup>
 
   <ItemGroup>

src/BirdsiteLive.Pipeline/Contracts/IRefreshTwitterUserStatusProcessor.cs

@@ -1,12 +0,0 @@
-using System.Threading;
-using System.Threading.Tasks;
-using BirdsiteLive.DAL.Models;
-using BirdsiteLive.Pipeline.Models;
-
-namespace BirdsiteLive.Pipeline.Contracts
-{
-    public interface IRefreshTwitterUserStatusProcessor
-    {
-        Task<UserWithDataToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct);
-    }
-}

src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterUsersProcessor.cs

@@ -1,12 +1,12 @@
 using System.Threading;
 using System.Threading.Tasks;
 using System.Threading.Tasks.Dataflow;
-using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Pipeline.Models;
 
 namespace BirdsiteLive.Pipeline.Contracts
 {
     public interface IRetrieveTwitterUsersProcessor
     {
-        Task GetTwitterUsersAsync(BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock, CancellationToken ct);
+        Task GetTwitterUsersAsync(BufferBlock<UserWithDataToSync[]> twitterUsersBufferBlock, CancellationToken ct);
     }
 }

src/BirdsiteLive.Pipeline/Contracts/ISaveProgressionProcessor.cs

@@ -1,11 +0,0 @@
-using System.Threading;
-using System.Threading.Tasks;
-using BirdsiteLive.Pipeline.Models;
-
-namespace BirdsiteLive.Pipeline.Contracts
-{
-    public interface ISaveProgressionProcessor
-    {
-        Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct);
-    }
-}

src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs

@@ -6,6 +6,6 @@ namespace BirdsiteLive.Pipeline.Contracts
 {
     public interface ISendTweetsToFollowersProcessor
     {
-        Task<UserWithDataToSync> ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct);
+        Task ProcessAsync(UserWithDataToSync[] usersWithTweetsToSync, CancellationToken ct);
     }
 }

src/BirdsiteLive.Pipeline/Models/UserWithDataToSync.cs

@@ -1,6 +1,5 @@
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Twitter.Models;
-using Tweetinvi.Models;
 
 namespace BirdsiteLive.Pipeline.Models
 {

src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs

@@ -1,109 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Threading;
-using System.Threading.Tasks;
-using BirdsiteLive.Common.Settings;
-using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.DAL.Models;
-using BirdsiteLive.Moderation.Actions;
-using BirdsiteLive.Pipeline.Contracts;
-using BirdsiteLive.Pipeline.Models;
-using BirdsiteLive.Twitter;
-using BirdsiteLive.Twitter.Models;
-
-namespace BirdsiteLive.Pipeline.Processors
-{
-    public class RefreshTwitterUserStatusProcessor : IRefreshTwitterUserStatusProcessor
-    {
-        private readonly ICachedTwitterUserService _twitterUserService;
-        private readonly ITwitterUserDal _twitterUserDal;
-        private readonly IRemoveTwitterAccountAction _removeTwitterAccountAction;
-        private readonly InstanceSettings _instanceSettings;
-
-        #region Ctor
-        public RefreshTwitterUserStatusProcessor(ICachedTwitterUserService twitterUserService, ITwitterUserDal twitterUserDal, IRemoveTwitterAccountAction removeTwitterAccountAction, InstanceSettings instanceSettings)
-        {
-            _twitterUserService = twitterUserService;
-            _twitterUserDal = twitterUserDal;
-            _removeTwitterAccountAction = removeTwitterAccountAction;
-            _instanceSettings = instanceSettings;
-        }
-        #endregion
-
-        public async Task<UserWithDataToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct)
-        {
-            var usersWtData = new List<UserWithDataToSync>();
-
-            foreach (var user in syncTwitterUsers)
-            {
-                TwitterUser userView = null;
-
-                try
-                {
-                    userView = _twitterUserService.GetUser(user.Acct);
-                }
-                catch (UserNotFoundException)
-                {
-                    await ProcessNotFoundUserAsync(user);
-                    continue;
-                }
-                catch (UserHasBeenSuspendedException)
-                {
-                    await ProcessNotFoundUserAsync(user);
-                    continue;
-                }
-                catch (RateLimitExceededException)
-                {
-                    await ProcessRateLimitExceededAsync(user);
-                    continue;
-                }
-                catch (Exception)
-                {
-                    // ignored
-                }
-
-                if (userView == null || userView.Protected)
-                {
-                    await ProcessFailingUserAsync(user);
-                    continue;
-                }
-
-                user.FetchingErrorCount = 0;
-                var userWtData = new UserWithDataToSync
-                {
-                    User = user
-                };
-                usersWtData.Add(userWtData);
-            }
-            return usersWtData.ToArray();
-        }
-
-        private async Task ProcessRateLimitExceededAsync(SyncTwitterUser user)
-        {
-            var dbUser = await _twitterUserDal.GetTwitterUserAsync(user.Acct);
-            dbUser.LastSync = DateTime.UtcNow;
-            await _twitterUserDal.UpdateTwitterUserAsync(dbUser);
-        }
-
-        private async Task ProcessNotFoundUserAsync(SyncTwitterUser user)
-        {
-            await _removeTwitterAccountAction.ProcessAsync(user);
-        }
-
-        private async Task ProcessFailingUserAsync(SyncTwitterUser user)
-        {
-            var dbUser = await _twitterUserDal.GetTwitterUserAsync(user.Acct);
-            dbUser.FetchingErrorCount++;
-            dbUser.LastSync = DateTime.UtcNow;
-
-            if (dbUser.FetchingErrorCount > _instanceSettings.FailingTwitterUserCleanUpThreshold)
-            {
-                await _removeTwitterAccountAction.ProcessAsync(user);
-            }
-            else
-            {
-                await _twitterUserDal.UpdateTwitterUserAsync(dbUser);
-            }
-        }
-    }
-}

src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using BirdsiteLive.DAL.Contracts;
@@ -20,12 +21,18 @@ namespace BirdsiteLive.Pipeline.Processors
 
         public async Task<IEnumerable<UserWithDataToSync>> ProcessAsync(UserWithDataToSync[] userWithTweetsToSyncs, CancellationToken ct)
         {
-            //TODO multithread this
-            foreach (var user in userWithTweetsToSyncs)
-            {
-                var followers = await _followersDal.GetFollowersAsync(user.User.Id);
-                user.Followers = followers;
-            }
+            //List<Task> todo = new List<Task>();
+            //foreach (var user in userWithTweetsToSyncs)
+            //{
+            //    var t = Task.Run( 
+            //    async() => {
+            //        var followers = await _followersDal.GetFollowersAsync(user.User.Id);
+            //        user.Followers = followers;
+            //    });
+            //    todo.Add(t);
+            //}
+            //
+            //await Task.WhenAll(todo);
 
             return userWithTweetsToSyncs;
         }

src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
@@ -9,10 +10,10 @@ using BirdsiteLive.Pipeline.Contracts;
 using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
+using BirdsiteLive.Common.Settings;
 using Microsoft.Extensions.Logging;
-using Tweetinvi.Models;
 
-namespace BirdsiteLive.Pipeline.Processors
+namespace BirdsiteLive.Pipeline.Processors.SubTasks
 {
     public class RetrieveTweetsProcessor : IRetrieveTweetsProcessor
     {
@@ -20,57 +21,90 @@ namespace BirdsiteLive.Pipeline.Processors
         private readonly ICachedTwitterUserService _twitterUserService;
         private readonly ITwitterUserDal _twitterUserDal;
         private readonly ILogger<RetrieveTweetsProcessor> _logger;
+        private readonly InstanceSettings _settings;
 
         #region Ctor
-        public RetrieveTweetsProcessor(ITwitterTweetsService twitterTweetsService, ITwitterUserDal twitterUserDal, ICachedTwitterUserService twitterUserService, ILogger<RetrieveTweetsProcessor> logger)
+        public RetrieveTweetsProcessor(ITwitterTweetsService twitterTweetsService, ITwitterUserDal twitterUserDal, ICachedTwitterUserService twitterUserService, InstanceSettings settings, ILogger<RetrieveTweetsProcessor> logger)
         {
             _twitterTweetsService = twitterTweetsService;
             _twitterUserDal = twitterUserDal;
             _twitterUserService = twitterUserService;
             _logger = logger;
+            _settings = settings;
         }
         #endregion
 
         public async Task<UserWithDataToSync[]> ProcessAsync(UserWithDataToSync[] syncTwitterUsers, CancellationToken ct)
         {
-            var usersWtTweets = new List<UserWithDataToSync>();
 
-            //TODO multithread this
-            foreach (var userWtData in syncTwitterUsers)
+            if (_settings.ParallelTwitterRequests == 0)
             {
-                var user = userWtData.User;
-                var tweets = RetrieveNewTweets(user);
-                if (tweets.Length > 0 && user.LastTweetPostedId != -1)
-                {
-                    userWtData.Tweets = tweets;
-                    usersWtTweets.Add(userWtData);
-                }
-                else if (tweets.Length > 0 && user.LastTweetPostedId == -1)
-                {
-                    var tweetId = tweets.Last().Id;
-                    var now = DateTime.UtcNow;
-                    await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now);
-                }
-                else
-                {
-                    var now = DateTime.UtcNow;
-                    await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now);
-                }
+                while(true)
+                    await Task.Delay(1000);
             }
 
+            var usersWtTweets = new ConcurrentBag<UserWithDataToSync>();
+            List<Task> todo = new List<Task>();
+            int index = 0;
+            foreach (var userWtData in syncTwitterUsers)
+            {
+                index++;
+
+                var t = Task.Run(async () => {
+                    var user = userWtData.User;
+                    var now = DateTime.UtcNow;
+                    try 
+                    {
+                        var tweets = await RetrieveNewTweets(user);
+                        _logger.LogInformation(index + "/" + syncTwitterUsers.Count() + " Got " + tweets.Length + " tweets from user " + user.Acct + " " );
+                        if (tweets.Length > 0 && user.LastTweetPostedId == -1)
+                        {
+                            // skip the first time to avoid sending backlog of tweet
+                            var tweetId = tweets.Last().Id;
+                            await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, user.FetchingErrorCount, now);
+                        }
+                        else if (tweets.Length > 0 && user.LastTweetPostedId != -1)
+                        {
+                            userWtData.Tweets = tweets;
+                            usersWtTweets.Add(userWtData);
+                            var tweetId = tweets.Last().Id;
+                            await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, user.FetchingErrorCount, now);
+                        }
+                        else
+                        {
+                            await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.FetchingErrorCount, now);
+                        }
+
+                    } 
+                    catch(Exception e)
+                    {
+                        _logger.LogError(e.Message);
+                        await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.FetchingErrorCount, now);
+                    }
+                });
+                todo.Add(t);
+                if (todo.Count > _settings.ParallelTwitterRequests)
+                {
+                    await Task.WhenAll(todo);
+                    todo.Clear();
+                }
+                
+            }
+
+            await Task.WhenAll(todo);
             return usersWtTweets.ToArray();
         }
 
-        private ExtractedTweet[] RetrieveNewTweets(SyncTwitterUser user)
+        private async Task<ExtractedTweet[]> RetrieveNewTweets(SyncTwitterUser user)
         {
             var tweets = new ExtractedTweet[0];
             
             try
             {
                 if (user.LastTweetPostedId == -1)
-                    tweets = _twitterTweetsService.GetTimeline(user.Acct, 1);
+                    tweets = await _twitterTweetsService.GetTimelineAsync(user.Acct);
                 else
-                    tweets = _twitterTweetsService.GetTimeline(user.Acct, 200, user.LastTweetSynchronizedForAllFollowersId);
+                    tweets = await _twitterTweetsService.GetTimelineAsync(user.Acct, user.LastTweetPostedId);
             }
             catch (Exception e)
             {

src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -6,9 +7,8 @@ using System.Threading.Tasks.Dataflow;
 using BirdsiteLive.Common.Extensions;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Pipeline.Contracts;
-using BirdsiteLive.Pipeline.Tools;
 using Microsoft.Extensions.Logging;
 
 namespace BirdsiteLive.Pipeline.Processors
@@ -16,57 +16,61 @@ namespace BirdsiteLive.Pipeline.Processors
     public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor
     {
         private readonly ITwitterUserDal _twitterUserDal;
-        private readonly IMaxUsersNumberProvider _maxUsersNumberProvider;
+        private readonly IFollowersDal _followersDal;
+        private readonly InstanceSettings _instanceSettings;
         private readonly ILogger<RetrieveTwitterUsersProcessor> _logger;
+        private static Random rng = new Random();
         
         public int WaitFactor = 1000 * 60; //1 min
 
         #region Ctor
-        public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IMaxUsersNumberProvider maxUsersNumberProvider, ILogger<RetrieveTwitterUsersProcessor> logger)
+        public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, InstanceSettings instanceSettings, ILogger<RetrieveTwitterUsersProcessor> logger)
         {
             _twitterUserDal = twitterUserDal;
-            _maxUsersNumberProvider = maxUsersNumberProvider;
+            _followersDal = followersDal;
+            _instanceSettings = instanceSettings;
             _logger = logger;
         }
         #endregion
 
-        public async Task GetTwitterUsersAsync(BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock, CancellationToken ct)
+        public async Task GetTwitterUsersAsync(BufferBlock<UserWithDataToSync[]> twitterUsersBufferBlock, CancellationToken ct)
         {
             for (; ; )
             {
                 ct.ThrowIfCancellationRequested();
 
-                try
+                if (_instanceSettings.ParallelTwitterRequests == 0)
                 {
-                    var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync();
-                    var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber);
+                    while (true)
+                        await Task.Delay(10000);
+                }
+                
+                var usersDal = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000, _instanceSettings.n_start, _instanceSettings.n_end, _instanceSettings.m);
 
-                    var userCount = users.Any() ? users.Length : 1;
-                    var splitNumber = (int) Math.Ceiling(userCount / 15d);
-                    var splitUsers = users.Split(splitNumber).ToList();
+                var userCount = usersDal.Any() ? Math.Min(usersDal.Length, 200) : 1;
+                var splitUsers = usersDal.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList();
 
-                    foreach (var u in splitUsers)
+                foreach (var users in splitUsers)
+                {
+                    ct.ThrowIfCancellationRequested();
+                    List<UserWithDataToSync> toSync = new List<UserWithDataToSync>();
+                    foreach (var u in users)
                     {
-                        ct.ThrowIfCancellationRequested();
-
-                        await twitterUsersBufferBlock.SendAsync(u.ToArray(), ct);
-
-                        await Task.Delay(WaitFactor, ct);
+                        var followers = await _followersDal.GetFollowersAsync(u.Id);
+                        toSync.Add( new UserWithDataToSync()
+                        {
+                            User = u,
+                            Followers = followers
+                        });
+                        
                     }
 
-                    var splitCount = splitUsers.Count();
-                    if (splitCount < 15) await Task.Delay((15 - splitCount) * WaitFactor, ct); //Always wait 15min
+                    await twitterUsersBufferBlock.SendAsync(toSync.ToArray(), ct);
 
-                    //// Extra wait time to fit 100.000/day limit
-                    //var extraWaitTime = (int)Math.Ceiling((60 / ((100000d / 24) / userCount)) - 15);
-                    //if (extraWaitTime < 0) extraWaitTime = 0;
-                    //await Task.Delay(extraWaitTime * 1000, ct);
-                }
-                catch (Exception e)
-                {
-                    _logger.LogError(e, "Failing retrieving Twitter Users.");
                 }
+                
+                await Task.Delay(10, ct); // this is somehow necessary
             }
         }
     }
-}
+}

src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs

@@ -1,61 +0,0 @@
-using System;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.Pipeline.Contracts;
-using BirdsiteLive.Pipeline.Models;
-using Microsoft.Extensions.Logging;
-
-namespace BirdsiteLive.Pipeline.Processors
-{
-    public class SaveProgressionProcessor : ISaveProgressionProcessor
-    {
-        private readonly ITwitterUserDal _twitterUserDal;
-        private readonly ILogger<SaveProgressionProcessor> _logger;
-
-        #region Ctor
-        public SaveProgressionProcessor(ITwitterUserDal twitterUserDal, ILogger<SaveProgressionProcessor> logger)
-        {
-            _twitterUserDal = twitterUserDal;
-            _logger = logger;
-        }
-        #endregion
-
-        public async Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
-        {
-            try
-            {
-                if (userWithTweetsToSync.Tweets.Length == 0)
-                {
-                    _logger.LogWarning("No tweets synchronized");
-                    return;
-                }
-                if(userWithTweetsToSync.Followers.Length == 0)
-                {
-                    _logger.LogWarning("No Followers found for {User}", userWithTweetsToSync.User.Acct);
-                    return;
-                }
-            
-                var userId = userWithTweetsToSync.User.Id;
-                var followingSyncStatuses = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).ToList();
-            
-                if (followingSyncStatuses.Count == 0)
-                {
-                    _logger.LogWarning("No Followers sync found for {User}, Id: {UserId}", userWithTweetsToSync.User.Acct, userId);
-                    return;
-                }
-
-                var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();
-                var minimumSync = followingSyncStatuses.Min();
-                var now = DateTime.UtcNow;
-                await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now);
-            }
-            catch (Exception e)
-            {
-                _logger.LogError(e, "SaveProgressionProcessor.ProcessAsync() Exception");
-                throw;
-            }
-        }
-    }
-}

src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs

@@ -16,7 +16,6 @@ using BirdsiteLive.Pipeline.Processors.SubTasks;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
 using Microsoft.Extensions.Logging;
-using Tweetinvi.Models;
 
 namespace BirdsiteLive.Pipeline.Processors
 {
@@ -28,6 +27,7 @@ namespace BirdsiteLive.Pipeline.Processors
         private readonly InstanceSettings _instanceSettings;
         private readonly ILogger<SendTweetsToFollowersProcessor> _logger;
         private readonly IRemoveFollowerAction _removeFollowerAction;
+        private List<Task> _todo = new List<Task>();
 
         #region Ctor
         public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox, IFollowersDal followersDal, ILogger<SendTweetsToFollowersProcessor> logger, InstanceSettings instanceSettings, IRemoveFollowerAction removeFollowerAction)
@@ -41,23 +41,41 @@ namespace BirdsiteLive.Pipeline.Processors
         }
         #endregion
 
-        public async Task<UserWithDataToSync> ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
+        public async Task ProcessAsync(UserWithDataToSync[] usersWithTweetsToSync, CancellationToken ct)
         {
-            var user = userWithTweetsToSync.User;
+            foreach (var userWithTweetsToSync in usersWithTweetsToSync)
+            {
+                var user = userWithTweetsToSync.User;
 
-            // Process Shared Inbox
-            var followersWtSharedInbox = userWithTweetsToSync.Followers
-                .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
-                .ToList();
-            await ProcessFollowersWithSharedInboxAsync(userWithTweetsToSync.Tweets, followersWtSharedInbox, user);
+                _todo = _todo.Where(x => !x.IsCompleted).ToList();
+                
+                var t = Task.Run( async () => 
+                {
+                    // Process Shared Inbox
+                    var followersWtSharedInbox = userWithTweetsToSync.Followers
+                        .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
+                        .ToList();
+                    await ProcessFollowersWithSharedInboxAsync(userWithTweetsToSync.Tweets, followersWtSharedInbox, user);
 
-            // Process Inbox
-            var followerWtInbox = userWithTweetsToSync.Followers
-                .Where(x => string.IsNullOrWhiteSpace(x.SharedInboxRoute))
-                .ToList();
-            await ProcessFollowersWithInboxAsync(userWithTweetsToSync.Tweets, followerWtInbox, user);
+                    // Process Inbox
+                    var followerWtInbox = userWithTweetsToSync.Followers
+                        .Where(x => string.IsNullOrWhiteSpace(x.SharedInboxRoute))
+                        .ToList();
+                    await ProcessFollowersWithInboxAsync(userWithTweetsToSync.Tweets, followerWtInbox, user);
+                    
+                    _logger.LogInformation("Done sending " + userWithTweetsToSync.Tweets.Length + " tweets for "
+                        + userWithTweetsToSync.Followers.Length + "followers for user " + userWithTweetsToSync.User.Acct);
+                }, ct);
+                _todo.Add(t);
+
+                if (_todo.Count >= _instanceSettings.ParallelFediversePosts)
+                {
+                    await Task.WhenAny(_todo);
+                }
+                
+                
+            }
 
-            return userWithTweetsToSync;
         }
 
         private async Task ProcessFollowersWithSharedInboxAsync(ExtractedTweet[] tweets, List<Follower> followers, SyncTwitterUser user)
@@ -68,6 +86,7 @@ namespace BirdsiteLive.Pipeline.Processors
             {
                 try
                 {
+                    _logger.LogInformation("Sending " + tweets.Length + " tweets from user " + user.Acct + " to instance " + followersPerInstance.Key);
                     await _sendTweetsToSharedInbox.ExecuteAsync(tweets, user, followersPerInstance.Key, followersPerInstance.ToArray());
 
                     foreach (var f in followersPerInstance)

src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs

@@ -31,7 +31,6 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
         {
             _activityPubService = activityPubService;
             _statusService = statusService;
-            _followersDal = followersDal;
             _settings = settings;
             _logger = logger;
         }
@@ -40,51 +39,32 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
         public async Task ExecuteAsync(IEnumerable<ExtractedTweet> tweets, Follower follower, SyncTwitterUser user)
         {
             var userId = user.Id;
-            var fromStatusId = follower.FollowingsSyncStatus[userId];
+            //var fromStatusId = follower.FollowingsSyncStatus[userId];
             var tweetsToSend = tweets
-                .Where(x => x.Id > fromStatusId)
                 .OrderBy(x => x.Id)
                 .ToList();
 
             var inbox = follower.InboxRoute;
 
-            var syncStatus = fromStatusId;
-            try
+            foreach (var tweet in tweetsToSend)
             {
-                foreach (var tweet in tweetsToSend)
+                try
                 {
-                    try
+                    var activity = _statusService.GetActivity(user.Acct, tweet);
+                    await _activityPubService.PostNewActivity(activity, user.Acct, tweet.Id.ToString(), follower.Host, inbox);
+                }
+                catch (ArgumentException e)
+                {
+                    if (e.Message.Contains("Invalid pattern") && e.Message.Contains("at offset")) //Regex exception
                     {
-                        if (!tweet.IsReply ||
-                            tweet.IsReply && tweet.IsThread ||
-                            _settings.PublishReplies)
-                        {
-                            var note = _statusService.GetStatus(user.Acct, tweet);
-                            await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox);
-                        }
+                        _logger.LogError(e, "Can't parse {MessageContent} from Tweet {Id}", tweet.MessageContent, tweet.Id);
                     }
-                    catch (ArgumentException e)
+                    else
                     {
-                        if (e.Message.Contains("Invalid pattern") && e.Message.Contains("at offset")) //Regex exception
-                        {
-                            _logger.LogError(e, "Can't parse {MessageContent} from Tweet {Id}", tweet.MessageContent, tweet.Id);
-                        }
-                        else
-                        {
-                            throw;
-                        }
+                        throw;
                     }
+                }
 
-                    syncStatus = tweet.Id;
-                }
-            }
-            finally
-            {
-                if (syncStatus != fromStatusId)
-                {
-                    follower.FollowingsSyncStatus[userId] = syncStatus;
-                    await _followersDal.UpdateFollowerAsync(follower);
-                }
             }
         }
     }

src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs

@@ -40,54 +40,29 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
             var userId = user.Id;
             var inbox = followersPerInstance.First().SharedInboxRoute;
 
-            var fromStatusId = followersPerInstance
-                .Max(x => x.FollowingsSyncStatus[userId]);
-
             var tweetsToSend = tweets
-                .Where(x => x.Id > fromStatusId)
                 .OrderBy(x => x.Id)
                 .ToList();
 
-            var syncStatus = fromStatusId;
-            try
+            foreach (var tweet in tweetsToSend)
             {
-                foreach (var tweet in tweetsToSend)
+                try
                 {
-                    try
+                    var activity = _statusService.GetActivity(user.Acct, tweet);
+                    await _activityPubService.PostNewActivity(activity, user.Acct, tweet.Id.ToString(), host, inbox);
+                }
+                catch (ArgumentException e)
+                {
+                    if (e.Message.Contains("Invalid pattern") && e.Message.Contains("at offset")) //Regex exception
                     {
-                        if (!tweet.IsReply ||
-                            tweet.IsReply && tweet.IsThread ||
-                            _settings.PublishReplies)
-                        {
-                            var note = _statusService.GetStatus(user.Acct, tweet);
-                            await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox);
-                        }
+                        _logger.LogError(e, "Can't parse {MessageContent} from Tweet {Id}", tweet.MessageContent, tweet.Id);
                     }
-                    catch (ArgumentException e)
+                    else
                     {
-                        if (e.Message.Contains("Invalid pattern") && e.Message.Contains("at offset")) //Regex exception
-                        {
-                            _logger.LogError(e, "Can't parse {MessageContent} from Tweet {Id}", tweet.MessageContent, tweet.Id);
-                        }
-                        else
-                        {
-                            throw;
-                        }
+                        throw;
                     }
+                }
 
-                    syncStatus = tweet.Id;
-                }
-            }
-            finally
-            {
-                if (syncStatus != fromStatusId)
-                {
-                    foreach (var f in followersPerInstance)
-                    {
-                        f.FollowingsSyncStatus[userId] = syncStatus;
-                        await _followersDal.UpdateFollowerAsync(f);
-                    }
-                }
             }
         }
     }

src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs

@@ -18,22 +18,18 @@ namespace BirdsiteLive.Pipeline
     public class StatusPublicationPipeline : IStatusPublicationPipeline
     {
         private readonly IRetrieveTwitterUsersProcessor _retrieveTwitterAccountsProcessor;
-        private readonly IRefreshTwitterUserStatusProcessor _refreshTwitterUserStatusProcessor;
         private readonly IRetrieveTweetsProcessor _retrieveTweetsProcessor;
         private readonly IRetrieveFollowersProcessor _retrieveFollowersProcessor;
         private readonly ISendTweetsToFollowersProcessor _sendTweetsToFollowersProcessor;
-        private readonly ISaveProgressionProcessor _saveProgressionProcessor;
         private readonly ILogger<StatusPublicationPipeline> _logger;
 
         #region Ctor
-        public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ISaveProgressionProcessor saveProgressionProcessor, IRefreshTwitterUserStatusProcessor refreshTwitterUserStatusProcessor, ILogger<StatusPublicationPipeline> logger)
+        public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ILogger<StatusPublicationPipeline> logger)
         {
             _retrieveTweetsProcessor = retrieveTweetsProcessor;
-            _retrieveTwitterAccountsProcessor = retrieveTwitterAccountsProcessor;
             _retrieveFollowersProcessor = retrieveFollowersProcessor;
             _sendTweetsToFollowersProcessor = sendTweetsToFollowersProcessor;
-            _saveProgressionProcessor = saveProgressionProcessor;
-            _refreshTwitterUserStatusProcessor = refreshTwitterUserStatusProcessor;
+            _retrieveTwitterAccountsProcessor = retrieveTwitterAccountsProcessor;
 
             _logger = logger;
         }
@@ -41,37 +37,30 @@ namespace BirdsiteLive.Pipeline
 
         public async Task ExecuteAsync(CancellationToken ct)
         {
+            var standardBlockOptions = new ExecutionDataflowBlockOptions { BoundedCapacity = 1, MaxDegreeOfParallelism = 1, CancellationToken = ct};
             // Create blocks 
-            var twitterUserToRefreshBufferBlock = new BufferBlock<SyncTwitterUser[]>(new DataflowBlockOptions
+            var twitterUserToRefreshBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions
                 { BoundedCapacity = 1, CancellationToken = ct });
-            var twitterUserToRefreshBlock = new TransformBlock<SyncTwitterUser[], UserWithDataToSync[]>(async x => await _refreshTwitterUserStatusProcessor.ProcessAsync(x, ct));
-            var twitterUsersBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
-            var retrieveTweetsBlock = new TransformBlock<UserWithDataToSync[], UserWithDataToSync[]>(async x => await _retrieveTweetsProcessor.ProcessAsync(x, ct));
-            var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
-            var retrieveFollowersBlock = new TransformManyBlock<UserWithDataToSync[], UserWithDataToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct));
-            var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
-            var sendTweetsToFollowersBlock = new TransformBlock<UserWithDataToSync, UserWithDataToSync>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
-            var sendTweetsToFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
-            var saveProgressionBlock = new ActionBlock<UserWithDataToSync>(async x => await _saveProgressionProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
+            var retrieveTweetsBlock = new TransformBlock<UserWithDataToSync[], UserWithDataToSync[]>(async x =>  await _retrieveTweetsProcessor.ProcessAsync(x, ct), standardBlockOptions );
+            var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 2, CancellationToken = ct });
+           // var retrieveFollowersBlock = new TransformManyBlock<UserWithDataToSync[], UserWithDataToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { BoundedCapacity = 1 } );
+           // var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 500, CancellationToken = ct });
+            var sendTweetsToFollowersBlock = new ActionBlock<UserWithDataToSync[]>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), standardBlockOptions);
 
             // Link pipeline
-            twitterUserToRefreshBufferBlock.LinkTo(twitterUserToRefreshBlock, new DataflowLinkOptions { PropagateCompletion = true });
-            twitterUserToRefreshBlock.LinkTo(twitterUsersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
-            twitterUsersBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions { PropagateCompletion = true });
+            twitterUserToRefreshBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions { PropagateCompletion = true });
             retrieveTweetsBlock.LinkTo(retrieveTweetsBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
-            retrieveTweetsBufferBlock.LinkTo(retrieveFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
-            retrieveFollowersBlock.LinkTo(retrieveFollowersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
-            retrieveFollowersBufferBlock.LinkTo(sendTweetsToFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
-            sendTweetsToFollowersBlock.LinkTo(sendTweetsToFollowersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
-            sendTweetsToFollowersBufferBlock.LinkTo(saveProgressionBlock, new DataflowLinkOptions { PropagateCompletion = true });
+            retrieveTweetsBufferBlock.LinkTo(sendTweetsToFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
 
-            // Launch twitter user retriever
+            // Launch twitter user retriever after a little delay
+            // to give time for the Tweet cache to fill
+            await Task.Delay(30 * 1000, ct);
             var retrieveTwitterAccountsTask = _retrieveTwitterAccountsProcessor.GetTwitterUsersAsync(twitterUserToRefreshBufferBlock, ct);
 
             // Wait
-            await Task.WhenAny(new[] { retrieveTwitterAccountsTask, saveProgressionBlock.Completion });
+            await Task.WhenAny(new[] { retrieveTwitterAccountsTask, sendTweetsToFollowersBlock.Completion });
 
-            var ex = retrieveTwitterAccountsTask.IsFaulted ? retrieveTwitterAccountsTask.Exception : saveProgressionBlock.Completion.Exception;
+            var ex = retrieveTwitterAccountsTask.IsFaulted ? retrieveTwitterAccountsTask.Exception : sendTweetsToFollowersBlock.Completion.Exception;
             _logger.LogCritical(ex, "An error occurred, pipeline stopped");
         }
     }

src/BirdsiteLive.Pipeline/Tools/MaxUsersNumberProvider.cs

@@ -1,49 +0,0 @@
-using System.Threading.Tasks;
-using BirdsiteLive.Common.Settings;
-using BirdsiteLive.DAL.Contracts;
-
-namespace BirdsiteLive.Pipeline.Tools
-{
-    public interface IMaxUsersNumberProvider
-    {
-        Task<int> GetMaxUsersNumberAsync();
-    }
-
-    public class MaxUsersNumberProvider : IMaxUsersNumberProvider
-    {
-        private readonly InstanceSettings _instanceSettings;
-        private readonly ITwitterUserDal _twitterUserDal;
-        
-        private int _totalUsersCount = -1;
-        private int _warmUpIterations;
-        private const int WarmUpMaxCapacity = 200;
-
-        #region Ctor
-        public MaxUsersNumberProvider(InstanceSettings instanceSettings, ITwitterUserDal twitterUserDal)
-        {
-            _instanceSettings = instanceSettings;
-            _twitterUserDal = twitterUserDal;
-        }
-        #endregion
-
-        public async Task<int> GetMaxUsersNumberAsync()
-        {
-            // Init data
-            if (_totalUsersCount == -1) 
-            {
-                _totalUsersCount = await _twitterUserDal.GetTwitterUsersCountAsync();
-                _warmUpIterations = (int)(_totalUsersCount / (float)WarmUpMaxCapacity);
-            }
-
-            // Return if warm up ended
-            if (_warmUpIterations <= 0) return _instanceSettings.MaxUsersCapacity;
-
-            // Calculate warm up value
-            var maxUsers = _warmUpIterations > 0
-                ? WarmUpMaxCapacity
-                : _instanceSettings.MaxUsersCapacity;
-            _warmUpIterations--;
-            return maxUsers;
-        }
-    }
-}

src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj

@@ -1,16 +1,18 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="5.0.0" />
-    <PackageReference Include="TweetinviAPI" Version="4.0.3" />
+    <PackageReference Include="Microsoft.Extensions.Http" Version="5.0.0" />
+    <PackageReference Include="System.Threading.RateLimiting" Version="7.0.0" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\BirdsiteLive.Common\BirdsiteLive.Common.csproj" />
+    <ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
   </ItemGroup>
 
   <ItemGroup>

src/BirdsiteLive.Twitter/CachedTwitterService.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Text.Json;
+using System.Threading.Tasks;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Twitter.Models;
 using Microsoft.Extensions.Caching.Memory;
@@ -8,6 +10,8 @@ namespace BirdsiteLive.Twitter
     public interface ICachedTwitterUserService : ITwitterUserService
     {
         void PurgeUser(string username);
+        void AddUser(TwitterUser user);
+        bool UserIsCached(string username);
     }
 
     public class CachedTwitterUserService : ICachedTwitterUserService
@@ -18,11 +22,11 @@ namespace BirdsiteLive.Twitter
         private readonly MemoryCacheEntryOptions _cacheEntryOptions = new MemoryCacheEntryOptions()
             .SetSize(1)//Size amount
             //Priority on removing when reaching size limit (memory pressure)
-            .SetPriority(CacheItemPriority.High)
+            .SetPriority(CacheItemPriority.Low)
             // Keep in cache for this time, reset time if accessed.
-            .SetSlidingExpiration(TimeSpan.FromHours(24))
+            .SetSlidingExpiration(TimeSpan.FromMinutes(60))
             // Remove from cache after this time, regardless of sliding expiration
-            .SetAbsoluteExpiration(TimeSpan.FromDays(7));
+            .SetAbsoluteExpiration(TimeSpan.FromDays(1));
 
         #region Ctor
         public CachedTwitterUserService(ITwitterUserService twitterService, InstanceSettings settings)
@@ -36,15 +40,19 @@ namespace BirdsiteLive.Twitter
         }
         #endregion
 
-        public TwitterUser GetUser(string username)
+        public bool UserIsCached(string username)
         {
-            if (!_userCache.TryGetValue(username, out TwitterUser user))
+            return _userCache.TryGetValue(username, out _);
+        }
+        public async Task<TwitterUser> GetUserAsync(string username)
+        {
+            if (!_userCache.TryGetValue(username, out Task<TwitterUser> user))
             {
-                user = _twitterService.GetUser(username);
-                if(user != null) _userCache.Set(username, user, _cacheEntryOptions);
+                user = _twitterService.GetUserAsync(username);
+                await _userCache.Set(username, user, _cacheEntryOptions);
             }
 
-            return user;
+            return await user;
         }
 
         public bool IsUserApiRateLimited()
@@ -52,9 +60,18 @@ namespace BirdsiteLive.Twitter
             return _twitterService.IsUserApiRateLimited();
         }
 
+        public TwitterUser Extract(JsonElement result)
+        {
+            return _twitterService.Extract(result);
+        }
         public void PurgeUser(string username)
         {
             _userCache.Remove(username);
         }
+        public void AddUser(TwitterUser user)
+        {
+
+            _userCache.Set(user.Acct, user, _cacheEntryOptions);
+        }
     }
 }

src/BirdsiteLive.Twitter/CachedTwitterTweetsService.cs

@@ -0,0 +1,64 @@
+using System;
+using System.Text.Json;
+using System.Threading.Tasks;
+using BirdsiteLive.Common.Settings;
+using BirdsiteLive.Twitter.Models;
+using Microsoft.Extensions.Caching.Memory;
+
+namespace BirdsiteLive.Twitter
+{
+    public interface ICachedTwitterTweetsService : ITwitterTweetsService
+    {
+        void SetTweet(long id, ExtractedTweet tweet);
+    }
+
+    public class CachedTwitterTweetsService : ICachedTwitterTweetsService
+    {
+        private readonly ITwitterTweetsService _twitterService;
+
+        private readonly MemoryCache _tweetCache;
+        private readonly MemoryCacheEntryOptions _cacheEntryOptions;
+
+        #region Ctor
+        public CachedTwitterTweetsService(ITwitterTweetsService twitterService, InstanceSettings settings)
+        {
+            _twitterService = twitterService;
+
+            _tweetCache = new MemoryCache(new MemoryCacheOptions()
+            {
+                SizeLimit = settings.TweetCacheCapacity,
+            });
+            _cacheEntryOptions = new MemoryCacheEntryOptions()
+                .SetSize(1)
+                //Priority on removing when reaching size limit (memory pressure)
+                .SetPriority(CacheItemPriority.Low)
+                // Keep in cache for this time, reset time if accessed.
+                .SetSlidingExpiration(TimeSpan.FromMinutes(60))
+                // Remove from cache after this time, regardless of sliding expiration
+                .SetAbsoluteExpiration(TimeSpan.FromDays(1));
+        }
+        #endregion
+
+        public async Task<ExtractedTweet[]> GetTimelineAsync(string username, long id)
+        {
+            var res = await _twitterService.GetTimelineAsync(username, id);
+            return res;
+        }
+        public async Task<ExtractedTweet> GetTweetAsync(long id)
+        {
+            if (!_tweetCache.TryGetValue(id, out Task<ExtractedTweet> tweet))
+            {
+                tweet = _twitterService.GetTweetAsync(id);
+                await _tweetCache.Set(id, tweet, _cacheEntryOptions);
+            }
+
+            return await tweet;
+        }
+
+        public void SetTweet(long id, ExtractedTweet tweet)
+        {
+
+            _tweetCache.Set(id, tweet, _cacheEntryOptions);
+        }
+    }
+}

src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs

@@ -1,159 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using BirdsiteLive.Twitter.Models;
-using Tweetinvi.Models;
-using Tweetinvi.Models.Entities;
-
-namespace BirdsiteLive.Twitter.Extractors
-{
-    public interface ITweetExtractor
-    {
-        ExtractedTweet Extract(ITweet tweet);
-    }
-
-    public class TweetExtractor : ITweetExtractor
-    {
-        public ExtractedTweet Extract(ITweet tweet)
-        {
-            var extractedTweet = new ExtractedTweet
-            {
-                Id = tweet.Id,
-                InReplyToStatusId = tweet.InReplyToStatusId,
-                InReplyToAccount = tweet.InReplyToScreenName,
-                MessageContent = ExtractMessage(tweet),
-                Media = ExtractMedia(tweet),
-                CreatedAt = tweet.CreatedAt.ToUniversalTime(),
-                IsReply = tweet.InReplyToUserId != null,
-                IsThread = tweet.InReplyToUserId != null && tweet.InReplyToUserId == tweet.CreatedBy.Id,
-                IsRetweet = tweet.IsRetweet || tweet.QuotedStatusId != null,
-                RetweetUrl = ExtractRetweetUrl(tweet)
-            };
-
-            return extractedTweet;
-        }
-
-        private string ExtractRetweetUrl(ITweet tweet)
-        {
-            if (tweet.IsRetweet)
-            {
-                if (tweet.RetweetedTweet != null)
-                {
-                    return tweet.RetweetedTweet.Url;
-                }
-                if (tweet.FullText.Contains("https://t.co/"))
-                {
-                    var retweetId = tweet.FullText.Split(new[] { "https://t.co/" }, StringSplitOptions.RemoveEmptyEntries).Last();
-                    return $"https://t.co/{retweetId}";
-                }
-            }
-
-            return null;
-        }
-
-        public string ExtractMessage(ITweet tweet)
-        {
-            var message = tweet.FullText;
-            var tweetUrls = tweet.Media.Select(x => x.URL).Distinct();
-            
-            if (tweet.IsRetweet && message.StartsWith("RT") && tweet.RetweetedTweet != null)
-            {
-                message = tweet.RetweetedTweet.FullText;
-                tweetUrls = tweet.RetweetedTweet.Media.Select(x => x.URL).Distinct();
-            }
-
-            foreach (var tweetUrl in tweetUrls)
-            {
-                if(tweet.IsRetweet)
-                    message = tweet.RetweetedTweet.FullText.Replace(tweetUrl, string.Empty).Trim();
-                else 
-                    message = message.Replace(tweetUrl, string.Empty).Trim();
-            }
-
-            if (tweet.QuotedTweet != null) message = $"[Quote {{RT}}]{Environment.NewLine}{message}";
-            if (tweet.IsRetweet)
-            {
-                if (tweet.RetweetedTweet != null && !message.StartsWith("RT"))
-                    message = $"[{{RT}} @{tweet.RetweetedTweet.CreatedBy.ScreenName}]{Environment.NewLine}{message}";
-                else if (tweet.RetweetedTweet != null && message.StartsWith($"RT @{tweet.RetweetedTweet.CreatedBy.ScreenName}:"))
-                    message = message.Replace($"RT @{tweet.RetweetedTweet.CreatedBy.ScreenName}:", $"[{{RT}} @{tweet.RetweetedTweet.CreatedBy.ScreenName}]{Environment.NewLine}");
-                else
-                    message = message.Replace("RT", "[{{RT}}]");
-            }
-
-            // Expand URLs
-            foreach (var url in tweet.Urls.OrderByDescending(x => x.URL.Length))
-                message = message.Replace(url.URL, url.ExpandedURL);
-
-            return message;
-        }
-
-        public ExtractedMedia[] ExtractMedia(ITweet tweet)
-        {
-            var media = tweet.Media;
-            if (tweet.IsRetweet && tweet.RetweetedTweet != null)
-                media = tweet.RetweetedTweet.Media;
-
-            var result = new List<ExtractedMedia>();
-            foreach (var m in media)
-            {
-                var mediaUrl = GetMediaUrl(m);
-                var mediaType = GetMediaType(m.MediaType, mediaUrl);
-                if (mediaType == null) continue;
-
-                var att = new ExtractedMedia
-                {
-                    MediaType = mediaType,
-                    Url = mediaUrl
-                };
-                result.Add(att);
-            }
-
-            return result.ToArray();
-        }
-
-        public string GetMediaUrl(IMediaEntity media)
-        {
-            switch (media.MediaType)
-            {
-                case "photo": return media.MediaURLHttps;
-                case "animated_gif": return media.VideoDetails.Variants[0].URL;
-                case "video": return media.VideoDetails.Variants.OrderByDescending(x => x.Bitrate).First().URL;
-                default: return null;
-            }
-        }
-
-        public string GetMediaType(string mediaType, string mediaUrl)
-        {
-            switch (mediaType)
-            {
-                case "photo":
-                    var pExt = Path.GetExtension(mediaUrl);
-                    switch (pExt)
-                    {
-                        case ".jpg":
-                        case ".jpeg":
-                            return "image/jpeg";
-                        case ".png":
-                            return "image/png";
-                    }
-                    return null;
-
-                case "animated_gif":
-                    var vExt = Path.GetExtension(mediaUrl);
-                    switch (vExt)
-                    {
-                        case ".gif":
-                            return "image/gif";
-                        case ".mp4":
-                            return "video/mp4";
-                    }
-                    return "image/gif";
-                case "video":
-                    return "video/mp4";
-            }
-            return null;
-        }
-    }
-}

src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs

@@ -15,5 +15,8 @@ namespace BirdsiteLive.Twitter.Models
         public bool IsThread { get; set; }
         public bool IsRetweet { get; set; }
         public string RetweetUrl { get; set; }
+        public long RetweetId { get; set; }
+        public TwitterUser OriginalAuthor { get; set; }
+		public string QuoteTweetUrl { get; set; }
     }
 }

src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs

@@ -1,64 +1,139 @@
 using System;
 using System.Threading;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using System.Linq;
 using System.Threading.Tasks;
 using BirdsiteLive.Common.Settings;
 using Microsoft.Extensions.Logging;
-using Tweetinvi;
+using System.Net.Http;
+using System.Net;
+using System.Net.Http.Headers;
+using System.Text;
+using System.Text.Json;
+using System.Threading.RateLimiting;
 
 namespace BirdsiteLive.Twitter.Tools
 {
     public interface ITwitterAuthenticationInitializer
     {
-        void EnsureAuthenticationIsInitialized();
+        Task<HttpClient> MakeHttpClient();
+        HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint, bool addToken);
+        Task RefreshClient(HttpRequestMessage client);
     }
 
     public class TwitterAuthenticationInitializer : ITwitterAuthenticationInitializer
     {
-        private readonly TwitterSettings _settings;
         private readonly ILogger<TwitterAuthenticationInitializer> _logger;
         private static bool _initialized;
-        private readonly SemaphoreSlim _semaphoregate = new SemaphoreSlim(1);
+        private readonly IHttpClientFactory _httpClientFactory;
+        private ConcurrentDictionary<String, String> _token2 = new ConcurrentDictionary<string, string>();
+        static Random rnd = new Random();
+        private RateLimiter _rateLimiter;
+        private const int _targetClients = 3;
+        private InstanceSettings _instanceSettings;
+        private readonly (string, string)[] _apiKeys = new[]
+        {
+            ("IQKbtAYlXLripLGPWd0HUA", "GgDYlkSvaPxGxC4X8liwpUoqKwwr3lCADbz8A7ADU"), // iPhone
+            ("3nVuSoBZnx6U4vzUxf5w", "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"), // Android
+            ("CjulERsDeqhhjSme66ECg", "IQWdVyqFxghAtURHGeGiWAsmCAGmdW3WmbEx6Hck"), // iPad
+            ("3rJOl1ODzm9yZy63FACdg", "5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8"), // Mac
+        };
+        public String BearerToken { 
+            get
+            {
+                return _instanceSettings.TwitterBearerToken;
+            }
+        }
 
         #region Ctor
-        public TwitterAuthenticationInitializer(TwitterSettings settings, ILogger<TwitterAuthenticationInitializer> logger)
+        public TwitterAuthenticationInitializer(IHttpClientFactory httpClientFactory, InstanceSettings settings, ILogger<TwitterAuthenticationInitializer> logger)
         {
-            _settings = settings;
             _logger = logger;
+            _instanceSettings = settings;
+            _httpClientFactory = httpClientFactory;
         }
         #endregion
 
-        public void EnsureAuthenticationIsInitialized()
+        private async Task<string> GenerateBearerToken()
         {
-            if (_initialized) return;
-            _semaphoregate.Wait();
-           
-            try
+            var httpClient = _httpClientFactory.CreateClient();
+            using (var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.twitter.com/oauth2/token?grant_type=client_credentials"))
             {
-                if (_initialized) return;
-                InitTwitterCredentials();
-            }
-            finally
-            {
-                _semaphoregate.Release();
+                int r = rnd.Next(_apiKeys.Length);
+                var (login, password) = _apiKeys[r];
+                var authValue = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{login}:{password}")));
+                request.Headers.Authorization = authValue;
+
+                var httpResponse = await httpClient.SendAsync(request);
+
+                var c = await httpResponse.Content.ReadAsStringAsync();
+                httpResponse.EnsureSuccessStatusCode();
+                var doc = JsonDocument.Parse(c);
+                var token = doc.RootElement.GetProperty("access_token").GetString();
+                return token;
             }
+            
         }
 
-        private void InitTwitterCredentials()
+
+        public async Task RefreshClient(HttpRequestMessage req)
         {
-            for (;;)
+            string token = req.Headers.GetValues("x-guest-token").First();
+
+            _token2.TryRemove(token, out _);
+
+            await RefreshCred();
+            await Task.Delay(1000);
+            await RefreshCred();
+        }
+
+        private async Task RefreshCred()
+        {
+            (string bearer, string guest) = await GetCred();
+            _token2.TryAdd(guest, bearer);
+        }
+
+        private async Task<(string, string)> GetCred()
+        {
+            string token;
+            var httpClient = _httpClientFactory.CreateClient();
+            string bearer = await GenerateBearerToken();
+            using (var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.twitter.com/1.1/guest/activate.json"))
             {
-                try
-                {
-                    Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true);
-                    _initialized = true;
-                    return;
-                }
-                catch (Exception e)
-                {
-                    _logger.LogError(e, "Twitter Authentication Failed");
-                    Thread.Sleep(250);
-                }
+                request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer); 
+
+                var httpResponse = await httpClient.SendAsync(request);
+
+                var c = await httpResponse.Content.ReadAsStringAsync();
+                httpResponse.EnsureSuccessStatusCode();
+                var doc = JsonDocument.Parse(c);
+                token = doc.RootElement.GetProperty("guest_token").GetString();
             }
+
+            return (bearer, token);
+
+        }
+
+        public async Task<HttpClient> MakeHttpClient()
+        {
+            if (_token2.Count < _targetClients)
+                await RefreshCred();
+            return _httpClientFactory.CreateClient();
+        }
+        public HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint, bool addToken)
+        {
+            var request = new HttpRequestMessage(m, endpoint);
+            //(string bearer, string token) = _tokens[r];
+            (string token, string bearer) = _token2.MaxBy(x => rnd.Next());
+            request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer); 
+            request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");
+            request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes");
+            if (addToken)
+                request.Headers.TryAddWithoutValidation("x-guest-token", token);
+            //request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");
+            //request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes");
+            return request;
         }
     }
 }

src/BirdsiteLive.Twitter/TwitterTweetsService.cs

@@ -1,93 +1,372 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
+using System.Net.Http;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
 using BirdsiteLive.Common.Settings;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Statistics.Domain;
-using BirdsiteLive.Twitter.Extractors;
 using BirdsiteLive.Twitter.Models;
 using BirdsiteLive.Twitter.Tools;
 using Microsoft.Extensions.Logging;
-using Tweetinvi;
-using Tweetinvi.Models;
-using Tweetinvi.Parameters;
 
 namespace BirdsiteLive.Twitter
 {
     public interface ITwitterTweetsService
     {
-        ExtractedTweet GetTweet(long statusId);
-        ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1);
+        Task<ExtractedTweet> GetTweetAsync(long statusId);
+        Task<ExtractedTweet[]> GetTimelineAsync(string username, long fromTweetId = -1);
     }
 
     public class TwitterTweetsService : ITwitterTweetsService
     {
         private readonly ITwitterAuthenticationInitializer _twitterAuthenticationInitializer;
-        private readonly ITweetExtractor _tweetExtractor;
         private readonly ITwitterStatisticsHandler _statisticsHandler;
-        private readonly ITwitterUserService _twitterUserService;
+        private readonly ICachedTwitterUserService _twitterUserService;
+        private readonly ITwitterUserDal _twitterUserDal;
         private readonly ILogger<TwitterTweetsService> _logger;
+        private readonly InstanceSettings _instanceSettings;
 
         #region Ctor
-        public TwitterTweetsService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITweetExtractor tweetExtractor, ITwitterStatisticsHandler statisticsHandler, ITwitterUserService twitterUserService, ILogger<TwitterTweetsService> logger)
+        public TwitterTweetsService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITwitterStatisticsHandler statisticsHandler, ICachedTwitterUserService twitterUserService, ITwitterUserDal twitterUserDal, InstanceSettings instanceSettings, ILogger<TwitterTweetsService> logger)
         {
             _twitterAuthenticationInitializer = twitterAuthenticationInitializer;
-            _tweetExtractor = tweetExtractor;
             _statisticsHandler = statisticsHandler;
             _twitterUserService = twitterUserService;
+            _twitterUserDal = twitterUserDal;
+            _instanceSettings = instanceSettings;
             _logger = logger;
         }
         #endregion
 
-        public ExtractedTweet GetTweet(long statusId)
+
+        public async Task<ExtractedTweet> GetTweetAsync(long statusId)
         {
+
+            var client = await _twitterAuthenticationInitializer.MakeHttpClient();
+
+
+            string reqURL =
+                "https://api.twitter.com/graphql/XjlydVWHFIDaAUny86oh2g/TweetDetail?variables=%7B%22focalTweetId%22%3A%22"
+                + statusId +
+                "%22,%22with_rux_injections%22%3Atrue,%22includePromotedContent%22%3Afalse,%22withCommunity%22%3Afalse,%22withQuickPromoteEligibilityTweetFields%22%3Afalse,%22withBirdwatchNotes%22%3Afalse,%22withSuperFollowsUserFields%22%3Afalse,%22withDownvotePerspective%22%3Afalse,%22withReactionsMetadata%22%3Afalse,%22withReactionsPerspective%22%3Afalse,%22withSuperFollowsTweetFields%22%3Afalse,%22withVoice%22%3Atrue,%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue,%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue,%22verified_phone_label_enabled%22%3Afalse,%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue,%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse,%22tweetypie_unmention_optimization_enabled%22%3Atrue,%22vibe_api_enabled%22%3Atrue,%22responsive_web_edit_tweet_api_enabled%22%3Atrue,%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Afalse,%22view_counts_everywhere_api_enabled%22%3Atrue,%22longform_notetweets_consumption_enabled%22%3Atrue,%22tweet_awards_web_tipping_enabled%22%3Afalse,%22freedom_of_speech_not_reach_fetch_enabled%22%3Afalse,%22standardized_nudges_misinfo%22%3Atrue,%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse,%22interactive_text_enabled%22%3Atrue,%22responsive_web_text_conversations_enabled%22%3Afalse,%22longform_notetweets_richtext_consumption_enabled%22%3Afalse,%22responsive_web_enhance_cards_enabled%22%3Atrue%7D";
+            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL, true);
             try
             {
-                _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized();
-                ExceptionHandler.SwallowWebExceptions = false;
-                TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended;
+                JsonDocument tweet;
+                var httpResponse = await client.SendAsync(request);
+                httpResponse.EnsureSuccessStatusCode();
+                var c = await httpResponse.Content.ReadAsStringAsync();
+                tweet = JsonDocument.Parse(c);
 
-                var tweet = Tweet.GetTweet(statusId);
-                _statisticsHandler.CalledTweetApi();
-                if (tweet == null) return null; //TODO: test this
-                return _tweetExtractor.Extract(tweet);
+
+                var timeline = tweet.RootElement.GetProperty("data").GetProperty("threaded_conversation_with_injections_v2")
+                    .GetProperty("instructions").EnumerateArray().First().GetProperty("entries").EnumerateArray();
+
+                var tweetInDoc = timeline.Where(x => x.GetProperty("entryId").GetString() == "tweet-" + statusId)
+                    .ToArray().First();
+                return await Extract( tweetInDoc );
             }
             catch (Exception e)
             {
                 _logger.LogError(e, "Error retrieving tweet {TweetId}", statusId);
+                await _twitterAuthenticationInitializer.RefreshClient(request);
                 return null;
             }
         }
 
-        public ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1)
+        public async Task<ExtractedTweet[]> GetTimelineAsync(string username, long fromTweetId = -1)
         {
-            var tweets = new List<ITweet>();
 
-            _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized();
-            ExceptionHandler.SwallowWebExceptions = false;
-            TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended;
+            var client = await _twitterAuthenticationInitializer.MakeHttpClient();
 
-            var user = _twitterUserService.GetUser(username);
-            if (user == null || user.Protected) return new ExtractedTweet[0];
-
-            if (fromTweetId == -1)
+            long userId;
+            SyncTwitterUser user = await _twitterUserDal.GetTwitterUserAsync(username);
+            if (user.TwitterUserId == default) 
             {
-                var timeline = Timeline.GetUserTimeline(user.Id, nberTweets);
-                _statisticsHandler.CalledTimelineApi();
-                if (timeline != null) tweets.AddRange(timeline);
+                var user2 = await _twitterUserService.GetUserAsync(username);
+                userId = user2.Id;
+                await _twitterUserDal.UpdateTwitterUserIdAsync(username, user2.Id);
             }
-            else
+            else 
             {
-                var timelineRequestParameters = new UserTimelineParameters
+                userId = user.TwitterUserId;
+            }
+
+
+            var reqURL =
+                "https://api.twitter.com/graphql/pNl8WjKAvaegIoVH--FuoQ/UserTweetsAndReplies?variables=%7B%22userId%22%3A%22" +
+                userId + "%22,%22count%22%3A40,%22includePromotedContent%22%3Atrue,%22withCommunity%22%3Atrue,%22withSuperFollowsUserFields%22%3Atrue,%22withDownvotePerspective%22%3Afalse,%22withReactionsMetadata%22%3Afalse,%22withReactionsPerspective%22%3Afalse,%22withSuperFollowsTweetFields%22%3Atrue,%22withVoice%22%3Atrue,%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue,%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue,%22verified_phone_label_enabled%22%3Afalse,%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue,%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse,%22tweetypie_unmention_optimization_enabled%22%3Atrue,%22vibe_api_enabled%22%3Atrue,%22responsive_web_edit_tweet_api_enabled%22%3Atrue,%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue,%22view_counts_everywhere_api_enabled%22%3Atrue,%22longform_notetweets_consumption_enabled%22%3Atrue,%22tweet_awards_web_tipping_enabled%22%3Afalse,%22freedom_of_speech_not_reach_fetch_enabled%22%3Afalse,%22standardized_nudges_misinfo%22%3Atrue,%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse,%22interactive_text_enabled%22%3Atrue,%22responsive_web_text_conversations_enabled%22%3Afalse,%22longform_notetweets_richtext_consumption_enabled%22%3Afalse,%22responsive_web_enhance_cards_enabled%22%3Afalse%7D";
+            JsonDocument results;
+            List<ExtractedTweet> extractedTweets = new List<ExtractedTweet>();
+            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL, true);
+            try
+            {
+
+                var httpResponse = await client.SendAsync(request);
+                httpResponse.EnsureSuccessStatusCode();
+                var c = await httpResponse.Content.ReadAsStringAsync();
+                results = JsonDocument.Parse(c);
+
+                _statisticsHandler.CalledTweetApi();
+            }
+            catch (HttpRequestException e)
+            {
+                _logger.LogError(e, "Error retrieving timeline of {Username}; refreshing client", username);
+                await _twitterAuthenticationInitializer.RefreshClient(request);
+                return null;
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, "Error retrieving timeline ", username);
+                return null;
+            }
+
+            var timeline = results.RootElement.GetProperty("data").GetProperty("user").GetProperty("result")
+                .GetProperty("timeline_v2").GetProperty("timeline").GetProperty("instructions").EnumerateArray();
+
+            foreach (JsonElement timelineElement in timeline) 
+            {
+                if (timelineElement.GetProperty("type").GetString() != "TimelineAddEntries")
+                    continue;
+
+                
+                foreach (JsonElement tweet in timelineElement.GetProperty("entries").EnumerateArray())
                 {
-                    SinceId = fromTweetId,
-                    MaximumNumberOfTweetsToRetrieve = nberTweets
-                };
-                var timeline = Timeline.GetUserTimeline(user.Id, timelineRequestParameters);
-                _statisticsHandler.CalledTimelineApi();
-                if (timeline != null) tweets.AddRange(timeline);
+                    if (tweet.GetProperty("content").GetProperty("entryType").GetString() != "TimelineTimelineItem")
+                        continue;
+                    
+                    try 
+                    {
+                        JsonElement userDoc = tweet.GetProperty("content").GetProperty("itemContent")
+                                .GetProperty("tweet_results").GetProperty("core").GetProperty("user_results");
+
+                        TwitterUser tweetUser = _twitterUserService.Extract(userDoc);
+                        _twitterUserService.AddUser(tweetUser);
+                    }
+                    catch (Exception _)
+                    {}
+
+                    try 
+                    {   
+                        var extractedTweet = await Extract(tweet);
+
+                        if (extractedTweet.Id == fromTweetId)
+                            break;
+
+                        extractedTweets.Add(extractedTweet);
+                    }
+                    catch (Exception e)
+                    {
+                        _logger.LogError("Tried getting timeline from user " + username + ", but got error: \n" +
+                                         e.Message + e.StackTrace + e.Source);
+
+                    }
+
+                }
             }
 
-            return tweets.Select(_tweetExtractor.Extract).ToArray();
+            return extractedTweets.ToArray();
+        }
+
+        private async Task<ExtractedTweet> Extract(JsonElement tweet)
+        {
+
+            JsonElement retweet;
+            TwitterUser OriginalAuthor;
+            JsonElement inReplyToPostIdElement;
+            JsonElement inReplyToUserElement;
+            string inReplyToUser = null;
+            long? inReplyToPostId = null;
+            long retweetId = default;
+
+            string userName = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("core").GetProperty("user_results")
+                    .GetProperty("result").GetProperty("legacy").GetProperty("screen_name").GetString();
+
+            bool isReply = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .TryGetProperty("in_reply_to_status_id_str", out inReplyToPostIdElement);
+            tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .TryGetProperty("in_reply_to_screen_name", out inReplyToUserElement);
+            if (isReply) 
+            {
+                inReplyToPostId = Int64.Parse(inReplyToPostIdElement.GetString());
+                inReplyToUser = inReplyToUserElement.GetString();
+            }
+            bool isRetweet = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .TryGetProperty("retweeted_status_result", out retweet);
+            string MessageContent;
+            if (!isRetweet)
+            {
+                MessageContent = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .GetProperty("full_text").GetString();
+                bool isNote = tweet.GetProperty("content").GetProperty("itemContent")
+                        .GetProperty("tweet_results").GetProperty("result")
+                        .TryGetProperty("note_tweet", out var note);
+                if (isNote)
+                {
+                    MessageContent = note.GetProperty("note_tweet_results").GetProperty("result")
+                        .GetProperty("text").GetString();
+                }
+                OriginalAuthor = null;
+            }
+            else 
+            {
+                MessageContent = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .GetProperty("retweeted_status_result").GetProperty("result")
+                    .GetProperty("legacy").GetProperty("full_text").GetString();
+                bool isNote = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .GetProperty("retweeted_status_result").GetProperty("result")
+                    .TryGetProperty("note_tweet", out var note);
+                if (isNote)
+                {
+                    MessageContent = note.GetProperty("note_tweet_results").GetProperty("result")
+                        .GetProperty("text").GetString();
+                }
+                string OriginalAuthorUsername = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .GetProperty("retweeted_status_result").GetProperty("result")
+                    .GetProperty("core").GetProperty("user_results").GetProperty("result")
+                    .GetProperty("legacy").GetProperty("screen_name").GetString();
+                OriginalAuthor = await _twitterUserService.GetUserAsync(OriginalAuthorUsername);
+                retweetId = Int64.Parse(tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .GetProperty("retweeted_status_result").GetProperty("result")
+                    .GetProperty("rest_id").GetString());
+            }
+
+            string creationTime = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .GetProperty("created_at").GetString().Replace(" +0000", "");
+
+            JsonElement extendedEntities;
+            bool hasMedia = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .TryGetProperty("extended_entities", out extendedEntities);
+
+            JsonElement.ArrayEnumerator urls = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .GetProperty("entities").GetProperty("urls").EnumerateArray();
+            foreach (JsonElement url in urls)
+            {
+                string tco = url.GetProperty("url").GetString();
+                string goodUrl = url.GetProperty("expanded_url").GetString();
+                MessageContent = MessageContent.Replace(tco, goodUrl);
+            }
+            
+            List<ExtractedMedia> Media = new List<ExtractedMedia>();
+            if (hasMedia) 
+            {
+                foreach (JsonElement media in extendedEntities.GetProperty("media").EnumerateArray())
+                {
+                    var type = media.GetProperty("type").GetString();
+                    string url = "";
+                    if (type == "video" || type == "animated_gif")
+                    {
+                        var bitrate = -1;
+                        foreach (JsonElement v in media.GetProperty("video_info").GetProperty("variants").EnumerateArray())
+                        {
+                            if (v.GetProperty("content_type").GetString() !=  "video/mp4")
+                                continue;
+                            int vBitrate = v.GetProperty("bitrate").GetInt32();
+                            if (vBitrate > bitrate)
+                            {
+                                bitrate = vBitrate;
+                                url = v.GetProperty("url").GetString();
+                            }
+                        }
+                    }
+                    else 
+                    {
+                        url = media.GetProperty("media_url_https").GetString();
+                    }
+                    var m = new ExtractedMedia
+                    {
+                        MediaType = GetMediaType(type, media.GetProperty("media_url_https").GetString()),
+                        Url = url,
+                    };
+                    Media.Add(m);
+
+                    MessageContent = MessageContent.Replace(media.GetProperty("url").GetString(), "");
+                }
+            }
+
+            bool isQuoteTweet = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .GetProperty("is_quote_status").GetBoolean();
+
+			string quoteTweetLink = "";
+			if (isQuoteTweet) 
+            {
+
+                quoteTweetLink = tweet.GetProperty("content").GetProperty("itemContent")
+                        .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                        .GetProperty("quoted_status_permalink").GetProperty("expanded").GetString();
+                quoteTweetLink = quoteTweetLink.Replace("https://twitter.com/", $"https://{_instanceSettings.Domain}/users/");
+                quoteTweetLink = quoteTweetLink.Replace("/status/", "/statuses/");
+            }
+            var extractedTweet = new ExtractedTweet
+            {
+                Id = Int64.Parse(tweet.GetProperty("entryId").GetString().Replace("tweet-", "")),
+                InReplyToStatusId = inReplyToPostId,
+                InReplyToAccount = inReplyToUser,
+                MessageContent = MessageContent.Trim(),
+                CreatedAt = DateTime.ParseExact(creationTime, "ddd MMM dd HH:mm:ss yyyy", System.Globalization.CultureInfo.InvariantCulture),
+                IsReply = isReply,
+                IsThread = userName == inReplyToUser,
+                IsRetweet = isRetweet,
+                Media = Media.Count() == 0 ? null : Media.ToArray(),
+                RetweetUrl = "https://t.co/123",
+                RetweetId = retweetId,
+                OriginalAuthor = OriginalAuthor,
+            };
+
+            if (isQuoteTweet) extractedTweet.QuoteTweetUrl = quoteTweetLink;
+
+			return extractedTweet;
+         
+        }
+        private string GetMediaType(string mediaType, string mediaUrl)
+        {
+            switch (mediaType)
+            {
+                case "photo":
+                    var pExt = Path.GetExtension(mediaUrl);
+                    switch (pExt)
+                    {
+                        case ".jpg":
+                        case ".jpeg":
+                            return "image/jpeg";
+                        case ".png":
+                            return "image/png";
+                    }
+                    return null;
+
+                case "animated_gif":
+                    var vExt = Path.GetExtension(mediaUrl);
+                    switch (vExt)
+                    {
+                        case ".gif":
+                            return "image/gif";
+                        case ".mp4":
+                            return "video/mp4";
+                    }
+                    return "image/gif";
+                case "video":
+                    return "video/mp4";
+            }
+            return null;
         }
     }
-}
+}

src/BirdsiteLive.Twitter/TwitterUserService.cs

@@ -1,19 +1,20 @@
 using System;
 using System.Linq;
+using System.Net.Http;
+using System.Text.Json;
+using System.Threading.Tasks;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Statistics.Domain;
 using BirdsiteLive.Twitter.Models;
 using BirdsiteLive.Twitter.Tools;
 using Microsoft.Extensions.Logging;
-using Tweetinvi;
-using Tweetinvi.Exceptions;
-using Tweetinvi.Models;
 
 namespace BirdsiteLive.Twitter
 {
     public interface ITwitterUserService
     {
-        TwitterUser GetUser(string username);
+        Task<TwitterUser> GetUserAsync(string username);
+        TwitterUser Extract (JsonElement result);
         bool IsUserApiRateLimited();
     }
 
@@ -22,6 +23,8 @@ namespace BirdsiteLive.Twitter
         private readonly ITwitterAuthenticationInitializer _twitterAuthenticationInitializer;
         private readonly ITwitterStatisticsHandler _statisticsHandler;
         private readonly ILogger<TwitterUserService> _logger;
+        
+        private readonly string endpoint = "https://twitter.com/i/api/graphql/4LB4fkCe3RDLDmOEEYtueg/UserByScreenName?variables=%7B%22screen_name%22%3A%22elonmusk%22%2C%22withSafetyModeUserFields%22%3Atrue%2C%22withSuperFollowsUserFields%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22responsive_web_twitter_blue_new_verification_copy_is_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D";
 
         #region Ctor
         public TwitterUserService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITwitterStatisticsHandler statisticsHandler, ILogger<TwitterUserService> logger)
@@ -32,39 +35,44 @@ namespace BirdsiteLive.Twitter
         }
         #endregion
 
-        public TwitterUser GetUser(string username)
+        public async Task<TwitterUser> GetUserAsync(string username)
         {
-            //Check if API is saturated 
-            if (IsUserApiRateLimited()) throw new RateLimitExceededException();
 
-            //Proceed to account retrieval
-            _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized();
-            ExceptionHandler.SwallowWebExceptions = false;
-            RateLimit.RateLimitTrackerMode = RateLimitTrackerMode.TrackOnly;
-
-            IUser user;
+            JsonDocument res;
+            var client = await _twitterAuthenticationInitializer.MakeHttpClient();
+            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), endpoint.Replace("elonmusk", username), true);
             try
             {
-                user = User.GetUserFromScreenName(username);
+
+                var httpResponse = await client.SendAsync(request);
+                httpResponse.EnsureSuccessStatusCode();
+
+                var c = await httpResponse.Content.ReadAsStringAsync();
+                res = JsonDocument.Parse(c);
+                var result = res.RootElement.GetProperty("data").GetProperty("user").GetProperty("result");
+                return Extract(result);
             }
-            catch (TwitterException e)
+            catch (System.Collections.Generic.KeyNotFoundException)
             {
-                if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("User has been suspended".ToLowerInvariant())))
-                {
-                    throw new UserHasBeenSuspendedException();
-                }
-                else if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("User not found".ToLowerInvariant())))
-                {
-                    throw new UserNotFoundException();
-                }
-                else if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("Rate limit exceeded".ToLowerInvariant())))
-                {
-                    throw new RateLimitExceededException();
-                }
-                else
-                {
-                    throw;
-                }
+                throw new UserNotFoundException();
+                //if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("User has been suspended".ToLowerInvariant())))
+                //{
+                //    throw new UserHasBeenSuspendedException();
+                //}
+                //else if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("User not found".ToLowerInvariant())))
+                //{
+                //    throw new UserNotFoundException();
+                //}
+                //else
+                //{
+                //    throw;
+                //}
+            }
+            catch (HttpRequestException e)
+            {
+                _logger.LogError(e, "Error retrieving user {Username}, Refreshing client", username);
+                await _twitterAuthenticationInitializer.RefreshClient(request);
+                return null;
             }
             catch (Exception e)
             {
@@ -77,49 +85,39 @@ namespace BirdsiteLive.Twitter
             }
 
             // Expand URLs
-            var description = user.Description;
-            foreach (var descriptionUrl in user.Entities?.Description?.Urls?.OrderByDescending(x => x.URL.Length))
-                description = description.Replace(descriptionUrl.URL, descriptionUrl.ExpandedURL);
+            //var description = user.Description;
+            //foreach (var descriptionUrl in user.Entities?.Description?.Urls?.OrderByDescending(x => x.URL.Length))
+            //    description = description.Replace(descriptionUrl.URL, descriptionUrl.ExpandedURL);
+
+        }
+
+        public TwitterUser Extract(JsonElement result)
+        {
+            string profileBannerURL = null;
+            JsonElement profileBannerURLObject;
+            if (result.GetProperty("legacy").TryGetProperty("profile_banner_url", out profileBannerURLObject))
+            {
+                profileBannerURL = profileBannerURLObject.GetString();
+            }
 
             return new TwitterUser
             {
-                Id = user.Id,
-                Acct = username,
-                Name = user.Name,
-                Description = description,
-                Url = $"https://twitter.com/{username}",
-                ProfileImageUrl = user.ProfileImageUrlFullSize.Replace("http://", "https://"),
-                ProfileBackgroundImageUrl = user.ProfileBackgroundImageUrlHttps,
-                ProfileBannerURL = user.ProfileBannerURL,
-                Protected = user.Protected
+                Id = long.Parse(result.GetProperty("rest_id").GetString()),
+                Acct = result.GetProperty("legacy").GetProperty("screen_name").GetString(), 
+                Name =  result.GetProperty("legacy").GetProperty("name").GetString(), //res.RootElement.GetProperty("data").GetProperty("name").GetString(),
+                Description =  "", //res.RootElement.GetProperty("data").GetProperty("description").GetString(),
+                Url =  "", //res.RootElement.GetProperty("data").GetProperty("url").GetString(),
+                ProfileImageUrl =  result.GetProperty("legacy").GetProperty("profile_image_url_https").GetString().Replace("normal", "400x400"), 
+                ProfileBackgroundImageUrl =  profileBannerURL,
+                ProfileBannerURL = profileBannerURL,
+                Protected = false, //res.RootElement.GetProperty("data").GetProperty("protected").GetBoolean(), 
             };
+
         }
 
         public bool IsUserApiRateLimited()
         {
-            // Retrieve limit from tooling
-            _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized();
-            ExceptionHandler.SwallowWebExceptions = false;
-            RateLimit.RateLimitTrackerMode = RateLimitTrackerMode.TrackOnly;
-
-            try
-            {
-                var queryRateLimits = RateLimit.GetQueryRateLimit("https://api.twitter.com/1.1/users/show.json?screen_name=mastodon");
-
-                if (queryRateLimits != null)
-                {
-                    return queryRateLimits.Remaining <= 0;
-                }
-            }
-            catch (Exception e)
-            {
-                _logger.LogError(e, "Error retrieving rate limits");
-            }
-
-            // Fallback
-            var currentCalls = _statisticsHandler.GetCurrentUserCalls();
-            var maxCalls = _statisticsHandler.GetStatistics().UserCallsMax;
-            return currentCalls >= maxCalls;
+            return false;
         }
     }
 }

src/BirdsiteLive.sln

@@ -47,9 +47,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Moderation.Tes
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Common.Tests", "Tests\BirdsiteLive.Common.Tests\BirdsiteLive.Common.Tests.csproj", "{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BSLManager", "BSLManager\BSLManager.csproj", "{4A84D351-E91B-4E58-8E20-211F0F4991D7}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BSLManager.Tests", "Tests\BSLManager.Tests\BSLManager.Tests.csproj", "{D4457271-620E-465A-B08E-7FC63C99A2F6}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdsiteLive.Twitter.Tests", "Tests\BirdsiteLive.Twitter.Tests\BirdsiteLive.Twitter.Tests.csproj", "{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -129,14 +127,10 @@ Global
 		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Release|Any CPU.Build.0 = Release|Any CPU
-		{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Release|Any CPU.Build.0 = Release|Any CPU
-		{D4457271-620E-465A-B08E-7FC63C99A2F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{D4457271-620E-465A-B08E-7FC63C99A2F6}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{D4457271-620E-465A-B08E-7FC63C99A2F6}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{D4457271-620E-465A-B08E-7FC63C99A2F6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -159,7 +153,7 @@ Global
 		{4BE541AC-8A93-4FA3-98AC-956CC2D5B748} = {DA3C160C-4811-4E26-A5AD-42B81FAF2D7C}
 		{0A311BF3-4FD9-4303-940A-A3778890561C} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
 		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
-		{D4457271-620E-465A-B08E-7FC63C99A2F6} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
+		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {69E8DCAD-4C37-4010-858F-5F94E6FBABCE}

src/BirdsiteLive/BirdsiteLive.csproj

@@ -1,17 +1,17 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
     <UserSecretsId>d21486de-a812-47eb-a419-05682bb68856</UserSecretsId>
     <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
-    <Version>0.20.0</Version>
+    <Version>1.0</Version>
+    <ContainerImageName>cloutier/bird.makeup</ContainerImageName>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Lamar.Microsoft.DependencyInjection" Version="5.0.0" />
     <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.16.0" />
-    <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.10.8" />
-    <PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="3.1.3" />
+    <PackageReference Include="Microsoft.NET.Build.Containers" Version="0.3.2" />
   </ItemGroup>
 
   <ItemGroup>
@@ -23,7 +23,4 @@
     <ProjectReference Include="..\BirdsiteLive.Twitter\BirdsiteLive.Twitter.csproj" />
     <ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL.Postgres\BirdsiteLive.DAL.Postgres.csproj" />
   </ItemGroup>
-
-
-
 </Project>

src/BirdsiteLive/Component/NodeInfoViewComponent.cs

@@ -7,7 +7,6 @@ using BirdsiteLive.Domain.Repository;
 using BirdsiteLive.Services;
 using BirdsiteLive.Statistics.Domain;
 using Microsoft.AspNetCore.Mvc;
-using Microsoft.AspNetCore.Razor.Language.Intermediate;
 
 namespace BirdsiteLive.Component
 {
@@ -37,7 +36,7 @@ namespace BirdsiteLive.Component
                                       twitterAccountPolicy == ModerationTypeEnum.BlackListing,
                 WhitelistingEnabled = followerPolicy == ModerationTypeEnum.WhiteListing ||
                                       twitterAccountPolicy == ModerationTypeEnum.WhiteListing,
-                InstanceSaturation = statistics.Saturation
+                SyncLag = statistics.SyncLag
             };
             
             //viewModel = new NodeInfoViewModel
@@ -55,5 +54,6 @@ namespace BirdsiteLive.Component
         public bool BlacklistingEnabled { get; set; }
         public bool WhitelistingEnabled { get; set; }
         public int InstanceSaturation { get; set; }
+        public TimeSpan SyncLag { get; set; }
     }
 }

src/BirdsiteLive/Controllers/AboutController.cs

@@ -27,18 +27,6 @@ namespace BirdsiteLive.Controllers
             return View(stats);
         }
 
-        public IActionResult Blacklisting()
-        {
-            var status = GetModerationStatus();
-            return View("Blacklisting", status);
-        }
-
-        public IActionResult Whitelisting()
-        {
-            var status = GetModerationStatus();
-            return View("Whitelisting", status);
-        }
-
         private ModerationStatus GetModerationStatus()
         {
             var status = new ModerationStatus

src/BirdsiteLive/Controllers/DebugingController.cs

@@ -10,7 +10,6 @@ using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Domain;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.Mvc.ViewFeatures;
-using Newtonsoft.Json;
 
 namespace BirdsiteLive.Controllers
 {

src/BirdsiteLive/Controllers/UsersController.cs

@@ -2,8 +2,8 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text.Json;
 using System.Net.Mime;
-using System.Runtime.InteropServices.WindowsRuntime;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
@@ -11,6 +11,8 @@ using BirdsiteLive.ActivityPub;
 using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.Common.Regexes;
 using BirdsiteLive.Common.Settings;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Domain;
 using BirdsiteLive.Models;
 using BirdsiteLive.Tools;
@@ -20,27 +22,32 @@ using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Primitives;
-using Newtonsoft.Json;
 
 namespace BirdsiteLive.Controllers
 {
     public class UsersController : Controller
     {
-        private readonly ITwitterUserService _twitterUserService;
-        private readonly ITwitterTweetsService _twitterTweetService;
+        private readonly ICachedTwitterUserService _twitterUserService;
+        private readonly ICachedTwitterTweetsService _twitterTweetService;
         private readonly IUserService _userService;
         private readonly IStatusService _statusService;
         private readonly InstanceSettings _instanceSettings;
+        private readonly IFollowersDal _followersDal;
+        private readonly ITwitterUserDal _twitterUserDal;
+        private readonly IActivityPubService _activityPubService;
         private readonly ILogger<UsersController> _logger;
 
         #region Ctor
-        public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ITwitterTweetsService twitterTweetService, ILogger<UsersController> logger)
+        public UsersController(ICachedTwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ICachedTwitterTweetsService twitterTweetService, IFollowersDal followersDal, ITwitterUserDal twitterUserDal, IActivityPubService activityPubService, ILogger<UsersController> logger)
         {
             _twitterUserService = twitterUserService;
             _userService = userService;
             _statusService = statusService;
             _instanceSettings = instanceSettings;
             _twitterTweetService = twitterTweetService;
+            _followersDal = followersDal;
+            _twitterUserDal = twitterUserDal;
+            _activityPubService = activityPubService;
             _logger = logger;
         }
         #endregion
@@ -60,7 +67,7 @@ namespace BirdsiteLive.Controllers
         [Route("/@{id}")]
         [Route("/users/{id}")]
         [Route("/users/{id}/remote_follow")]
-        public IActionResult Index(string id)
+        public async Task<IActionResult> Index(string id)
         {
             _logger.LogTrace("User Index: {Id}", id);
 
@@ -76,7 +83,7 @@ namespace BirdsiteLive.Controllers
             {
                 try
                 {
-                    user = _twitterUserService.GetUser(id);
+                    user = await _twitterUserService.GetUserAsync(id);
                 }
                 catch (UserNotFoundException)
                 {
@@ -112,7 +119,7 @@ namespace BirdsiteLive.Controllers
                     if (isSaturated) return new ObjectResult("Too Many Requests") { StatusCode = 429 };
                     if (notFound) return NotFound();
                     var apUser = _userService.GetUser(user);
-                    var jsonApUser = JsonConvert.SerializeObject(apUser);
+                    var jsonApUser = System.Text.Json.JsonSerializer.Serialize(apUser);
                     return Content(jsonApUser, "application/activity+json; charset=utf-8");
                 }
             }
@@ -120,6 +127,12 @@ namespace BirdsiteLive.Controllers
             if (isSaturated) return View("ApiSaturated");
             if (notFound) return View("UserNotFound");
 
+            Follower[] followers = new Follower[] { };
+            
+            var userDal = await _twitterUserDal.GetTwitterUserAsync(user.Acct);
+            if (userDal != null)
+                followers = await _followersDal.GetFollowersAsync(userDal.Id);
+
             var displayableUser = new DisplayTwitterUser
             {
                 Name = user.Name,
@@ -128,6 +141,8 @@ namespace BirdsiteLive.Controllers
                 Url = user.Url,
                 ProfileImageUrl = user.ProfileImageUrl,
                 Protected = user.Protected,
+                FollowerCount = followers.Length,
+                MostPopularServer = followers.GroupBy(x => x.Host).OrderByDescending(x => x.Count()).Select(x => x.Key).FirstOrDefault("N/A"),
 
                 InstanceHandle = $"@{user.Acct.ToLowerInvariant()}@{_instanceSettings.Domain}"
             };
@@ -136,31 +151,58 @@ namespace BirdsiteLive.Controllers
 
         [Route("/@{id}/{statusId}")]
         [Route("/users/{id}/statuses/{statusId}")]
-        public IActionResult Tweet(string id, string statusId)
+        public async Task<IActionResult> Tweet(string id, string statusId)
         {
             var acceptHeaders = Request.Headers["Accept"];
+            if (!long.TryParse(statusId, out var parsedStatusId))
+                return NotFound();
+
+            var tweet = await _twitterTweetService.GetTweetAsync(parsedStatusId);
+            if (tweet == null)
+                return NotFound();
+            
+            var user = await _twitterUserService.GetUserAsync(id);
+
+            var status = _statusService.GetStatus(id, tweet);
+
             if (acceptHeaders.Any())
             {
                 var r = acceptHeaders.First();
+
                 if (r.Contains("application/activity+json"))
                 {
-                    if (!long.TryParse(statusId, out var parsedStatusId))
-                        return NotFound();
-
-                    var tweet = _twitterTweetService.GetTweet(parsedStatusId);
-                    if (tweet == null)
-                        return NotFound();
-
-                    //var user = _twitterService.GetUser(id);
-                    //if (user == null) return NotFound();
-
-                    var status = _statusService.GetStatus(id, tweet);
-                    var jsonApUser = JsonConvert.SerializeObject(status);
+                    var jsonApUser = JsonSerializer.Serialize(status);
                     return Content(jsonApUser, "application/activity+json; charset=utf-8");
                 }
             }
 
-            return Redirect($"https://twitter.com/{id}/status/{statusId}");
+            //return Redirect($"https://twitter.com/{id}/status/{statusId}");
+            var displayTweet = new DisplayTweet 
+            {
+                Text = tweet.MessageContent,
+                OgUrl = $"https://twitter.com/{id}/status/{statusId}",
+                UserProfileImage = user.ProfileImageUrl,
+                UserName = user.Name,
+            };
+            return View(displayTweet);
+        }
+
+        [Route("/users/{id}/statuses/{statusId}/activity")]
+        public async Task<IActionResult> Activity(string id, string statusId)
+        {
+            if (!long.TryParse(statusId, out var parsedStatusId))
+                return NotFound();
+
+            var tweet = await _twitterTweetService.GetTweetAsync(parsedStatusId);
+            if (tweet == null)
+                return NotFound();
+            
+            var user = await _twitterUserService.GetUserAsync(id);
+
+            var status = _statusService.GetActivity(id, tweet);
+
+            var jsonApUser = JsonSerializer.Serialize(status);
+            return Content(jsonApUser, "application/activity+json; charset=utf-8");
         }
 
         [Route("/users/{id}/inbox")]
@@ -243,8 +285,54 @@ namespace BirdsiteLive.Controllers
             {
                 id = $"https://{_instanceSettings.Domain}/users/{id}/followers"
             };
-            var jsonApUser = JsonConvert.SerializeObject(followers);
+            var jsonApUser = JsonSerializer.Serialize(followers);
             return Content(jsonApUser, "application/activity+json; charset=utf-8");
         }
+
+        [Route("/users/{actor}/remote_follow")]
+        [HttpPost]
+        public async Task<IActionResult> RemoteFollow(string actor)
+        {
+            StringValues webfingerValues;
+
+            if (!Request.Form.TryGetValue("webfinger", out webfingerValues)) return BadRequest();
+
+            var webfinger = webfingerValues.First();
+
+            if (webfinger.Length < 1 || actor.Length < 1) return BadRequest();
+
+            if (webfinger[0] == '@') webfinger = webfinger[1..];
+
+            if (webfinger.IndexOf("@") < 0 || ! new Regex("^[A-Za-z0-9_]*$").IsMatch(webfinger.Split('@')[0]) || ! new Regex("^[A-Za-z0-9_]*$").IsMatch(actor) || Uri.CheckHostName(webfinger.Split('@')[1]) == UriHostNameType.Unknown)
+            {
+                return BadRequest();
+            }
+
+            WebFingerData webfingerData;
+
+            try
+            {
+                webfingerData = await _activityPubService.WebFinger(webfinger);
+            }
+            catch(Exception e)
+            {
+                _logger.LogError("Could not WebFinger {user}: {exception}", webfinger, e);
+                return NotFound();
+            }
+
+            string redirectLink = "";
+
+            foreach(var link in webfingerData.links)
+            {
+                if(link.rel == "http://ostatus.org/schema/1.0/subscribe" && link.template.Length > 0)
+                {
+                    redirectLink = link.template.Replace("{uri}", "https://" + _instanceSettings.Domain + "/users/" + actor);
+                }
+            }
+
+            if (redirectLink == "") return NotFound();
+
+            return Redirect(redirectLink);
+        }
     }
 }

src/BirdsiteLive/Controllers/WellKnownController.cs

@@ -59,6 +59,12 @@ namespace BirdsiteLive.Controllers
             return new JsonResult(nodeInfo);
         }
 
+        [Route("/.well-known/host-meta")]
+        public IActionResult HostMeta()
+        {
+            return Content($"<?xml version=\"1.0\" encoding=\"UTF-8\"?><XRD xmlns=\"http://docs.oasis-open.org/ns/xri/xrd-1.0\"><Link rel=\"lrdd\" template=\"https://{_settings.Domain}/.well-known/webfinger?resource={{uri}}\" type=\"application/xrd+xml\" /></XRD>", "application/xrd+xml; charset=utf-8");
+        }
+
         [Route("/nodeinfo/{id}.json")]
         public async Task<IActionResult> NodeInfo(string id)
         {
@@ -142,7 +148,7 @@ namespace BirdsiteLive.Controllers
         }
 
         [Route("/.well-known/webfinger")]
-        public IActionResult Webfinger(string resource = null)
+        public async Task<IActionResult> Webfinger(string resource = null)
         {
             if (string.IsNullOrWhiteSpace(resource)) 
                 return BadRequest();
@@ -203,7 +209,7 @@ namespace BirdsiteLive.Controllers
 
             try
             {
-                _twitterUserService.GetUser(name);
+                await _twitterUserService.GetUserAsync(name);
             }
             catch (UserNotFoundException)
             {

src/BirdsiteLive/Models/DisplayTweet.cs

@@ -0,0 +1,10 @@
+namespace BirdsiteLive.Models
+{
+    public class DisplayTweet
+    {
+        public string Text { get; set; }
+        public string OgUrl { get; set; }
+        public string UserProfileImage { get; set; }
+        public string UserName { get; set; }
+    }
+}

src/BirdsiteLive/Models/DisplayTwitterUser.cs

@@ -8,6 +8,8 @@
         public string Url { get; set; }
         public string ProfileImageUrl { get; set; }
         public bool Protected { get; set; }
+        public int FollowerCount { get; set; }
+        public string MostPopularServer { get; set; }
 
         public string InstanceHandle { get; set; }
     }

src/BirdsiteLive/Properties/launchSettings.json

@@ -19,16 +19,18 @@
       "commandName": "Project",
       "launchBrowser": true,
       "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development"
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "Instance__ParallelTwitterRequests": "0"
       },
       "applicationUrl": "http://localhost:5000"
     },
     "Docker": {
       "commandName": "Docker",
       "launchBrowser": true,
+      "applicationUrl": "http://localhost:5000",
       "launchUrl": "{Scheme}://{ServiceHost}:{ServicePort}",
       "publishAllPorts": true,
-      "useSSL": true
+      "useSSL": false
     }
   }
 }

src/BirdsiteLive/Services/CachedStatisticsService.cs

@@ -13,41 +13,55 @@ namespace BirdsiteLive.Services
     public class CachedStatisticsService : ICachedStatisticsService
     {
         private readonly ITwitterUserDal _twitterUserDal;
+        private readonly IFollowersDal _followersDal;
 
-        private static CachedStatistics _cachedStatistics;
+        private static Task<CachedStatistics> _cachedStatistics;
         private readonly InstanceSettings _instanceSettings;
 
         #region Ctor
-        public CachedStatisticsService(ITwitterUserDal twitterUserDal, InstanceSettings instanceSettings)
+        public CachedStatisticsService(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, InstanceSettings instanceSettings)
         {
             _twitterUserDal = twitterUserDal;
             _instanceSettings = instanceSettings;
+            _followersDal = followersDal;
+            _cachedStatistics = CreateStats();
         }
         #endregion
 
         public async Task<CachedStatistics> GetStatisticsAsync()
         {
-            if (_cachedStatistics == null ||
-                (DateTime.UtcNow - _cachedStatistics.RefreshedTime).TotalMinutes > 15)
+            var stats = await _cachedStatistics;
+            if ((DateTime.UtcNow - stats.RefreshedTime).TotalMinutes > 5)
             {
-                var twitterUserMax = _instanceSettings.MaxUsersCapacity;
-                var twitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync();
-                var saturation = (int)((double)twitterUserCount / twitterUserMax * 100);
-
-                _cachedStatistics = new CachedStatistics
-                {
-                    RefreshedTime = DateTime.UtcNow,
-                    Saturation = saturation
-                };
+                _cachedStatistics = CreateStats();
             }
 
-            return _cachedStatistics;
+            return stats;
+        }
+
+        private async Task<CachedStatistics> CreateStats()
+        {
+            var twitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync();
+            var twitterSyncLag = await _twitterUserDal.GetTwitterSyncLag();
+            var fediverseUsers = await _followersDal.GetFollowersCountAsync();
+
+            var stats = new CachedStatistics
+            {
+                RefreshedTime = DateTime.UtcNow,
+                SyncLag = twitterSyncLag,
+                TwitterUsers = twitterUserCount,
+                FediverseUsers = fediverseUsers
+            };
+        
+            return stats;
         }
     }
 
     public class CachedStatistics
     {
         public DateTime RefreshedTime { get; set; }
-        public int Saturation { get; set; }
+        public TimeSpan SyncLag { get; set; }
+        public int TwitterUsers { get; set; }
+        public int FediverseUsers { get; set; }
     }
 }

src/BirdsiteLive/Startup.cs

@@ -2,6 +2,7 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Net.Http;
+using System.Net.Http.Headers;
 using System.Threading.Tasks;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Common.Structs;
@@ -9,6 +10,7 @@ using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Postgres.DataAccessLayers;
 using BirdsiteLive.DAL.Postgres.Settings;
 using BirdsiteLive.Models;
+using BirdsiteLive.Services;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Tools;
 using Lamar;
@@ -50,14 +52,19 @@ namespace BirdsiteLive
 
             services.AddControllersWithViews();
 
-            services.AddHttpClient();
-        }
+			services.AddHttpClient("BirdsiteLIVE", httpClient =>
+			{
+				ProductInfoHeaderValue product = new("BirdsiteLIVE", $"fishe");
+				ProductInfoHeaderValue comment = new($"(+https://{Configuration["Instance:Domain"]})");
+				httpClient.DefaultRequestHeaders.UserAgent.Add(product);
+				httpClient.DefaultRequestHeaders.UserAgent.Add(comment);
+			});
+			services.AddHttpClient();
+
+		}
 
         public void ConfigureContainer(ServiceRegistry services)
         {
-            var twitterSettings = Configuration.GetSection("Twitter").Get<TwitterSettings>();
-            services.For<TwitterSettings>().Use(x => twitterSettings);
-
             var instanceSettings = Configuration.GetSection("Instance").Get<InstanceSettings>();
             services.For<InstanceSettings>().Use(x => instanceSettings);
 
@@ -92,6 +99,8 @@ namespace BirdsiteLive
             services.For<ITwitterUserService>().Use<TwitterUserService>().Singleton();
 
             services.For<ITwitterAuthenticationInitializer>().Use<TwitterAuthenticationInitializer>().Singleton();
+            
+            services.For<ICachedStatisticsService>().Use<CachedStatisticsService>().Singleton();
 
             services.Scan(_ =>
             {

src/BirdsiteLive/Views/About/Blacklisting.cshtml

@@ -1,27 +0,0 @@
-@using BirdsiteLive.Domain.Repository
-@model BirdsiteLive.Controllers.ModerationStatus
-@{
-    ViewData["Title"] = "Blacklisting";
-}
-
-<div class="col-12 col-sm-12 col-md-10 col-lg-8 mx-auto">
-    <h2>Blacklisting</h2>
-
-    @if (Model.Followers == ModerationTypeEnum.BlackListing)
-    {
-        <p><br />This node is blacklisting some instances and/or Fediverse users.<br /><br /></p>
-    }
-
-    @if (Model.TwitterAccounts == ModerationTypeEnum.BlackListing)
-    {
-        <p><br />This node is blacklisting some twitter users.<br /><br /></p>
-    }
-
-    @if (Model.Followers != ModerationTypeEnum.BlackListing && Model.TwitterAccounts != ModerationTypeEnum.BlackListing)
-    {
-        <p><br />This node is not using blacklisting.<br /><br /></p>
-    }
-
-    @*<h2>FAQ</h2>
-    <p>TODO</p>*@
-</div>

src/BirdsiteLive/Views/About/Index.cshtml

@@ -4,27 +4,12 @@
 }
 
 <div class="col-12 col-sm-12 col-md-10 col-lg-8 mx-auto">
-    <h2>Node Saturation</h2>
+    <h2>Service load</h2>
 
     <p>
         <br/>
-        This node usage is at @Model.Saturation%<br/>
+        There are @Model.FediverseUsers fediverse users following @Model.TwitterUsers twitter users<br/>
         <br/>
     </p>
 
-    <h2>FAQ</h2>
-    <h4>Why is there a limit on the node?</h4>
-
-    <p>BirdsiteLIVE rely on the Twitter API to provide high quality content. This API has limitations and therefore limits node capacity.</p>
-
-    <h4>What happen when the node is saturated?</h4>
-
-    <p>
-        When the saturation rate goes above 100% the node will no longer update all accounts every 15 minutes and instead will reduce the pooling rate to stay under the API limits, the more saturated a node is the less efficient it will be.<br />
-        The software doesn't scale, and it's by design.
-    </p>
-
-    <h4>How can I reduce the node's saturation?</h4>
-
-    <p>If you're not on your own node, be reasonable and don't follow too much accounts. And if you can, host your own node. BirdsiteLIVE doesn't require a lot of resources to work and therefore is really cheap to self-host.</p>
 </div>

src/BirdsiteLive/Views/About/Whitelisting.cshtml

@@ -1,27 +0,0 @@
-@using BirdsiteLive.Domain.Repository
-@model BirdsiteLive.Controllers.ModerationStatus
-@{
-    ViewData["Title"] = "Whitelisting";
-}
-
-<div class="col-12 col-sm-12 col-md-10 col-lg-8 mx-auto">
-    <h2>Whitelisting</h2>
-
-    @if (Model.Followers == ModerationTypeEnum.WhiteListing)
-    {
-        <p><br />This node is whitelisting some instances and/or Fediverse users.<br /><br /></p>
-    }
-
-    @if (Model.TwitterAccounts == ModerationTypeEnum.WhiteListing)
-    {
-        <p><br />This node is whitelisting some twitter users.<br /><br /></p>
-    }
-
-    @if (Model.Followers != ModerationTypeEnum.WhiteListing && Model.TwitterAccounts != ModerationTypeEnum.WhiteListing)
-    {
-        <p><br />This node is not using whitelisting.<br /><br /></p>
-    }
-
-    @*<h2>FAQ</h2>
-    <p>TODO</p>*@
-</div>

src/BirdsiteLive/Views/Home/Index.cshtml

@@ -7,24 +7,17 @@
     <h1 class="display-4">Welcome</h1>
     <p>
         <br />
-        BirdsiteLIVE is a Twitter to ActivityPub bridge.<br />
+        This is a Twitter to ActivityPub bridge.<br />
         Find a Twitter account below:
     </p>
     
     <form method="POST">
-        @*<div class="form-group">
-            <label for="exampleInputEmail1">Email address</label>
-            <input type="email" class="form-control" id="exampleInputEmail1" aria-describedby="emailHelp" placeholder="Enter email">
-            <small id="emailHelp" class="form-text text-muted">We'll never share your email with anyone else.</small>
-        </div>*@
         <div class="form-group">
-            @*<label for="exampleInputPassword1">Password</label>*@
             <input type="text" class="form-control col-8 col-sm-8 col-md-6 col-lg-4 mx-auto" id="handle" name="handle" autocomplete="off" placeholder="Twitter Handle">
         </div>
         <button type="submit" class="btn btn-primary">Show</button>
     </form>
 
-
     @*@if (HtmlHelperExtensions.IsDebug())
     {
         <a class="nav-link text-dark" asp-area="" asp-controller="Debuging" asp-action="Index">Debug</a>

src/BirdsiteLive/Views/Shared/Components/NodeInfo/Default.cshtml

@@ -1,22 +1,10 @@
 @model BirdsiteLive.Component.NodeInfoViewModel
 
 <div>
-    @if (ViewData.Model.WhitelistingEnabled)
-    {
-        <a asp-controller="About" asp-action="Whitelisting" class="badge badge-light" title="What does this mean?">Whitelisting Enabled</a>
-    }
-    @if (ViewData.Model.BlacklistingEnabled)
-    {
-        <a asp-controller="About" asp-action="Blacklisting" class="badge badge-light" title="What does this mean?">Blacklisting Enabled</a>
-    }
-
     <div class="node-progress-bar">
-        <div class="node-progress-bar__label"><a asp-controller="About" asp-action="Index">Instance saturation:</a></div>
-        <div class="progress node-progress-bar__bar">
-            <div class="progress-bar 
-                 @((ViewData.Model.InstanceSaturation > 50 && ViewData.Model.InstanceSaturation < 75) ? "bg-warning ":"")
-                 @((ViewData.Model.InstanceSaturation > 75 && ViewData.Model.InstanceSaturation < 100) ? "bg-danger ":"")
-                 @((ViewData.Model.InstanceSaturation > 100) ? "bg-saturation-danger ":"")" style="width: @ViewData.Model.InstanceSaturation%">@ViewData.Model.InstanceSaturation%</div>
-        </div>
+        <div class="node-progress-bar__label">
+            <a asp-controller="About" asp-action="Index">Service load:</a>
+            @Math.Ceiling(ViewData.Model.SyncLag.TotalMinutes) minutes to fetch all twitter users 
+         </div>
     </div>
 </div>

src/BirdsiteLive/Views/Shared/_Layout.cshtml

@@ -6,6 +6,24 @@
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>@Configuration.GetSection("Instance")["Name"] - @ViewData["Title"]</title>
+    @if(ViewData["AlternateLink"] != null) 
+    {
+    <link href='@ViewData["AlternateLink"]' rel='alternate' type='application/activity+json'>
+    <meta content='@ViewData["AlternateLink"]' property="og:url" />
+    }
+    @if(ViewData["MetaDescription"] != null) 
+    {
+    <meta content='@ViewData["MetaDescription"]' name='description'>
+    <meta content='@ViewData["MetaDescription"]' property="og:description" />
+    }
+    @if(ViewData["MetaTitle"] != null) 
+    {
+    <meta content='@ViewData["MetaTitle"]' name='og:title'>
+    }
+    @if(ViewData["MetaImage"] != null) 
+    {
+    <meta content='@ViewData["MetaImage"]' property="og:image" />
+    }
     <link rel="stylesheet" href="~/lib/bootstrap/dist/css/bootstrap.min.css" />
     <link rel="stylesheet" href="~/css/site.css" />
     <link rel="stylesheet" href="~/css/birdsite.css" />
@@ -47,9 +65,7 @@
         </div>
         <div class="container">
 
-            <a href="https://github.com/NicolasConstant/BirdsiteLive">Github</a> @*<a asp-area="" asp-controller="Home" asp-action="Privacy">Privacy</a>*@
-
-            <span style="float: right;">BirdsiteLIVE @System.Reflection.Assembly.GetEntryAssembly().GetName().Version.ToString(3)</span>
+            <a href="https://git.froth.zone/sam/birdsitelive/">Source Code</a> @*<a asp-area="" asp-controller="Home" asp-action="Privacy">Privacy</a>*@
         </div>
     </footer>
     <script src="~/lib/jquery/dist/jquery.min.js"></script>

src/BirdsiteLive/Views/Users/Index.cshtml

@@ -1,5 +1,4 @@
-@using Tweetinvi.Streams.Model.AccountActivity
-@model DisplayTwitterUser
+@model DisplayTwitterUser
 @{
     ViewData["Title"] = "User";
 }
@@ -29,6 +28,9 @@
         </div>
     </a>
     <br />
+    <div>
+        This account has @ViewData.Model.FollowerCount followers on the fediverse. The server with the most followers for this account is: @ViewData.Model.MostPopularServer
+    </div>
     <br />
 
     @if (ViewData.Model.Protected)
@@ -40,9 +42,14 @@
     else
     {
         <div>
-            <p>Search this handle to find it in your instance:</p>
+            <form action="/users/@ViewData.Model.Acct/remote_follow" method="post">
+                <input type="text" class="form-control mb-2" placeholder="your handle, i.e. @@lain@@pleroma.com" name="webfinger" />
+                <input type="submit" class="btn btn-primary w-100 mb-2" value="Remote follow" />
+            </form>
+
+            <p>or search this handle to find it in your instance:</p>
 
             <input type="text" name="textbox" value="@ViewData.Model.InstanceHandle" onclick="this.select()" class="form-control" readonly />
         </div>
     }
-</div>
+</div>

src/BirdsiteLive/Views/Users/Tweet.cshtml

@@ -1,8 +1,14 @@
-@{
+@using Microsoft.AspNetCore.Http.Extensions
+@model DisplayTweet
+@{
     ViewData["Title"] = "Tweet";
+    ViewData["AlternateLink"] = Context.Request.GetDisplayUrl().Replace("http", "https");
+    ViewData["MetaDescription"] = ViewData.Model.Text;
+    ViewData["MetaImage"] = ViewData.Model.UserProfileImage;
+    ViewData["MetaTitle"] = ViewData.Model.UserName;
 }
 
 <div align="center">
-    <blockquote class="twitter-tweet"><p lang="en" dir="ltr">Embedded tweet <a href="https://twitter.com/TwitterSupport/status/@ViewData.Model">Tweet</a></blockquote>
+    <blockquote class="twitter-tweet"><p lang="en" dir="ltr">@ViewData.Model.Text<br><a href="@ViewData.Model.OgUrl">See Tweet</a></blockquote>
     <script async src="https://platform.twitter.com/widgets.js" charset="utf-8"></script>
 </div>

src/BirdsiteLive/appsettings.json

@@ -10,7 +10,8 @@
     "LogLevel": {
       "Default": "Information",
       "Microsoft": "Warning",
-      "Microsoft.Hosting.Lifetime": "Information"
+      "Microsoft.Hosting.Lifetime": "Information",
+      "System.Net.Http.HttpClient": "Warning"
     }
   },
   "AllowedHosts": "*",
@@ -30,13 +31,9 @@
   "Db": {
     "Type": "postgres",
     "Host": "127.0.0.1",
-    "Name": "mydb",
-    "User": "username",
-    "Password": "password"
-  },
-  "Twitter": {
-    "ConsumerKey": "twitter.api.key",
-    "ConsumerSecret": "twitter.api.key"
+    "Name": "birdsitelive",
+    "User": "birdsitelive",
+    "Password": "birdsitelive"
   },
   "Moderation": {
     "FollowersWhiteListing": null,

src/DataAccessLayers/BirdsiteLive.DAL.Postgres/BirdsiteLive.DAL.Postgres.csproj

@@ -1,12 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Dapper" Version="2.0.35" />
-    <PackageReference Include="Npgsql" Version="4.1.3.1" />
+    <PackageReference Include="Dapper" Version="2.0.123" />
+    <PackageReference Include="Npgsql" Version="7.0.2" />
   </ItemGroup>
 
   <ItemGroup>

src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/Base/PostgresBase.cs

@@ -1,4 +1,5 @@
 using BirdsiteLive.DAL.Postgres.Settings;
+using BirdsiteLive.DAL.Models;
 using Npgsql;
 
 namespace BirdsiteLive.DAL.Postgres.DataAccessLayers.Base
@@ -6,19 +7,29 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers.Base
     public class PostgresBase
     {
         protected readonly PostgresSettings _settings;
+        protected NpgsqlDataSource _dataSource;
 
         #region Ctor
         protected PostgresBase(PostgresSettings settings)
         {
             _settings = settings;
+            var dataSourceBuilder = new NpgsqlDataSourceBuilder(settings.ConnString);
+            _dataSource = dataSourceBuilder.Build();
         }
         #endregion
 
+        protected NpgsqlDataSource DataSource
+        {
+            get
+            {
+                return _dataSource;
+            }
+        }
         protected NpgsqlConnection Connection
         {
             get
             {
-                return new NpgsqlConnection(_settings.ConnString);
+                return _dataSource.CreateConnection();
             }
         }
     }