stop begging for money when your software does not work

typo not intentional but funny

.builds/arch.yml

@@ -0,0 +1,25 @@
+image: archlinux
+packages:
+  - dotnet-sdk
+  - dotnet-runtime-6.0
+  - docker
+sources:
+  - https://git.sr.ht/~cloutier/bird.makeup
+secrets:
+  -  d9970e85-5aef-4cfd-b6ed-0ccf1be5308b
+tasks:
+  - test: |
+      sudo systemctl start docker
+      sudo docker run -d -p 5432:5432 -e POSTGRES_PASSWORD=birdsitelive -e POSTGRES_USER=birdsitelive -e POSTGRES_DB=birdsitelive postgres:15
+      cd bird.makeup/src
+      dotnet test
+  - publish-arm: |
+      cd bird.makeup/src/BirdsiteLive
+      dotnet publish --os linux --arch arm64 /t:PublishContainer -c Release
+      docker tag cloutier/bird.makeup:1.0 cloutier/bird.makeup:latest-arm
+      docker push cloutier/bird.makeup:latest-arm
+  - publish-x64: |
+      cd bird.makeup/src/BirdsiteLive
+      dotnet publish --os linux --arch x64 /t:PublishContainer -c Release
+      docker tag cloutier/bird.makeup:1.0 cloutier/bird.makeup:latest
+      docker push cloutier/bird.makeup:latest

.drone.yml

@@ -1,40 +1,29 @@
-kind: pipeline
-name: testing
-type: docker
+# kind: pipeline
+# name: testing
+# type: docker
 
-steps:
-  - name: Install Dependencies
-    image: mcr.microsoft.com/dotnet/sdk:6.0
-    commands: 
-      - dotnet restore ./src
+# steps:
+#   - name: Test
+#     image: mcr.microsoft.com/dotnet/sdk:6.0
+#     commands:
+#       - sed -i "s/127\.0\.0\.1/database/g" ./src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/Base/PostgresTestingBase.cs
+#       - dotnet test --verbosity minimal ./src
 
-  - name: Build
-    image: mcr.microsoft.com/dotnet/sdk:6.0
-    commands: 
-      - dotnet build --configuration Release ./src
-
-  - name: Test
-    image: mcr.microsoft.com/dotnet/sdk:6.0
-    commands: 
-      - sed -i "s/127\.0\.0\.1/database/g" ./src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/Base/PostgresTestingBase.cs
-      - dotnet test --verbosity minimal ./src
-
-services:
-  - name: database
-    image: postgres:15
-    environment:
-      POSTGRES_USER: birdtest
-      POSTGRES_PASSWORD: mysecretpassword 
-      POSTGRES_DB: birdsitetest
-
----
+# services:
+#   - name: database
+#     image: postgres:15
+#     environment:
+#       POSTGRES_USER: birdsitelive
+#       POSTGRES_PASSWORD: birdsitelive
+#       POSTGRES_DB: birdsitelive
 
+# ---
 kind: pipeline
 name: docker-publish
 type: docker
 
-depends_on:
-  - testing
+# depends_on:
+#   - testing
 
 steps:
   - name: Build & Publish
@@ -42,18 +31,21 @@ steps:
     image: quay.io/thegeeklab/drone-docker-buildx
     settings:
       auto_tag: true
+      tags:
+        - makeup
       repo: git.froth.zone/sam/birdsitelive
       registry: git.froth.zone
       username: sam
       password:
         from_secret: password
-      platforms: 
+      platforms:
         - linux/amd64
         - linux/arm64
     when:
       branch:
         - master
+        - makeup
       event:
         - push
     depends_on:
-      - "clone"
+      - "clone"

.github/workflows/dotnet-core.yml

@@ -10,11 +10,11 @@ jobs:
       working-directory: ./src
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v2
     - name: Launch Db for testing
       run: docker run --name postgres -e POSTGRES_DB=mytestdb -e POSTGRES_PASSWORD=mysecretpassword -d -p 5432:5432 postgres
     - name: Setup .NET Core
-      uses: actions/setup-dotnet@v3
+      uses: actions/setup-dotnet@v1
       with:
         dotnet-version: 3.1.101
     - name: Install dependencies

.gitignore

@@ -91,7 +91,6 @@ StyleCopReport.xml
 *.log
 *.vspscc
 *.vssscc
-.builds
 *.pidb
 *.svclog
 *.scc
@@ -346,11 +345,10 @@ ASALocalRun/
 # BeatPulse healthcheck temp database
 healthchecksdb
 
-# Backup folder for Package Reference Convert tool in Visual Studio 2017
-MigrationBackup/
-
 # Ionide (cross platform F# VS Code tools) working folder
 .ionide/
 /src/BSLManager/Properties/launchSettings.json
 
+backups
+
 .dccache

Dockerfile

@@ -1,17 +1,14 @@
-#See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging.
-
 FROM mcr.microsoft.com/dotnet/aspnet:6.0 AS base
 WORKDIR /app
 EXPOSE 80
 EXPOSE 443
 
 FROM mcr.microsoft.com/dotnet/sdk:6.0 AS publish
-WORKDIR /
 COPY ./src/ ./src/
-RUN dotnet publish "/src/BirdsiteLive/BirdsiteLive.csproj" -c Release -o /app/publish \
-	&& dotnet publish "/src/BSLManager/BSLManager.csproj" -r linux-x64 --self-contained true -p:PublishSingleFile=true -p:IncludeAllContentForSelfExtract=true -c Release -o /app/publish
+RUN dotnet publish "/src/BirdsiteLive/BirdsiteLive.csproj" -c Release -o /app/publish
+# RUN dotnet publish "/src/BSLManager/BSLManager.csproj" -r linux-x64 --self-contained true -p:PublishSingleFile=true -p:IncludeAllContentForSelfExtract=true -c Release -o /app/publish
 
 FROM base AS final
 WORKDIR /app
 COPY --from=publish /app/publish .
-ENTRYPOINT ["dotnet", "BirdsiteLive.dll"]
+ENTRYPOINT ["dotnet", "BirdsiteLive.dll"]

INSTALLATION.md

@@ -1,19 +1,15 @@
 # Installation
 
-## Prerequisites 
-
-You will need a Twitter API key to make BirdsiteLIVE working. First create an **Standalone App** in the [Twitter developer portal](https://developer.twitter.com/en/portal/projects-and-apps) and retrieve the API Key and API Secret Key. 
-
 ## Server prerequisites
 
 Your instance will need [docker](https://docs.docker.com/engine/install/) and [docker-compose](https://docs.docker.com/compose/install/) installed and working. 
 
 ## Setup
 
-Download the [docker-compose file](https://git.froth.zone/sam/BirdsiteLIVE/raw/branch/master/docker-compose.yml): 
+Download the [docker-compose file](https://github.com/NicolasConstant/BirdsiteLive/blob/master/docker-compose.yml): 
 
 ```
-sudo curl -L https://git.froth.zone/sam/BirdsiteLIVE/raw/branch/master/docker-compose.yml -o docker-compose.yml
+sudo curl -L https://raw.githubusercontent.com/NicolasConstant/BirdsiteLive/master/docker-compose.yml -o docker-compose.yml
 ```
 
 Then edit file: 
@@ -26,10 +22,8 @@ sudo nano docker-compose.yml
 
 #### Personal info
 
-* `Instance:Domain` the domain name you'll be using, for example use `birdsite.example.com` for the URL `https://birdsite.example.com`
+* `Instance:Domain` the domain name you'll be using, for example use `birdsite.live` for the URL `https://birdsite.live`
 * `Instance:AdminEmail` the admin's email, will be displayed in the instance /.well-known/nodeinfo endpoint
-* `Twitter:ConsumerKey` the Twitter API key
-* `Twitter:ConsumerSecret` the Twitter API secret key
 
 #### Database credentials
 
@@ -55,14 +49,35 @@ docker-compose up -d
 
 By default the app will be available on the port 5000
 
-## Nginx configuration
+## Nginx 
 
-Fill your service block as follow:
+On a Debian based distrib:
+
+```
+sudo apt update
+sudo apt install nginx
+```
+
+Check nginx status: 
+
+```
+sudo systemctl status nginx
+```
+
+### Create nginx configuration
+
+Create your nginx configuration
+
+```
+sudo nano /etc/nginx/sites-enabled/{your-domain-name.com}
+```
+
+And fill your service block as follow:
 
 ```
 server {
     listen        80;
-    server_name   birdsite.example.com;
+    server_name   {your-domain-name.com};
     location / {
         proxy_pass         http://localhost:5000;
         proxy_http_version 1.1;
@@ -90,31 +105,16 @@ After having a domain name pointing to your instance, install and setup certbot:
 
 ```
 sudo apt install certbot python3-certbot-nginx
-sudo certbot --nginx -d birdsite.example.com
+sudo certbot --nginx -d {your-domain-name.com}
 ```
 
 Make sure you're redirecting all traffic to https when asked.
 
-Finally check that the auto-renewal will work as expected:
+Finally check that the auto-revewal will work as espected:
 
 ```
 sudo certbot renew --dry-run
 ```
-## Caddy
-
-
-Or, you can use [caddy](https://caddyserver.com)
-
-```caddyfile
-birdsite.example.com {
-        encode gzip
-        header ?Cache-Control "max-age=3600"
-        reverse_proxy http://localhost:5000 {
-            header_down -Server
-        }
-}
-```
-Everything 
 
 ### Set the firewall 
 
@@ -159,11 +159,11 @@ networks:
 
 services:
     server:
-        image: pasture/birdsitelive:latest
+        image: nicolasconstant/birdsitelive:latest
         [...]
 
     db:
-        image: postgres:13
+        image: postgres:9.6
         [...]
         
 +   watchtower:

README.md

@@ -1,38 +1,44 @@
-# BirdsiteLIVE: Twitter -> ActivityPub
+# bird.makeup
 
-[![Build Status](https://ci.git.froth.zone/api/badges/sam/BirdsiteLIVE/status.svg)](https://ci.git.froth.zone/sam/BirdsiteLIVE)
-
-This project is a _fork_ of [Pasture's fork](https://git.gamers.exposed/pasture/BirdsiteLIVE) of [the original BirdsiteLIVE from NicolasConstant](https://github.com/NicolasConstant/BirdsiteLive). This fork runs in production on [bird.froth.zone](https://bird.froth.zone). Changes made in this fork include:
-
-- Rebasing the forks together.
-- (this space intentionally left blank)
-
-This fork is also available as a Docker image as `git.froth.zone/sam/birdsitelive`.
-
-The project's original README is below:
+[![builds.sr.ht status](https://builds.sr.ht/~cloutier/bird.makeup/commits/master/arch.yml.svg)](https://builds.sr.ht/~cloutier/bird.makeup/commits/master/arch.yml?)
 
 ## About
 
-BirdsiteLIVE is an ActivityPub bridge from Twitter, it's mostly a pet project/playground for me to handle ActivityPub concepts. Feel free to deploy your own instance (especially if you plan to follow a lot of users) since it use a proper Twitter API key and therefore will have limited calls ceiling (it won't scale, and it's by design).
+Bird.makeup is a way to follow Twitter users from any ActivityPub service. The aim is to make tweets appear as native a possible to the fediverse, while being as scalable as possible. The project started from BirdsiteLive, but has now been improved significantly. 
 
-## State of development
+Compared to BirdsiteLive, bird.makeup is:
 
-The code is pretty messy and far from a good state, since it's a playground for me the aim was to understand some AP concepts, not to provide a good state-of-the-art codebase. But I might refactor it to make it cleaner.
+More scalable:
+ - Twitter API calls are not rate-limited
+ - It is possible to split the Twitter crawling to multiple servers
+ - There are now integration tests for the non-official api
+ - The core pipeline has been tweaked to remove bottlenecks. As of writing this, bird.makeup supports without problems more than 20k users. 
+ - Twitter users with no followers on the fediverse will stop being fetched
 
-## Official instance
+More native to the fediverse:
+ - Retweets are propagated as boosts
+ - Activities are now "unlisted" which means that they won't polute the public timeline, but they can still be boosted
+ - WIP support for QT
 
-There's none! Please read [here why I've stopped it](https://write.as/nicolas-constant/closing-the-official-bsl-instance).
+More modern:
+ - Moved from .net core 3.1 to .net 6 which is still supported
+ - Moved from postgres 9 to 15
+ - Moved from Newtonsoft.Json to System.Text.Json
 
-## Installation
+## Official instance 
 
-I'm providing a [docker build](https://git.froth.zone/sam/-/packages/container/birdsitelive/latest). To install it on your own server, please follow [those instructions](./INSTALLATION.md). More [options](./VARIABLES.md) are also available.
+You can find the official instance here: [bird.makeup](https://bird.makeup). If you are an instance admin that prefers to not have tweets federated to you, please block the entire instance. 
 
-Also a (likely broken) [CLI](./BSLManager.md) is available for administrative tasks.
+Please consider if you really need another instance before spinning up a new one, as having multiple domain makes it harder for moderators to block twitter content. 
 
 ## License
 
-This project is licensed under the AGPLv3 License - see [LICENSE](./LICENSE) for details.
+Original code started from [BirdsiteLive](https://github.com/NicolasConstant/BirdsiteLive).
+
+This project is licensed under the AGPLv3 License - see [LICENSE](https://git.sr.ht/~cloutier/bird.makeup/tree/master/item/LICENSE) for details.
 
 ## Contact
 
-You can contact me via ActivityPub <a rel="me" href="https://fosstodon.org/@BirdsiteLIVE">here</a>.
+You can contact me via ActivityPub <a rel="me" href="https://social.librem.one/@vincent">here</a>.
+
+

VARIABLES.md

@@ -46,18 +46,9 @@ If both whitelisting and blacklisting are set, only the whitelisting will be act
 * `Instance:Name` (default: BirdsiteLIVE) the name of the instance
 * `Instance:ResolveMentionsInProfiles` (default: true) to enable or disable mentions parsing in profile's description. Resolving it will consume more User's API calls since newly discovered account can also contain references to others accounts as well. On a big instance it is recommended to disable it.
 * `Instance:PublishReplies` (default: false) to enable or disable replies publishing.
-* `Instance:UnlistedTwitterAccounts` (default: null) to enable unlisted publication for selected twitter accounts, separated by `;` (please limit this to brands and other public profiles).
-* `Instance:TwitterDomain` (default: twitter.com) redirect to a different domain (i.e. a Nitter instance) instead of Twitter in most areas
-* `Instance:TwitterDomainLabel` (default: "") if TwitterDomain is set, use this label on profile pages instead of the domain itself (i.e. you can set this to "Nitter" to show that on profiles instead of "twiiit.com")
-* `Instance:InfoBanner` (default: "") text to show in a banner on the front page
-* `Instance:ShowAboutInstanceOnProfiles` (default: true) show "About [instance name]" on profiles with a link to /About
-* `Instance:MaxFollowsPerUser` (default: 0 - no limit) limit the number of follows per user - any follow count above this number will be Rejected
-* `Instance:DiscloseInstanceRestrictions` (default: false) disclose your instance's restrictions on its About page
 * `Instance:UnlistedTwitterAccounts` (default: null) to enable unlisted publication for selected twitter accounts, separated by `;` (please limit this to brands and other public profiles). 
 * `Instance:SensitiveTwitterAccounts` (default: null) mark all media from given accounts as sensitive by default, separated by `;`. 
 * `Instance:FailingTwitterUserCleanUpThreshold` (default: 700) set the max allowed errors (due to a banned/deleted/private account) from a Twitter Account retrieval before auto-removal. (by default an account is called every 15 mins)
-* `Instance:MaxStatusFetchAge` (default: 0 - no limit) statuses with a Snowflake older than this age in days will not be fetched by the service and will instead return 410 Gone
-* `Instance:EnableQuoteRT` (default: false) enable Soapbox-style quote-RTs
 * `Instance:FailingFollowerCleanUpThreshold` (default: 30000) set the max allowed errors from a Follower (Fediverse) Account before auto-removal. (often due to account suppression, instance issues, etc)
 * `Instance:UserCacheCapacity` (default: 10000) set the caching limit of the Twitter User retrieval. Must be higher than the number of synchronized accounts on the instance.
 
@@ -73,7 +64,7 @@ networks:
 
 services:
     server:
-        image: pasture/birdsitelive:latest
+        image: nicolasconstant/birdsitelive:latest
         [...]
         environment:
             - Instance:Domain=domain.name
@@ -91,16 +82,12 @@ services:
 +           - Instance:ResolveMentionsInProfiles=false
 +           - Instance:PublishReplies=true
 +           - Instance:UnlistedTwitterAccounts=cocacola;twitter
-+           - Instance:TwitterDomain=twiiit.com
-+           - Instance:TwitterDomainLabel=Nitter
-+           - Instance:InfoBanner=This is my BirdsiteLIVE instance. There are many like it, but this one is mine.
-+           - Instance:ShowAboutInstanceOnProfiles=true
 +           - Instance:SensitiveTwitterAccounts=archillect
         networks:
         [...]
 
     db:
-        image: postgres:13
+        image: postgres:9.6
         [...]
 ```
 

docker-compose.yml

@@ -1,31 +1,31 @@
 version: "3"
 
-networks:
-    birdsitelivenetwork:
-        external: false
-
 services:
+
     server:
-        image: git.froth.zone/birdsitelive:latest
+        image: cloutier/bird.makeup:latest
         restart: always
-        container_name: birdsitelive
+        container_name: birdmakeup
         environment:
-            - Instance:Domain=domain.name
+            - Instance:Domain=bird.makeup
+            - Instance:Name=bird.makeup
             - Instance:AdminEmail=name@domain.ext
+            - Instance:ParallelTwitterRequests=50
+            - Instance:ParallelFediverseRequests=20
             - Db:Type=postgres
             - Db:Host=db
             - Db:Name=birdsitelive
             - Db:User=birdsitelive
             - Db:Password=birdsitelive
-            - Twitter:ConsumerKey=twitter.api.key
-            - Twitter:ConsumerSecret=twitter.api.key
-        networks:
-            - birdsitelivenetwork
+            - Moderation:FollowersBlackListing=bae.st
         ports:
             - "5000:80"
+        volumes:
+            - type: bind
+              source: ../key.json
+              target: /app/key.json
         depends_on:
             - db
-
     db:
         image: postgres:15
         restart: always
@@ -33,7 +33,8 @@ services:
             - POSTGRES_USER=birdsitelive
             - POSTGRES_PASSWORD=birdsitelive
             - POSTGRES_DB=birdsitelive
-        networks:
-            - birdsitelivenetwork
         volumes:
-            - ./postgres:/var/lib/postgresql/data
+            - ../postgres15:/var/lib/postgresql/data
+        ports:
+            - "5432:5432"
+

renovate.json

@@ -1,12 +0,0 @@
-{
-  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
-  "extends": [
-    "config:base",
-    ":npm",
-    ":gomod",
-    ":pinSkipCi",
-    ":docker",
-    ":enableVulnerabilityAlerts",
-    ":semanticCommits"
-  ]
-}

sql.md

@@ -0,0 +1,36 @@
+
+# Most common servers
+
+```SQL
+SELECT COUNT(*), host FROM followers GROUP BY host ORDER BY count DESC;
+```
+
+# Most popular twitter users
+
+```SQL
+SELECT COUNT(*), acct FROM (SELECT unnest(followings) as follow FROM followers) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id GROUP BY acct ORDER BY count DESC;
+```
+
+```SQL
+SELECT COUNT(*), acct, id FROM (SELECT unnest(followings) as follow FROM followers) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id WHERE id IN ( SELECT unnest(followings) FROM followers WHERE host='social.librem.one' AND acct = 'vincent' ) GROUP BY acct, id ORDER BY count DESC;
+```
+
+# Most active users
+
+```SQL
+SELECT array_length(followings, 1) AS l, acct, host FROM followers ORDER BY l DESC;
+```
+
+# Lag
+
+```SQL
+SELECT COUNT(*), date_trunc('day', lastsync) FROM (SELECT unnest(followings) as follow FROM followers GROUP BY follow) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id GROUP BY date_trunc;
+
+SELECT COUNT(*), date_trunc('hour', lastsync) FROM (SELECT unnest(followings) as follow FROM followers GROUP BY follow) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id GROUP BY date_trunc ORDER BY date_trunc;
+```
+
+# Connections 
+
+```SQL
+SELECT SUM(cardinality(followings)) FROM followers;
+```

src/BSLManager/App.cs

@@ -1,254 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Reflection;
-using System.Threading.Tasks;
-using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.DAL.Models;
-using BirdsiteLive.Moderation.Actions;
-using BSLManager.Domain;
-using BSLManager.Tools;
-using Terminal.Gui;
-
-namespace BSLManager
-{
-    public class App
-    {
-        private readonly IFollowersDal _followersDal;
-        private readonly IRemoveFollowerAction _removeFollowerAction;
-
-        private readonly FollowersListState _state = new FollowersListState();
-
-        #region Ctor
-        public App(IFollowersDal followersDal, IRemoveFollowerAction removeFollowerAction)
-        {
-            _followersDal = followersDal;
-            _removeFollowerAction = removeFollowerAction;
-        }
-        #endregion
-
-        public void Run()
-        {
-            Application.UseSystemConsole = true;
-
-            Application.Init();
-            var top = Application.Top;
-
-            // Creates the top-level window to show
-            var win = new Window("BSL Manager")
-            {
-                X = 0,
-                Y = 1, // Leave one row for the toplevel menu
-
-                // By using Dim.Fill(), it will automatically resize without manual intervention
-                Width = Dim.Fill(),
-                Height = Dim.Fill()
-            };
-
-            top.Add(win);
-
-            // Creates a menubar, the item "New" has a help menu.
-            var menu = new MenuBar(new MenuBarItem[]
-            {
-                new MenuBarItem("_File", new MenuItem[]
-                {
-                    new MenuItem("_Quit", "", () =>
-                    {
-                        if (Quit()) top.Running = false;
-                    })
-                }),
-                //new MenuBarItem ("_Edit", new MenuItem [] {
-                //    new MenuItem ("_Copy", "", null),
-                //    new MenuItem ("C_ut", "", null),
-                //    new MenuItem ("_Paste", "", null)
-                //})
-            });
-            top.Add(menu);
-
-            static bool Quit()
-            {
-                var n = MessageBox.Query(50, 7, "Quit BSL Manager", "Are you sure you want to quit?", "Yes", "No");
-                return n == 0;
-            }
-            
-            RetrieveUserList();
-
-            var list = new ListView(_state.GetDisplayableList())
-            {
-                X = 1,
-                Y = 3,
-                Width = Dim.Fill(),
-                Height = Dim.Fill()
-            };
-
-            list.KeyDown += _ =>
-            {
-                if (_.KeyEvent.Key == Key.Enter)
-                {
-                    OpenFollowerDialog(list.SelectedItem);
-                } 
-                else if (_.KeyEvent.Key == Key.Delete
-                           || _.KeyEvent.Key == Key.DeleteChar
-                           || _.KeyEvent.Key == Key.Backspace
-                           || _.KeyEvent.Key == Key.D)
-                {
-                    OpenDeleteDialog(list.SelectedItem);
-                }
-            };
-
-            var listingFollowersLabel = new Label(1, 0, "Listing followers");
-            var filterLabel = new Label("Filter: ") { X = 1, Y = 1 };
-            var filterText = new TextField("")
-            {
-                X = Pos.Right(filterLabel),
-                Y = 1,
-                Width = 40
-            };
-
-            filterText.KeyDown += _ =>
-            {
-                var text = filterText.Text.ToString();
-                if (_.KeyEvent.Key == Key.Enter && !string.IsNullOrWhiteSpace(text))
-                {
-                    _state.FilterBy(text);
-                    ConsoleGui.RefreshUI();
-                }
-            };
-
-            win.Add(
-                listingFollowersLabel,
-                filterLabel,
-                filterText,
-                list
-            );
-
-            Application.Run();
-        }
-
-        private void OpenFollowerDialog(int selectedIndex)
-        {
-            var close = new Button(3, 14, "Close");
-            close.Clicked += () => Application.RequestStop();
-
-            var dialog = new Dialog("Info", 60, 18, close);
-
-            var follower = _state.GetElementAt(selectedIndex);
-
-            var name = new Label($"User: @{follower.Acct}@{follower.Host}")
-            {
-                X = 1,
-                Y = 1,
-                Width = Dim.Fill(),
-                Height = 1
-            };
-            var following = new Label($"Following Count: {follower.Followings.Count}")
-            {
-                X = 1,
-                Y = 3,
-                Width = Dim.Fill(),
-                Height = 1
-            };
-            var errors = new Label($"Posting Errors: {follower.PostingErrorCount}")
-            {
-                X = 1,
-                Y = 4,
-                Width = Dim.Fill(),
-                Height = 1
-            };
-            var inbox = new Label($"Inbox: {follower.InboxRoute}")
-            {
-                X = 1,
-                Y = 5,
-                Width = Dim.Fill(),
-                Height = 1
-            };
-            var sharedInbox = new Label($"Shared Inbox: {follower.SharedInboxRoute}")
-            {
-                X = 1,
-                Y = 6,
-                Width = Dim.Fill(),
-                Height = 1
-            };
-
-            dialog.Add(name);
-            dialog.Add(following);
-            dialog.Add(errors);
-            dialog.Add(inbox);
-            dialog.Add(sharedInbox);
-            dialog.Add(close);
-            Application.Run(dialog);
-        }
-
-        private void OpenDeleteDialog(int selectedIndex)
-        {
-            bool okpressed = false;
-            var ok = new Button(10, 14, "Yes");
-            ok.Clicked += () =>
-            {
-                Application.RequestStop();
-                okpressed = true;
-            };
-
-            var cancel = new Button(3, 14, "No");
-            cancel.Clicked += () => Application.RequestStop();
-
-            var dialog = new Dialog("Delete", 60, 18, cancel, ok);
-
-            var follower = _state.GetElementAt(selectedIndex);
-            var name = new Label($"User: @{follower.Acct}@{follower.Host}")
-            {
-                X = 1,
-                Y = 1,
-                Width = Dim.Fill(),
-                Height = 1
-            };
-            var entry = new Label("Delete user and remove all their followings?")
-            {
-                X = 1,
-                Y = 3,
-                Width = Dim.Fill(),
-                Height = 1
-            };
-            dialog.Add(name);
-            dialog.Add(entry);
-            Application.Run(dialog);
-
-            if (okpressed)
-            {
-                DeleteAndRemoveUser(selectedIndex);
-            }
-        }
-
-        private void DeleteAndRemoveUser(int el)
-        {
-            Application.MainLoop.Invoke(async () =>
-            {
-                try
-                {
-                    var userToDelete = _state.GetElementAt(el);
-                    
-                    BasicLogger.Log($"Delete {userToDelete.Acct}@{userToDelete.Host}");
-                    await _removeFollowerAction.ProcessAsync(userToDelete);
-                    BasicLogger.Log($"Remove user from list");
-                    _state.RemoveAt(el);
-                }
-                catch (Exception e)
-                {
-                    BasicLogger.Log(e.Message);
-                }
-
-                ConsoleGui.RefreshUI();
-            });
-        }
-
-        private void RetrieveUserList()
-        {
-            Application.MainLoop.Invoke(async () =>
-            {
-                var followers = await _followersDal.GetAllFollowersAsync();
-                _state.Load(followers.ToList());
-                ConsoleGui.RefreshUI();
-            });
-        }
-    }
-}

src/BSLManager/BSLManager.csproj

@@ -1,28 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="Lamar" Version="5.0.4" />
-    <PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="5.0.0" />
-    <PackageReference Include="Terminal.Gui" Version="1.12.1" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\BirdsiteLive.Common\BirdsiteLive.Common.csproj" />
-    <ProjectReference Include="..\BirdsiteLive.Moderation\BirdsiteLive.Moderation.csproj" />
-    <ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL.Postgres\BirdsiteLive.DAL.Postgres.csproj" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <None Update="key.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-  </ItemGroup>
-
-</Project>

src/BSLManager/Bootstrapper.cs

@@ -1,94 +0,0 @@
-using System;
-using System.Net.Http;
-using BirdsiteLive.Common.Settings;
-using BirdsiteLive.Common.Structs;
-using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.DAL.Postgres.DataAccessLayers;
-using BirdsiteLive.DAL.Postgres.Settings;
-using Lamar;
-using Lamar.Scanning.Conventions;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
-
-namespace BSLManager
-{
-    public class Bootstrapper
-    {
-        private readonly DbSettings _dbSettings;
-        private readonly InstanceSettings _instanceSettings;
-
-        #region Ctor
-        public Bootstrapper(DbSettings dbSettings, InstanceSettings instanceSettings)
-        {
-            _dbSettings = dbSettings;
-            _instanceSettings = instanceSettings;
-        }
-        #endregion
-
-        public Container Init()
-        {
-            var container = new Container(x =>
-            {
-                x.For<DbSettings>().Use(x => _dbSettings);
-
-                x.For<InstanceSettings>().Use(x => _instanceSettings);
-
-                if (string.Equals(_dbSettings.Type, DbTypes.Postgres, StringComparison.OrdinalIgnoreCase))
-                {
-                    var connString = $"Host={_dbSettings.Host};Username={_dbSettings.User};Password={_dbSettings.Password};Database={_dbSettings.Name}";
-                    var postgresSettings = new PostgresSettings
-                    {
-                        ConnString = connString
-                    };
-                    x.For<PostgresSettings>().Use(x => postgresSettings);
-
-                    x.For<ITwitterUserDal>().Use<TwitterUserPostgresDal>().Singleton();
-                    x.For<IFollowersDal>().Use<FollowersPostgresDal>().Singleton();
-                    x.For<IDbInitializerDal>().Use<DbInitializerPostgresDal>().Singleton();
-                }
-                else
-                {
-                    throw new NotImplementedException($"{_dbSettings.Type} is not supported");
-                }
-
-                var serviceProvider = new ServiceCollection().AddHttpClient().BuildServiceProvider();
-                x.For<IHttpClientFactory>().Use(_ => serviceProvider.GetService<IHttpClientFactory>());
-
-                x.For(typeof(ILogger<>)).Use(typeof(DummyLogger<>));
-
-                x.Scan(_ =>
-                {
-                    _.Assembly("BirdsiteLive.Twitter");
-                    _.Assembly("BirdsiteLive.Domain");
-                    _.Assembly("BirdsiteLive.DAL");
-                    _.Assembly("BirdsiteLive.DAL.Postgres");
-                    _.Assembly("BirdsiteLive.Moderation");
-
-                    _.TheCallingAssembly();
-
-                    _.WithDefaultConventions();
-
-                    _.LookForRegistries();
-                });
-            });
-            return container;
-        }
-
-        public class DummyLogger<T> : ILogger<T>
-        {
-            public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter)
-            {
-            }
-
-            public bool IsEnabled(LogLevel logLevel)
-            {
-                return false;
-            }
-
-            public IDisposable BeginScope<TState>(TState state)
-            {
-                return null;
-            }
-        }
-    }
-}

src/BSLManager/Domain/FollowersListState.cs

@@ -1,81 +0,0 @@
-using System.Collections.Generic;
-using System.Linq;
-using BirdsiteLive.DAL.Models;
-
-namespace BSLManager.Domain
-{
-    public class FollowersListState
-    {
-        private readonly List<string> _filteredDisplayableUserList = new List<string>();
-
-        private List<Follower> _sourceUserList = new List<Follower>();
-        private List<Follower> _filteredSourceUserList = new List<Follower>();
-        
-        public void Load(List<Follower> followers)
-        {
-            _sourceUserList = followers.OrderByDescending(x => x.Followings.Count).ToList();
-           
-            ResetLists();
-        }
-
-        private void ResetLists()
-        {
-            _filteredSourceUserList = _sourceUserList.ToList();
-
-            _filteredDisplayableUserList.Clear();
-
-            foreach (var follower in _sourceUserList)
-            {
-                var displayedUser = $"{GetFullHandle(follower)}   ({follower.Followings.Count})  (err:{follower.PostingErrorCount})";
-                _filteredDisplayableUserList.Add(displayedUser);
-            }
-        }
-
-        public List<string> GetDisplayableList()
-        {
-            return _filteredDisplayableUserList;
-        }
-
-        public void FilterBy(string pattern)
-        {
-            ResetLists();
-
-            if (!string.IsNullOrWhiteSpace(pattern))
-            {
-                var elToRemove = _filteredSourceUserList
-                    .Where(x => !GetFullHandle(x).Contains(pattern))
-                    .Select(x => x)
-                    .ToList();
-
-                foreach (var el in elToRemove)
-                {
-                    _filteredSourceUserList.Remove(el);
-                    
-                    var dElToRemove = _filteredDisplayableUserList.First(x => x.Contains(GetFullHandle(el)));
-                    _filteredDisplayableUserList.Remove(dElToRemove);
-                }
-            }
-        }
-
-        private string GetFullHandle(Follower follower)
-        {
-            return $"@{follower.Acct}@{follower.Host}";
-        }
-
-        public void RemoveAt(int index)
-        {
-            var displayableUser = _filteredDisplayableUserList[index];
-            var sourceUser = _filteredSourceUserList[index];
-
-            _filteredDisplayableUserList.Remove(displayableUser);
-
-            _filteredSourceUserList.Remove(sourceUser);
-            _sourceUserList.Remove(sourceUser);
-        }
-
-        public Follower GetElementAt(int index)
-        {
-            return _filteredSourceUserList[index];
-        }
-    }
-}

src/BSLManager/Program.cs

@@ -1,39 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Reflection;
-using System.Text;
-using System.Threading.Tasks;
-using BirdsiteLive.Common.Settings;
-using BirdsiteLive.DAL.Contracts;
-using BSLManager.Tools;
-using Microsoft.Extensions.Configuration;
-using NStack;
-using Terminal.Gui;
-
-namespace BSLManager
-{
-    class Program
-    {
-        static async Task Main(string[] args)
-        {
-            Console.OutputEncoding = Encoding.Default;
-
-            var settingsManager = new SettingsManager();
-            var settings = settingsManager.GetSettings();
-
-            //var builder = new ConfigurationBuilder()
-            //    .AddEnvironmentVariables();
-            //var configuration = builder.Build();
-
-            //var dbSettings = configuration.GetSection("Db").Get<DbSettings>();
-            //var instanceSettings = configuration.GetSection("Instance").Get<InstanceSettings>();
-
-            var bootstrapper = new Bootstrapper(settings.dbSettings, settings.instanceSettings);
-            var container = bootstrapper.Init();
-
-            var app = container.GetInstance<App>();
-            app.Run();
-        }
-    }
-}

src/BSLManager/Tools/BasicLogger.cs

@@ -1,13 +0,0 @@
-using System;
-using System.IO;
-
-namespace BSLManager.Tools
-{
-    public static class BasicLogger
-    {
-        public static void Log(string log)
-        {
-            File.AppendAllLines($"Log-{Guid.NewGuid()}.txt", new []{ log });
-        }
-    }
-}

src/BSLManager/Tools/ConsoleGui.cs

@@ -1,15 +0,0 @@
-using System.Reflection;
-using Terminal.Gui;
-
-namespace BSLManager.Tools
-{
-    public static class ConsoleGui
-    {
-        public static void RefreshUI()
-        {
-            typeof(Application)
-                .GetMethod("TerminalResized", BindingFlags.Static | BindingFlags.NonPublic)
-                .Invoke(null, null);
-        }
-    }
-}

src/BSLManager/Tools/SettingsManager.cs

@@ -1,124 +0,0 @@
-using System;
-using System.IO;
-using System.Runtime.CompilerServices;
-using BirdsiteLive.Common.Settings;
-using Newtonsoft.Json;
-using Org.BouncyCastle.Asn1.IsisMtt.X509;
-
-namespace BSLManager.Tools
-{
-    public class SettingsManager
-    {
-        private const string LocalFileName = "ManagerSettings.json";
-
-        public (DbSettings dbSettings, InstanceSettings instanceSettings) GetSettings()
-        {
-            var localSettingsData = GetLocalSettingsFile();
-            if (localSettingsData != null) return Convert(localSettingsData);
-
-            Console.WriteLine("We need to set up the manager");
-            Console.WriteLine("Please provide the following information as provided in the docker-compose file");
-
-            LocalSettingsData data;
-            do
-            {
-                data = GetDataFromUser();
-                Console.WriteLine();
-                Console.WriteLine("Please check if all is ok:");
-                Console.WriteLine();
-                Console.WriteLine($"Db Host: {data.DbHost}");
-                Console.WriteLine($"Db Name: {data.DbName}");
-                Console.WriteLine($"Db User: {data.DbUser}");
-                Console.WriteLine($"Db Password: {data.DbPassword}");
-                Console.WriteLine($"Instance Domain: {data.InstanceDomain}");
-                Console.WriteLine();
-
-                string resp;
-                do
-                {
-                    Console.WriteLine("Is it valid? (yes, no)");
-                    resp = Console.ReadLine()?.Trim().ToLowerInvariant();
-
-                    if (resp == "n" || resp == "no") data = null;
-
-                } while (resp != "y" && resp != "yes" && resp != "n" && resp != "no");
-
-            } while (data == null);
-            
-            SaveLocalSettings(data);
-            return Convert(data);
-        }
-
-        private LocalSettingsData GetDataFromUser()
-        {
-            var data = new LocalSettingsData();
-
-            Console.WriteLine("Db Host:");
-            data.DbHost = Console.ReadLine();
-
-            Console.WriteLine("Db Name:");
-            data.DbName = Console.ReadLine();
-
-            Console.WriteLine("Db User:");
-            data.DbUser = Console.ReadLine();
-
-            Console.WriteLine("Db Password:");
-            data.DbPassword = Console.ReadLine();
-
-            Console.WriteLine("Instance Domain:");
-            data.InstanceDomain = Console.ReadLine();
-
-            return data;
-        }
-
-        private (DbSettings dbSettings, InstanceSettings instanceSettings) Convert(LocalSettingsData data)
-        {
-            var dbSettings = new DbSettings
-            {
-                Type = data.DbType,
-                Host = data.DbHost,
-                Name = data.DbName,
-                User = data.DbUser,
-                Password = data.DbPassword
-            };
-            var instancesSettings = new InstanceSettings
-            {
-                Domain = data.InstanceDomain
-            };
-            return (dbSettings, instancesSettings);
-        }
-
-        private LocalSettingsData GetLocalSettingsFile()
-        {
-            try
-            {
-                if (!File.Exists(LocalFileName)) return null;
-
-                var jsonContent = File.ReadAllText(LocalFileName);
-                var content = JsonConvert.DeserializeObject<LocalSettingsData>(jsonContent);
-                return content;
-            }
-            catch (Exception)
-            {
-                return null;
-            }
-        }
-
-        private void SaveLocalSettings(LocalSettingsData data)
-        {
-            var jsonContent = JsonConvert.SerializeObject(data);
-            File.WriteAllText(LocalFileName, jsonContent);
-        }
-    }
-
-    internal class LocalSettingsData
-    {
-        public string DbType { get; set; } = "postgres";
-        public string DbHost { get; set; }
-        public string DbName { get; set; }
-        public string DbUser { get; set; }
-        public string DbPassword { get; set; }
-
-        public string InstanceDomain { get; set; }
-    }
-}

src/BirdsiteLive.ActivityPub/ApDeserializer.cs

@@ -1,6 +1,7 @@
 using System;
 using BirdsiteLive.ActivityPub.Models;
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
+using System.Text.Json;
 
 namespace BirdsiteLive.ActivityPub
 {
@@ -10,22 +11,23 @@ namespace BirdsiteLive.ActivityPub
         {
             try
             {
-                var activity = JsonConvert.DeserializeObject<Activity>(json);
+                Console.WriteLine("DEBUG: JSON");
+                Console.WriteLine(json);
+                var activity = JsonSerializer.Deserialize<Activity>(json);
                 switch (activity.type)
                 {
                     case "Follow":
-                        return JsonConvert.DeserializeObject<ActivityFollow>(json);
+                        return JsonSerializer.Deserialize<ActivityFollow>(json);
                     case "Undo":
-                        var a = JsonConvert.DeserializeObject<ActivityUndo>(json);
+                        var a = JsonSerializer.Deserialize<ActivityUndo>(json);
                         if(a.apObject.type == "Follow")
-                            return JsonConvert.DeserializeObject<ActivityUndoFollow>(json);
+                            return JsonSerializer.Deserialize<ActivityUndoFollow>(json);
                         break;
                     case "Delete":
-                        return JsonConvert.DeserializeObject<ActivityDelete>(json);
+                        return JsonSerializer.Deserialize<ActivityDelete>(json);
                     case "Accept":
-                        var accept = JsonConvert.DeserializeObject<ActivityAccept>(json);
-                        //var acceptType = JsonConvert.DeserializeObject<Activity>(accept.apObject);
-                        switch ((accept.apObject as dynamic).type.ToString())
+                        var accept = JsonSerializer.Deserialize<ActivityAccept>(json);
+                        switch (accept.apObject.type)
                         {
                             case "Follow":
                                 var acceptFollow = new ActivityAcceptFollow()
@@ -36,11 +38,12 @@ namespace BirdsiteLive.ActivityPub
                                     context = accept.context,
                                     apObject = new ActivityFollow()
                                     {
-                                        id = (accept.apObject as dynamic).id?.ToString(),
-                                        type = (accept.apObject as dynamic).type?.ToString(),
-                                        actor = (accept.apObject as dynamic).actor?.ToString(),
-                                        context = (accept.apObject as dynamic).context?.ToString(),
-                                        apObject = (accept.apObject as dynamic).@object?.ToString()
+
+                                        id = accept.apObject.id,
+                                        type = accept.apObject.type,
+                                        actor = accept.apObject.actor,
+                                        context = accept.apObject.context?.ToString(),
+                                        apObject = accept.apObject.apObject,
                                     }
                                 };
                                 return acceptFollow;
@@ -52,14 +55,7 @@ namespace BirdsiteLive.ActivityPub
             {
                 Console.WriteLine(e);
             }
-
             return null;
         }
-
-        private class Ac : Activity
-        {
-            [JsonProperty("object")]
-            public Activity apObject { get; set; }
-        }
     }
 }

src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj

@@ -1,13 +1,11 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="System.Text.Json" Version="4.7.2" />
   </ItemGroup>
 
 </Project>

src/BirdsiteLive.ActivityPub/Converters/ContextArrayConverter.cs

@@ -1,39 +0,0 @@
-using System;
-using System.Collections.Generic;
-using Newtonsoft.Json;
-
-namespace BirdsiteLive.ActivityPub.Converters
-{
-    public class ContextArrayConverter : JsonConverter
-    {
-        public override bool CanWrite { get { return false; } }
-        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
-        {
-            throw new NotImplementedException();
-        }
-
-        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
-        {
-            var result = new List<string>();
-
-            var list = serializer.Deserialize<List<object>>(reader);
-            foreach (var l in list)
-            {
-                if (l is string s)
-                    result.Add(s);
-                else
-                {
-                    var str = JsonConvert.SerializeObject(l);
-                    result.Add(str);
-                }
-            }
-
-            return result.ToArray();
-        }
-
-        public override bool CanConvert(Type objectType)
-        {
-            throw new NotImplementedException();
-        }
-    }
-}

src/BirdsiteLive.ActivityPub/Models/Activity.cs

@@ -1,36 +1,17 @@
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.Text.Json.Serialization;
-using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class Activity
     {
-        [Newtonsoft.Json.JsonIgnore]
-        public static readonly object[] DefaultContext = new object[] {
-            "https://www.w3.org/ns/activitystreams",
-            "https://w3id.org/security/v1",
-            new Dictionary<string, string>
-            {
-                { "Emoji", "toot:Emoji" },
-                { "Hashtag", "as:Hashtag" },
-                { "PropertyValue", "schema:PropertyValue" },
-                { "value", "schema:value" },
-                { "sensitive", "as:sensitive" },
-                { "quoteUrl", "as:quoteUrl" },
-
-                { "schema", "http://schema.org#" },
-                { "toot", "https://joinmastodon.org/ns#" }
-            }
-        };
-
-        [JsonProperty("@context")]
-        public object context { get; set; } = DefaultContext;
+        [JsonPropertyName("@context")]
+        public object context { get; set; }
         public string id { get; set; }
         public string type { get; set; }
         public string actor { get; set; }
 
-        //[JsonProperty("object")]
+        //[JsonPropertyName("object")]
         //public string apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityAccept.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityAccept : Activity
     {
-        [JsonProperty("object")]
-        public object apObject { get; set; }
+        [JsonPropertyName("object")]
+        public NestedActivity apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityAcceptFollow.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityAcceptFollow : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public ActivityFollow apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityAcceptUndoFollow.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityAcceptUndoFollow : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public ActivityUndoFollow apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityCreateNote.cs

@@ -1,6 +1,5 @@
-using System;
-using BirdsiteLive.ActivityPub.Models;
-using Newtonsoft.Json;
+using BirdsiteLive.ActivityPub.Models;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
@@ -10,7 +9,7 @@ namespace BirdsiteLive.ActivityPub
         public string[] to { get; set; }
         public string[] cc { get; set; }
 
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public Note apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs

@@ -1,11 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub.Models
 {
     public class ActivityDelete : Activity
     {
-        public string[] to { get; set; }
-        [JsonProperty("object")]
-        public object apObject { get; set; }
+        [JsonPropertyName("object")]
+        public string apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityFollow.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityFollow : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public string apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityRejectFollow.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityRejectFollow : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public ActivityFollow apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityUndo.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityUndo : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public Activity apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityUndoFollow.cs

@@ -1,10 +1,10 @@
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityUndoFollow : Activity
     {
-        [JsonProperty("object")]
+        [JsonPropertyName("object")]
         public ActivityFollow apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/Actor.cs

@@ -1,18 +1,13 @@
-using System;
-using System.Collections.Generic;
-using System.Net;
+using System.Net;
 using BirdsiteLive.ActivityPub.Converters;
-using BirdsiteLive.ActivityPub.Models;
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class Actor
     {
-        //[JsonPropertyName("@context")]
-        [JsonProperty("@context")]
-        [JsonConverter(typeof(ContextArrayConverter))]
-        public object[] context { get; set; } = Activity.DefaultContext;
+        [JsonPropertyName("@context")]
+        public object[] context { get; set; } = new string[] { "https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1" };
         public string id { get; set; }
         public string type { get; set; }
         public string followers { get; set; }
@@ -20,7 +15,6 @@ namespace BirdsiteLive.ActivityPub
         public string name { get; set; }
         public string summary { get; set; }
         public string url { get; set; }
-        public string movedTo { get; set; }
         public bool manuallyApprovesFollowers { get; set; }
         public string inbox { get; set; }
         public bool? discoverable { get; set; } = true;
@@ -29,6 +23,5 @@ namespace BirdsiteLive.ActivityPub
         public Image image { get; set; }
         public EndPoints endpoints { get; set; }
         public UserAttachment[] attachment { get; set; }
-        public List<Tag> tag;
     }
 }

src/BirdsiteLive.ActivityPub/Models/Followers.cs

@@ -1,13 +1,12 @@
 using BirdsiteLive.ActivityPub.Converters;
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub.Models
 {
     public class Followers
     {
-        [JsonProperty("@context")]
-        [JsonConverter(typeof(ContextArrayConverter))]
-        public object[] context { get; set; } = Activity.DefaultContext;
+        [JsonPropertyName("@context")]
+        public string context { get; set; } = "https://www.w3.org/ns/activitystreams";
 
         public string id { get; set; }
         public string type { get; set; } = "OrderedCollection";

src/BirdsiteLive.ActivityPub/Models/NestedActivity.cs

@@ -0,0 +1,17 @@
+using System.Text.Json.Serialization;
+
+namespace BirdsiteLive.ActivityPub
+{
+    public class NestedActivity
+    {
+        [JsonPropertyName("@context")]
+        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+        public object context { get; set; }
+        public string id { get; set; }
+        public string type { get; set; }
+        public string actor { get; set; }
+
+        [JsonPropertyName("object")]
+        public string apObject { get; set; }
+    }
+}

src/BirdsiteLive.ActivityPub/Models/Note.cs

@@ -1,16 +1,14 @@
-using BirdsiteLive.ActivityPub.Converters;
-using Newtonsoft.Json;
-using System.Collections.Generic;
+using System.Text.Json.Serialization;
 
 namespace BirdsiteLive.ActivityPub.Models
 {
     public class Note
     {
-        [JsonProperty("@context")]
-        [JsonConverter(typeof(ContextArrayConverter))]
-        public object[] context { get; set; } = Activity.DefaultContext;
+        [JsonPropertyName("@context")]
+        public string[] context { get; set; } = new[] { "https://www.w3.org/ns/activitystreams" };
 
         public string id { get; set; }
+        public string announceId { get; set; }
         public string type { get; } = "Note";
         public string summary { get; set; }
         public string inReplyTo { get; set; }
@@ -25,8 +23,8 @@ namespace BirdsiteLive.ActivityPub.Models
         //public Dictionary<string,string> contentMap { get; set; }
         public Attachment[] attachment { get; set; }
         public Tag[] tag { get; set; }
-        //public Dictionary<string, string> replies;
+		//public Dictionary<string, string> replies;
 
-        public string quoteUrl { get; set; }
+		public string quoteUrl { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/Tag.cs

@@ -1,19 +1,8 @@
-using System;
-
-namespace BirdsiteLive.ActivityPub.Models
+namespace BirdsiteLive.ActivityPub.Models
 {
     public class Tag {
-        public TagResource icon { get; set; } = null;
-        public string id { get; set; }
         public string type { get; set; } //Hashtag
         public string href { get; set; } //https://mastodon.social/tags/app
         public string name { get; set; } //#app
-        public DateTime updated { get; set; } = default(DateTime);
-    }
-
-    public class TagResource
-    {
-        public string type { get; set; }
-        public string url { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/WebFingerData.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Text;
 
@@ -18,4 +18,4 @@ namespace BirdsiteLive.ActivityPub.Models
         public string type { get; set; }
         public string template { get; set; }
     }
-}
+}

src/BirdsiteLive.Common/BirdsiteLive.Common.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
 </Project>

src/BirdsiteLive.Common/Regexes/UrlRegexes.cs

@@ -5,7 +5,5 @@ namespace BirdsiteLive.Common.Regexes
     public class UrlRegexes
     {
         public static readonly Regex Url = new Regex(@"(.?)(((http|ftp|https):\/\/)[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])?)");
-
-        public static readonly Regex Domain = new Regex(@"^[a-zA-Z0-9\-_]+(\.[a-zA-Z0-9\-_]+)+$");
     }
 }

src/BirdsiteLive.Common/Regexes/UserRegexes.cs

@@ -5,6 +5,6 @@ namespace BirdsiteLive.Common.Regexes
     public class UserRegexes
     {
         public static readonly Regex TwitterAccount = new Regex(@"^[a-zA-Z0-9_]+$");
-        public static readonly Regex Mention = new Regex(@"(.?)@([a-zA-Z0-9_]+)(\s|$|[\[\]<>,;:!?/|-]|(. ))");
+        public static readonly Regex Mention = new Regex(@"(.?)@([a-zA-Z0-9_]+)(\s|$|[\[\]<>,;:'\.’!?/—\|-]|(. ))");
     }
 }

src/BirdsiteLive.Common/Settings/InstanceSettings.cs

@@ -7,31 +7,21 @@
         public string AdminEmail { get; set; }
         public bool ResolveMentionsInProfiles { get; set; }
         public bool PublishReplies { get; set; }
-        public int MaxUsersCapacity { get; set; }
 
         public string UnlistedTwitterAccounts { get; set; }
-
-        public string TwitterDomain { get; set; }
-
-        public string InfoBanner { get; set; }
-
-        public string TwitterDomainLabel { get; set; }
-
-        public bool ShowAboutInstanceOnProfiles { get; set; }
-
-        public int MaxFollowsPerUser { get; set; }
-
-        public bool DiscloseInstanceRestrictions { get; set; }
-
         public string SensitiveTwitterAccounts { get; set; }
 
         public int FailingTwitterUserCleanUpThreshold { get; set; }
-
-        public int MaxStatusFetchAge { get; set; }
-
-        public bool EnableQuoteRT { get; set; }
         public int FailingFollowerCleanUpThreshold { get; set; } = -1;
 
-        public int UserCacheCapacity { get; set; }
+        public int UserCacheCapacity { get; set; } = 40_000;
+        public int TweetCacheCapacity { get; set; } = 20_000;
+        // "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
+        public string TwitterBearerToken { get; set; } = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA";
+        public int m { get; set; } = 1;
+        public int n_start { get; set; } = 0;
+        public int n_end { get; set; } = 1;
+        public int ParallelTwitterRequests { get; set; } = 10;
+        public int ParallelFediversePosts { get; set; } = 10;
     }
 }

src/BirdsiteLive.Common/Settings/TwitterSettings.cs

@@ -1,8 +0,0 @@
-namespace BirdsiteLive.Common.Settings
-{
-    public class TwitterSettings
-    {
-        public string ConsumerKey { get; set; }
-        public string ConsumerSecret { get; set; }
-    }
-}

src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj

@@ -1,13 +1,11 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Asn1" Version="1.0.9" />
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="Portable.BouncyCastle" Version="1.9.0" />
   </ItemGroup>
 
 </Project>

src/BirdsiteLive.Cryptography/MagicKey.cs

@@ -1,28 +1,12 @@
 using System;
-using System.Collections.Generic;
 using System.Security.Cryptography;
 using System.Text;
-using Newtonsoft.Json;
+using System.Text.Json;
 
 namespace BirdsiteLive.Cryptography
 {
     public class MagicKey
     {
-        //public class WebfingerLink
-        //{
-        //    public string rel { get; set; }
-        //    public string type { get; set; }
-        //    public string href { get; set; }
-        //    public string template { get; set; }
-        //}
-
-        //public class WebfingerResult
-        //{
-        //    public string subject { get; set; }
-        //    public List<string> aliases { get; set; }
-        //    public List<WebfingerLink> links { get; set; }
-        //}
-
         private string[] _parts;
         private RSA _rsa;
 
@@ -38,14 +22,14 @@ namespace BirdsiteLive.Cryptography
 
         private class RSAKeyParms
         {
-            public byte[] D;
-            public byte[] DP;
-            public byte[] DQ;
-            public byte[] Exponent;
-            public byte[] InverseQ;
-            public byte[] Modulus;
-            public byte[] P;
-            public byte[] Q;
+            public byte[] D { get; set; }
+            public byte[] DP {get; set; }
+            public byte[] DQ {get; set; }
+            public byte[] Exponent {get; set; }
+            public byte[] InverseQ {get; set; }
+            public byte[] Modulus {get; set; }
+            public byte[] P {get; set; }
+            public byte[] Q {get; set; }
 
             public static RSAKeyParms From(RSAParameters parms)
             {
@@ -81,7 +65,9 @@ namespace BirdsiteLive.Cryptography
             if (key[0] == '{')
             {
                 _rsa = RSA.Create();
-                _rsa.ImportParameters(JsonConvert.DeserializeObject<RSAKeyParms>(key).Make());
+                Console.WriteLine(key);
+                Console.WriteLine(JsonSerializer.Deserialize<RSAKeyParms>(key).Make());
+                _rsa.ImportParameters(JsonSerializer.Deserialize<RSAKeyParms>(key).Make());
             }
             else
             {
@@ -102,7 +88,7 @@ namespace BirdsiteLive.Cryptography
             var rsa = RSA.Create();
             rsa.KeySize = 2048;
 
-            return new MagicKey(JsonConvert.SerializeObject(RSAKeyParms.From(rsa.ExportParameters(true))));
+            return new MagicKey(JsonSerializer.Serialize<RSAKeyParms>(RSAKeyParms.From(rsa.ExportParameters(true))));
         }
 
         public byte[] BuildSignedData(string data, string dataType, string encoding, string algorithm)
@@ -140,7 +126,7 @@ namespace BirdsiteLive.Cryptography
 
         public string PrivateKey
         {
-            get { return JsonConvert.SerializeObject(RSAKeyParms.From(_rsa.ExportParameters(true))); }
+            get { return JsonSerializer.Serialize(RSAKeyParms.From(_rsa.ExportParameters(true))); }
         }
 
         public string PublicKey

src/BirdsiteLive.Cryptography/RsaGenerator.cs

@@ -1,99 +0,0 @@
-using System;
-using System.IO;
-using System.Security.Cryptography;
-
-namespace BirdsiteLive.Cryptography
-{
-    //https://gist.github.com/ststeiger/f4b29a140b1e3fd618679f89b7f3ff4a
-    //https://gist.github.com/valep27/4a720c25b35fff83fbf872516f847863
-    //https://gist.github.com/therightstuff/aa65356e95f8d0aae888e9f61aa29414
-    //https://stackoverflow.com/questions/52468125/export-rsa-public-key-in-der-format-and-decrypt-data
-    public class RsaGenerator
-    {
-        public string GetRsa()
-        {
-            var rsa = RSA.Create();
-            
-            var outputStream = new StringWriter();
-            var parameters = rsa.ExportParameters(true);
-            using (var stream = new MemoryStream())
-            {
-                var writer = new BinaryWriter(stream);
-                writer.Write((byte)0x30); // SEQUENCE
-                using (var innerStream = new MemoryStream())
-                {
-                    var innerWriter = new BinaryWriter(innerStream);
-                    innerWriter.Write((byte)0x30); // SEQUENCE
-                    EncodeLength(innerWriter, 13);
-                    innerWriter.Write((byte)0x06); // OBJECT IDENTIFIER
-                    var rsaEncryptionOid = new byte[] { 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01 };
-                    EncodeLength(innerWriter, rsaEncryptionOid.Length);
-                    innerWriter.Write(rsaEncryptionOid);
-                    innerWriter.Write((byte)0x05); // NULL
-                    EncodeLength(innerWriter, 0);
-                    innerWriter.Write((byte)0x03); // BIT STRING
-                    using (var bitStringStream = new MemoryStream())
-                    {
-                        var bitStringWriter = new BinaryWriter(bitStringStream);
-                        bitStringWriter.Write((byte)0x00); // # of unused bits
-                        bitStringWriter.Write((byte)0x30); // SEQUENCE
-                        using (var paramsStream = new MemoryStream())
-                        {
-                            var paramsWriter = new BinaryWriter(paramsStream);
-                            //EncodeIntegerBigEndian(paramsWriter, parameters.Modulus); // Modulus
-                            //EncodeIntegerBigEndian(paramsWriter, parameters.Exponent); // Exponent
-                            var paramsLength = (int)paramsStream.Length;
-                            EncodeLength(bitStringWriter, paramsLength);
-                            bitStringWriter.Write(paramsStream.GetBuffer(), 0, paramsLength);
-                        }
-                        var bitStringLength = (int)bitStringStream.Length;
-                        EncodeLength(innerWriter, bitStringLength);
-                        innerWriter.Write(bitStringStream.GetBuffer(), 0, bitStringLength);
-                    }
-                    var length = (int)innerStream.Length;
-                    EncodeLength(writer, length);
-                    writer.Write(innerStream.GetBuffer(), 0, length);
-                }
-
-                var base64 = Convert.ToBase64String(stream.GetBuffer(), 0, (int)stream.Length).ToCharArray();
-                // WriteLine terminates with \r\n, we want only \n
-                outputStream.Write("-----BEGIN PUBLIC KEY-----\n");
-                for (var i = 0; i < base64.Length; i += 64)
-                {
-                    outputStream.Write(base64, i, Math.Min(64, base64.Length - i));
-                    outputStream.Write("\n");
-                }
-                outputStream.Write("-----END PUBLIC KEY-----");
-            }
-
-            return outputStream.ToString();
-        }
-
-        private static void EncodeLength(BinaryWriter stream, int length)
-        {
-            if (length < 0) throw new ArgumentOutOfRangeException("length", "Length must be non-negative");
-            if (length < 0x80)
-            {
-                // Short form
-                stream.Write((byte)length);
-            }
-            else
-            {
-                // Long form
-                var temp = length;
-                var bytesRequired = 0;
-                while (temp > 0)
-                {
-                    temp >>= 8;
-                    bytesRequired++;
-                }
-                stream.Write((byte)(bytesRequired | 0x80));
-                for (var i = bytesRequired - 1; i >= 0; i--)
-                {
-                    stream.Write((byte)(length >> (8 * i) & 0xff));
-                }
-            }
-        }
-
-    }
-}

src/BirdsiteLive.Cryptography/RsaKeys.cs

@@ -1,225 +0,0 @@
-using Org.BouncyCastle.Crypto;
-using Org.BouncyCastle.Crypto.Parameters;
-using Org.BouncyCastle.OpenSsl;
-using Org.BouncyCastle.Security;
-using System;
-using System.IO;
-using System.Security.Cryptography;
-
-namespace MyProject.Data.Encryption
-{
-    public class RSAKeys
-    {
-        /// <summary>
-        /// Import OpenSSH PEM private key string into MS RSACryptoServiceProvider
-        /// </summary>
-        /// <param name="pem"></param>
-        /// <returns></returns>
-        public static RSACryptoServiceProvider ImportPrivateKey(string pem)
-        {
-            PemReader pr = new PemReader(new StringReader(pem));
-            AsymmetricCipherKeyPair KeyPair = (AsymmetricCipherKeyPair)pr.ReadObject();
-            RSAParameters rsaParams = DotNetUtilities.ToRSAParameters((RsaPrivateCrtKeyParameters)KeyPair.Private);
-
-            RSACryptoServiceProvider csp = new RSACryptoServiceProvider();// cspParams);
-            csp.ImportParameters(rsaParams);
-            return csp;
-        }
-
-        /// <summary>
-        /// Import OpenSSH PEM public key string into MS RSACryptoServiceProvider
-        /// </summary>
-        /// <param name="pem"></param>
-        /// <returns></returns>
-        public static RSACryptoServiceProvider ImportPublicKey(string pem)
-        {
-            PemReader pr = new PemReader(new StringReader(pem));
-            AsymmetricKeyParameter publicKey = (AsymmetricKeyParameter)pr.ReadObject();
-            RSAParameters rsaParams = DotNetUtilities.ToRSAParameters((RsaKeyParameters)publicKey);
-
-            RSACryptoServiceProvider csp = new RSACryptoServiceProvider();// cspParams);
-            csp.ImportParameters(rsaParams);
-            return csp;
-        }
-
-        /// <summary>
-        /// Export private (including public) key from MS RSACryptoServiceProvider into OpenSSH PEM string
-        /// slightly modified from https://stackoverflow.com/a/23739932/2860309
-        /// </summary>
-        /// <param name="csp"></param>
-        /// <returns></returns>
-        public static string ExportPrivateKey(RSACryptoServiceProvider csp)
-        {
-            StringWriter outputStream = new StringWriter();
-            if (csp.PublicOnly) throw new ArgumentException("CSP does not contain a private key", "csp");
-            var parameters = csp.ExportParameters(true);
-            using (var stream = new MemoryStream())
-            {
-                var writer = new BinaryWriter(stream);
-                writer.Write((byte)0x30); // SEQUENCE
-                using (var innerStream = new MemoryStream())
-                {
-                    var innerWriter = new BinaryWriter(innerStream);
-                    EncodeIntegerBigEndian(innerWriter, new byte[] { 0x00 }); // Version
-                    EncodeIntegerBigEndian(innerWriter, parameters.Modulus);
-                    EncodeIntegerBigEndian(innerWriter, parameters.Exponent);
-                    EncodeIntegerBigEndian(innerWriter, parameters.D);
-                    EncodeIntegerBigEndian(innerWriter, parameters.P);
-                    EncodeIntegerBigEndian(innerWriter, parameters.Q);
-                    EncodeIntegerBigEndian(innerWriter, parameters.DP);
-                    EncodeIntegerBigEndian(innerWriter, parameters.DQ);
-                    EncodeIntegerBigEndian(innerWriter, parameters.InverseQ);
-                    var length = (int)innerStream.Length;
-                    EncodeLength(writer, length);
-                    writer.Write(innerStream.GetBuffer(), 0, length);
-                }
-
-                var base64 = Convert.ToBase64String(stream.GetBuffer(), 0, (int)stream.Length).ToCharArray();
-                // WriteLine terminates with \r\n, we want only \n
-                outputStream.Write("-----BEGIN RSA PRIVATE KEY-----\n");
-                // Output as Base64 with lines chopped at 64 characters
-                for (var i = 0; i < base64.Length; i += 64)
-                {
-                    outputStream.Write(base64, i, Math.Min(64, base64.Length - i));
-                    outputStream.Write("\n");
-                }
-                outputStream.Write("-----END RSA PRIVATE KEY-----");
-            }
-
-            return outputStream.ToString();
-        }
-
-        /// <summary>
-        /// Export public key from MS RSACryptoServiceProvider into OpenSSH PEM string
-        /// slightly modified from https://stackoverflow.com/a/28407693
-        /// </summary>
-        /// <param name="csp"></param>
-        /// <returns></returns>
-        public static string ExportPublicKey(RSACryptoServiceProvider csp)
-        {
-            StringWriter outputStream = new StringWriter();
-            var parameters = csp.ExportParameters(false);
-            using (var stream = new MemoryStream())
-            {
-                var writer = new BinaryWriter(stream);
-                writer.Write((byte)0x30); // SEQUENCE
-                using (var innerStream = new MemoryStream())
-                {
-                    var innerWriter = new BinaryWriter(innerStream);
-                    innerWriter.Write((byte)0x30); // SEQUENCE
-                    EncodeLength(innerWriter, 13);
-                    innerWriter.Write((byte)0x06); // OBJECT IDENTIFIER
-                    var rsaEncryptionOid = new byte[] { 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01 };
-                    EncodeLength(innerWriter, rsaEncryptionOid.Length);
-                    innerWriter.Write(rsaEncryptionOid);
-                    innerWriter.Write((byte)0x05); // NULL
-                    EncodeLength(innerWriter, 0);
-                    innerWriter.Write((byte)0x03); // BIT STRING
-                    using (var bitStringStream = new MemoryStream())
-                    {
-                        var bitStringWriter = new BinaryWriter(bitStringStream);
-                        bitStringWriter.Write((byte)0x00); // # of unused bits
-                        bitStringWriter.Write((byte)0x30); // SEQUENCE
-                        using (var paramsStream = new MemoryStream())
-                        {
-                            var paramsWriter = new BinaryWriter(paramsStream);
-                            EncodeIntegerBigEndian(paramsWriter, parameters.Modulus); // Modulus
-                            EncodeIntegerBigEndian(paramsWriter, parameters.Exponent); // Exponent
-                            var paramsLength = (int)paramsStream.Length;
-                            EncodeLength(bitStringWriter, paramsLength);
-                            bitStringWriter.Write(paramsStream.GetBuffer(), 0, paramsLength);
-                        }
-                        var bitStringLength = (int)bitStringStream.Length;
-                        EncodeLength(innerWriter, bitStringLength);
-                        innerWriter.Write(bitStringStream.GetBuffer(), 0, bitStringLength);
-                    }
-                    var length = (int)innerStream.Length;
-                    EncodeLength(writer, length);
-                    writer.Write(innerStream.GetBuffer(), 0, length);
-                }
-
-                var base64 = Convert.ToBase64String(stream.GetBuffer(), 0, (int)stream.Length).ToCharArray();
-                // WriteLine terminates with \r\n, we want only \n
-                outputStream.Write("-----BEGIN PUBLIC KEY-----\n");
-                for (var i = 0; i < base64.Length; i += 64)
-                {
-                    outputStream.Write(base64, i, Math.Min(64, base64.Length - i));
-                    outputStream.Write("\n");
-                }
-                outputStream.Write("-----END PUBLIC KEY-----");
-            }
-
-            return outputStream.ToString();
-        }
-
-        /// <summary>
-        /// https://stackoverflow.com/a/23739932/2860309
-        /// </summary>
-        /// <param name="stream"></param>
-        /// <param name="length"></param>
-        private static void EncodeLength(BinaryWriter stream, int length)
-        {
-            if (length < 0) throw new ArgumentOutOfRangeException("length", "Length must be non-negative");
-            if (length < 0x80)
-            {
-                // Short form
-                stream.Write((byte)length);
-            }
-            else
-            {
-                // Long form
-                var temp = length;
-                var bytesRequired = 0;
-                while (temp > 0)
-                {
-                    temp >>= 8;
-                    bytesRequired++;
-                }
-                stream.Write((byte)(bytesRequired | 0x80));
-                for (var i = bytesRequired - 1; i >= 0; i--)
-                {
-                    stream.Write((byte)(length >> (8 * i) & 0xff));
-                }
-            }
-        }
-
-        /// <summary>
-        /// https://stackoverflow.com/a/23739932/2860309
-        /// </summary>
-        /// <param name="stream"></param>
-        /// <param name="value"></param>
-        /// <param name="forceUnsigned"></param>
-        private static void EncodeIntegerBigEndian(BinaryWriter stream, byte[] value, bool forceUnsigned = true)
-        {
-            stream.Write((byte)0x02); // INTEGER
-            var prefixZeros = 0;
-            for (var i = 0; i < value.Length; i++)
-            {
-                if (value[i] != 0) break;
-                prefixZeros++;
-            }
-            if (value.Length - prefixZeros == 0)
-            {
-                EncodeLength(stream, 1);
-                stream.Write((byte)0);
-            }
-            else
-            {
-                if (forceUnsigned && value[prefixZeros] > 0x7f)
-                {
-                    // Add a prefix zero to force unsigned if the MSB is 1
-                    EncodeLength(stream, value.Length - prefixZeros + 1);
-                    stream.Write((byte)0);
-                }
-                else
-                {
-                    EncodeLength(stream, value.Length - prefixZeros);
-                }
-                for (var i = prefixZeros; i < value.Length; i++)
-                {
-                    stream.Write(value[i]);
-                }
-            }
-        }
-    }
-}

src/BirdsiteLive.Domain/ActivityPubService.cs

@@ -5,31 +5,26 @@ using System.Net.Http;
 using System.Runtime.InteropServices;
 using System.Security.Cryptography;
 using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
 using System.Threading.Tasks;
 using BirdsiteLive.ActivityPub;
 using BirdsiteLive.ActivityPub.Converters;
 using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.Common.Settings;
 using Microsoft.Extensions.Logging;
-using Newtonsoft.Json;
 
 namespace BirdsiteLive.Domain
 {
     public interface IActivityPubService
     {
-        Task<string> GetUserIdAsync(string acct);
         Task<Actor> GetUser(string objectId);
         Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null);
-        Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost,
+        Task PostNewActivity(ActivityCreateNote note, string username, string noteId, string targetHost,
             string targetInbox);
-        Task DeleteUserAsync(string username, string targetHost, string targetInbox);
-		Task<WebFingerData> WebFinger(string account);
-    }
 
-    public class WebFinger
-    {
-        public string subject { get; set; }
-        public string[] aliases { get; set; }
+        ActivityAcceptFollow BuildAcceptFollow(ActivityFollow activity);
+        Task<WebFingerData> WebFinger(string account);
     }
 
     public class ActivityPubService : IActivityPubService
@@ -49,24 +44,6 @@ namespace BirdsiteLive.Domain
         }
         #endregion
 
-        public async Task<string> GetUserIdAsync(string acct)
-        {
-            var splittedAcct = acct.Trim('@').Split('@');
-
-            var url = $"https://{splittedAcct[1]}/.well-known/webfinger?resource=acct:{splittedAcct[0]}@{splittedAcct[1]}";
-
-            var httpClient = _httpClientFactory.CreateClient("BirdsiteLIVE");
-            httpClient.DefaultRequestHeaders.Add("Accept", "application/json");
-            var result = await httpClient.GetAsync(url);
-
-            result.EnsureSuccessStatusCode();
-
-            var content = await result.Content.ReadAsStringAsync();
-
-            var actor = JsonConvert.DeserializeObject<WebFinger>(content);
-            return actor.aliases.FirstOrDefault();
-        }
-
         public async Task<Actor> GetUser(string objectId)
         {
             var httpClient = _httpClientFactory.CreateClient("BirdsiteLIVE");
@@ -80,59 +57,17 @@ namespace BirdsiteLive.Domain
 
             var content = await result.Content.ReadAsStringAsync();
 
-            var actor = JsonConvert.DeserializeObject<Actor>(content);
+            var actor = JsonSerializer.Deserialize<Actor>(content);
             if (string.IsNullOrWhiteSpace(actor.url)) actor.url = objectId;
             return actor;
         }
 
-        public async Task DeleteUserAsync(string username, string targetHost, string targetInbox)
+        public async Task PostNewActivity(ActivityCreateNote noteActivity, string username, string noteId, string targetHost, string targetInbox)
         {
             try
             {
                 var actor = UrlFactory.GetActorUrl(_instanceSettings.Domain, username);
 
-                var deleteUser = new ActivityDelete
-                {
-                    context = "https://www.w3.org/ns/activitystreams",
-                    id = $"{actor}#delete",
-                    type = "Delete",
-                    actor = actor,
-                    to = new [] { "https://www.w3.org/ns/activitystreams#Public" },
-                    apObject = actor
-                };
-
-                await PostDataAsync(deleteUser, targetHost, actor, targetInbox);
-            }
-            catch (Exception e)
-            {
-                _logger.LogError(e, "Error deleting {Username} to {Host}{Inbox}", username, targetHost, targetInbox);
-                throw;
-            }
-        }
-
-        public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox)
-        {
-            try
-            {
-                var actor = UrlFactory.GetActorUrl(_instanceSettings.Domain, username);
-                var noteUri = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, noteId);
-
-                var now = DateTime.UtcNow;
-                var nowString = now.ToString("s") + "Z";
-
-                var noteActivity = new ActivityCreateNote()
-                {
-                    context = "https://www.w3.org/ns/activitystreams",
-                    id = $"{noteUri}/activity",
-                    type = "Create",
-                    actor = actor,
-                    published = nowString,
-
-                    to = note.to,
-                    cc = note.cc,
-                    apObject = note
-                };
-
                 await PostDataAsync(noteActivity, targetHost, actor, targetInbox);
             }
             catch (Exception e)
@@ -142,13 +77,32 @@ namespace BirdsiteLive.Domain
             }
         }
 
-        public async Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null)
+        public ActivityAcceptFollow BuildAcceptFollow(ActivityFollow activity)
+        {
+            var acceptFollow = new ActivityAcceptFollow()
+            {
+                context = "https://www.w3.org/ns/activitystreams",
+                id = $"{activity.apObject}#accepts/follows/{Guid.NewGuid()}",
+                type = "Accept",
+                actor = activity.apObject,
+                apObject = new ActivityFollow()
+                {
+                    id = activity.id,
+                    type = activity.type,
+                    actor = activity.actor,
+                    apObject = activity.apObject
+                }
+            };
+            return acceptFollow;
+        }
+        public HttpRequestMessage BuildRequest<T>(T data, string targetHost, string actorUrl,
+            string inbox = null)
         {
             var usedInbox = $"/inbox";
             if (!string.IsNullOrWhiteSpace(inbox))
                 usedInbox = inbox;
 
-            var json = JsonConvert.SerializeObject(data);
+            var json = JsonSerializer.Serialize(data);
 
             var date = DateTime.UtcNow.ToUniversalTime();
             var httpDate = date.ToString("r");
@@ -157,33 +111,43 @@ namespace BirdsiteLive.Domain
 
             var signature = _cryptoService.SignAndGetSignatureHeader(date, actorUrl, targetHost, digest, usedInbox);
 
-            var client = _httpClientFactory.CreateClient("BirdsiteLIVE");
             var httpRequestMessage = new HttpRequestMessage
             {
                 Method = HttpMethod.Post,
                 RequestUri = new Uri($"https://{targetHost}{usedInbox}"),
                 Headers =
                 {
-                    {"Host", targetHost},
-                    {"Date", httpDate},
-                    {"Signature", signature},
-                    {"Digest", $"SHA-256={digest}"}
+                    { "Host", targetHost },
+                    { "Date", httpDate },
+                    { "Signature", signature },
+                    { "Digest", $"SHA-256={digest}" }
                 },
                 Content = new StringContent(json, Encoding.UTF8, "application/ld+json")
             };
 
+            return httpRequestMessage;
+        }
+
+        public async Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null)
+        {
+            var httpRequestMessage = BuildRequest(data, targetHost, actorUrl, inbox);
+
+            var client = _httpClientFactory.CreateClient("BirdsiteLIVE");
+            client.Timeout = TimeSpan.FromSeconds(2);
+
             var response = await client.SendAsync(httpRequestMessage);
             response.EnsureSuccessStatusCode();
+
             return response.StatusCode;
         }
 
         public async Task<WebFingerData> WebFinger(string account)
         {
-            var httpClient = _httpClientFactory.CreateClient("BirdsiteLIVE");
+            var httpClient = _httpClientFactory.CreateClient();
             var result = await httpClient.GetAsync("https://" + account.Split('@')[1] + "/.well-known/webfinger?resource=acct:" + account);
             var content = await result.Content.ReadAsStringAsync();
 
-            return JsonConvert.DeserializeObject<WebFingerData>(content);
+            return JsonSerializer.Deserialize<WebFingerData>(content);
         }
     }
 }

src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
@@ -15,8 +15,4 @@
     <ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
   </ItemGroup>
 
-  <ItemGroup>
-    <Folder Include="Enum\" />
-  </ItemGroup>
-
 </Project>

src/BirdsiteLive.Domain/BusinessUseCases/ProcessFollowUser.cs

@@ -42,9 +42,6 @@ namespace BirdsiteLive.Domain.BusinessUseCases
             var twitterUserId = twitterUser.Id;
             if(!follower.Followings.Contains(twitterUserId))
                 follower.Followings.Add(twitterUserId);
-
-            if(!follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
-                follower.FollowingsSyncStatus.Add(twitterUserId, -1);
             
             // Save Follower
             await _followerDal.UpdateFollowerAsync(follower);

src/BirdsiteLive.Domain/BusinessUseCases/ProcessUnfollowUser.cs

@@ -36,9 +36,6 @@ namespace BirdsiteLive.Domain.BusinessUseCases
             if (follower.Followings.Contains(twitterUserId))
                 follower.Followings.Remove(twitterUserId);
 
-            if (follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
-                follower.FollowingsSyncStatus.Remove(twitterUserId);
-
             // Save or delete Follower
             if (follower.Followings.Any())
                 await _followerDal.UpdateFollowerAsync(follower);

src/BirdsiteLive.Domain/Enum/MigrationTypeEnum.cs

@@ -1,9 +0,0 @@
-namespace BirdsiteLive.Domain.Enum
-{
-    public enum MigrationTypeEnum
-    {
-        Unknown = 0,
-        Migration = 1,
-        Deletion = 2
-    }
-}

src/BirdsiteLive.Domain/MigrationService.cs

@@ -1,352 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using BirdsiteLive.Twitter;
-using System.Security.Cryptography;
-using System.Text;
-using System.Threading.Tasks;
-using BirdsiteLive.ActivityPub;
-using BirdsiteLive.ActivityPub.Models;
-using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.ActivityPub.Converters;
-using BirdsiteLive.Common.Settings;
-using BirdsiteLive.DAL.Models;
-using BirdsiteLive.Domain.Enum;
-using System.Net.Http;
-using BirdsiteLive.Common.Regexes;
-using Microsoft.Extensions.Logging;
-
-namespace BirdsiteLive.Domain
-{
-    public class MigrationService
-    {
-        private readonly InstanceSettings _instanceSettings;
-        private readonly ITheFedInfoService _theFedInfoService;
-        private readonly ITwitterTweetsService _twitterTweetsService;
-        private readonly IActivityPubService _activityPubService;
-        private readonly ITwitterUserDal _twitterUserDal;
-        private readonly IFollowersDal _followersDal;
-        private readonly IHttpClientFactory _httpClientFactory;
-        private readonly ILogger<MigrationService> _logger;
-
-        #region Ctor
-        public MigrationService(ITwitterTweetsService twitterTweetsService, IActivityPubService activityPubService, ITwitterUserDal twitterUserDal, IFollowersDal followersDal, InstanceSettings instanceSettings, ITheFedInfoService theFedInfoService, IHttpClientFactory httpClientFactory, ILogger<MigrationService> logger)
-        {
-            _twitterTweetsService = twitterTweetsService;
-            _activityPubService = activityPubService;
-            _twitterUserDal = twitterUserDal;
-            _followersDal = followersDal;
-            _instanceSettings = instanceSettings;
-            _theFedInfoService = theFedInfoService;
-            _httpClientFactory = httpClientFactory;
-            _logger = logger;
-        }
-        #endregion
-
-        public string GetMigrationCode(string acct)
-        {
-            var hash = GetHashString(acct);
-            return $"[[BirdsiteLIVE-MigrationCode|{hash.Substring(0, 10)}]]";
-        }
-
-        public string GetDeletionCode(string acct)
-        {
-            var hash = GetHashString(acct);
-            return $"[[BirdsiteLIVE-DeletionCode|{hash.Substring(0, 10)}]]";
-        }
-
-        public bool ValidateTweet(string acct, string tweetId, MigrationTypeEnum type)
-        {
-            string code;
-            if (type == MigrationTypeEnum.Migration)
-                code = GetMigrationCode(acct);
-            else if (type == MigrationTypeEnum.Deletion)
-                code = GetDeletionCode(acct);
-            else
-                throw new NotImplementedException();
-
-            var castedTweetId = ExtractedTweetId(tweetId);
-            var tweet = _twitterTweetsService.GetTweet(castedTweetId);
-
-            if (tweet == null)
-                throw new Exception("Tweet not found");
-
-            if (tweet.CreatorName.Trim().ToLowerInvariant() != acct.Trim().ToLowerInvariant())
-                throw new Exception($"Tweet not published by @{acct}");
-
-            if (!tweet.MessageContent.Contains(code))
-            {
-                var message = "Tweet don't have migration code";
-                if (type == MigrationTypeEnum.Deletion)
-                    message = "Tweet don't have deletion code";
-
-                throw new Exception(message);
-            }
-
-            return true;
-        }
-
-        private long ExtractedTweetId(string tweetId)
-        {
-            if (string.IsNullOrWhiteSpace(tweetId))
-                throw new ArgumentException("No provided Tweet ID");
-
-            long castedId;
-            if (long.TryParse(tweetId, out castedId))
-                return castedId;
-
-            var urlPart = tweetId.Split('/').LastOrDefault();
-            if (long.TryParse(urlPart, out castedId))
-                return castedId;
-
-            throw new ArgumentException("Unvalid Tweet ID");
-        }
-
-        public async Task<ValidatedFediverseUser> ValidateFediverseAcctAsync(string fediverseAcct)
-        {
-            if (string.IsNullOrWhiteSpace(fediverseAcct))
-                throw new ArgumentException("Please provide Fediverse account");
-
-            if (!fediverseAcct.Contains('@') || !fediverseAcct.StartsWith("@") || fediverseAcct.Trim('@').Split('@').Length != 2)
-                throw new ArgumentException("Please provide valid Fediverse handle");
-
-            var objectId = await _activityPubService.GetUserIdAsync(fediverseAcct);
-            var user = await _activityPubService.GetUser(objectId);
-
-            var result = new ValidatedFediverseUser
-            {
-                FediverseAcct = fediverseAcct,
-                ObjectId = objectId,
-                User = user,
-                IsValid = user != null
-            };
-
-            return result;
-        }
-
-        public async Task MigrateAccountAsync(ValidatedFediverseUser validatedUser, string acct)
-        {
-            // Apply moved to
-            var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct);
-            if (twitterAccount == null)
-            {
-                await _twitterUserDal.CreateTwitterUserAsync(acct, -1, validatedUser.ObjectId, validatedUser.FediverseAcct);
-                twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct);
-            }
-
-            twitterAccount.MovedTo = validatedUser.User.id;
-            twitterAccount.MovedToAcct = validatedUser.FediverseAcct;
-            twitterAccount.LastSync = DateTime.UtcNow;
-            await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount);
-
-            // Notify Followers
-            var message = $@"<p>[BSL MIRROR SERVICE NOTIFICATION]<br/>This bot has been disabled by its original owner.<br/>It has been redirected to {validatedUser.FediverseAcct}.</p>";
-            NotifyFollowers(acct, twitterAccount, message);
-        }
-
-        private void NotifyFollowers(string acct, SyncTwitterUser twitterAccount, string message)
-        {
-            var t = Task.Run(async () =>
-            {
-                var followers = await _followersDal.GetFollowersAsync(twitterAccount.Id);
-                foreach (var follower in followers)
-                {
-                    try
-                    {
-                        var noteId = Guid.NewGuid().ToString();
-                        var actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, acct);
-                        var noteUrl = UrlFactory.GetNoteUrl(_instanceSettings.Domain, acct, noteId);
-
-                        //var to = validatedUser.ObjectId;
-                        var to = follower.ActorId;
-                        var cc = new string[0];
-
-                        var note = new Note
-                        {
-                            id = noteUrl,
-
-                            published = DateTime.UtcNow.ToString("s") + "Z",
-                            url = noteUrl,
-                            attributedTo = actorUrl,
-
-                            to = new[] { to },
-                            cc = cc,
-
-                            content = message,
-                            tag = new Tag[]{
-                                new Tag()
-                                {
-                                    type = "Mention",
-                                    href = follower.ActorId,
-                                    name = $"@{follower.Acct}@{follower.Host}"
-                                }
-                            },
-                        };
-
-                        if (!string.IsNullOrWhiteSpace(follower.SharedInboxRoute))
-                            await _activityPubService.PostNewNoteActivity(note, acct, noteId, follower.Host, follower.SharedInboxRoute);
-                        else
-                            await _activityPubService.PostNewNoteActivity(note, acct, noteId, follower.Host, follower.InboxRoute);
-                    }
-                    catch (Exception e)
-                    {
-                        _logger.LogError(e, e.Message);
-                    }
-                }
-            });
-        }
-
-        public async Task DeleteAccountAsync(string acct)
-        {
-            // Apply deleted state
-            var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct);
-            if (twitterAccount == null)
-            {
-                await _twitterUserDal.CreateTwitterUserAsync(acct, -1);
-                twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct);
-            }
-
-            twitterAccount.Deleted = true;
-            twitterAccount.LastSync = DateTime.UtcNow;
-            await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount);
-
-            // Notify Followers
-            var message = $@"<p>[BSL MIRROR SERVICE NOTIFICATION]<br/>This bot has been deleted by its original owner.<br/></p>";
-            NotifyFollowers(acct, twitterAccount, message);
-
-            // Delete remote accounts
-            DeleteRemoteAccounts(acct);
-        }
-
-        private void DeleteRemoteAccounts(string acct)
-        {
-            var t = Task.Run(async () =>
-            {
-                var allUsers = await _followersDal.GetAllFollowersAsync();
-
-                var followersWtSharedInbox = allUsers
-                    .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
-                    .GroupBy(x => x.Host)
-                    .ToList();
-                foreach (var followerGroup in followersWtSharedInbox)
-                {
-                    var host = followerGroup.First().Host;
-                    var sharedInbox = followerGroup.First().SharedInboxRoute;
-
-                    var t1 = Task.Run(async () =>
-                    {
-                        try
-                        {
-                            await _activityPubService.DeleteUserAsync(acct, host, sharedInbox);
-                        }
-                        catch (Exception e)
-                        {
-                            _logger.LogError(e, e.Message);
-                        }
-                    });
-                }
-
-                var followerWtInbox = allUsers
-                    .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
-                    .ToList();
-                foreach (var followerGroup in followerWtInbox)
-                {
-                    var host = followerGroup.Host;
-                    var sharedInbox = followerGroup.InboxRoute;
-
-                    var t1 = Task.Run(async () =>
-                    {
-                        try
-                        {
-                            await _activityPubService.DeleteUserAsync(acct, host, sharedInbox);
-                        }
-                        catch (Exception e)
-                        {
-                            _logger.LogError(e, e.Message);
-                        }
-                    });
-                }
-            });
-        }
-
-        public async Task TriggerRemoteMigrationAsync(string id, string tweetIdStg, string handle)
-        {
-            var url = $"https://{{0}}/migration/move/{{1}}/{{2}}/{handle}";
-            await ProcessRemoteMigrationAsync(id, tweetIdStg, url);
-
-        }
-
-        public async Task TriggerRemoteDeleteAsync(string id, string tweetIdStg)
-        {
-            var url = $"https://{{0}}/migration/delete/{{1}}/{{2}}";
-            await ProcessRemoteMigrationAsync(id, tweetIdStg, url);
-        }
-
-        private async Task ProcessRemoteMigrationAsync(string id, string tweetIdStg, string urlPattern)
-        {
-            try
-            {
-                var instances = await RetrieveCompatibleBslInstancesAsync();
-                var tweetId = ExtractedTweetId(tweetIdStg);
-
-                foreach (var instance in instances)
-                {
-                    try
-                    {
-                        var host = instance.Host;
-                        if(!UrlRegexes.Domain.IsMatch(host)) continue;
-
-                        var url = string.Format(urlPattern, host, id, tweetId);
-
-                        var client = _httpClientFactory.CreateClient("BirdsiteLIVE");
-                        var result = await client.PostAsync(url, null);
-                        result.EnsureSuccessStatusCode();
-                    }
-                    catch (Exception e)
-                    {
-                        _logger.LogError(e, e.Message);
-                    }
-                }
-            }
-            catch (Exception e)
-            {
-                _logger.LogError(e, e.Message);
-            }
-        }
-
-        private async Task<List<BslInstanceInfo>> RetrieveCompatibleBslInstancesAsync()
-        {
-            var instances = await _theFedInfoService.GetBslInstanceListAsync();
-            var filteredInstances = instances
-                .Where(x => x.Version >= new Version(0, 21, 0))
-                .Where(x => string.Compare(x.Host, 
-                    _instanceSettings.Domain, 
-                    StringComparison.InvariantCultureIgnoreCase) != 0)
-                .ToList();
-            return filteredInstances;
-        }
-
-        private byte[] GetHash(string inputString)
-        {
-            using (HashAlgorithm algorithm = SHA256.Create())
-                return algorithm.ComputeHash(Encoding.UTF8.GetBytes(inputString));
-        }
-
-        private string GetHashString(string inputString)
-        {
-            StringBuilder sb = new StringBuilder();
-            foreach (byte b in GetHash(inputString))
-                sb.Append(b.ToString("X2"));
-
-            return sb.ToString();
-        }
-    }
-
-    public class ValidatedFediverseUser
-    {
-        public string FediverseAcct { get; set; }
-        public string ObjectId { get; set; }
-        public Actor User { get; set; }
-        public bool IsValid { get; set; }
-    }
-}

src/BirdsiteLive.Domain/Repository/ModerationRepository.cs

@@ -11,12 +11,6 @@ namespace BirdsiteLive.Domain.Repository
     {
         ModerationTypeEnum GetModerationType(ModerationEntityTypeEnum type);
         ModeratedTypeEnum CheckStatus(ModerationEntityTypeEnum type, string entity);
-
-        IEnumerable<string> GetWhitelistedFollowers();
-        IEnumerable<string> GetBlacklistedFollowers();
-        IEnumerable<string> GetWhitelistedAccounts();
-
-        IEnumerable<string> GetBlacklistedAccounts();
     }
 
     public class ModerationRepository : IModerationRepository
@@ -29,13 +23,9 @@ namespace BirdsiteLive.Domain.Repository
         private readonly Dictionary<ModerationEntityTypeEnum, ModerationTypeEnum> _modMode =
             new Dictionary<ModerationEntityTypeEnum, ModerationTypeEnum>();
 
-        private readonly ModerationSettings _settings;
-
         #region Ctor
         public ModerationRepository(ModerationSettings settings)
         {
-            _settings = settings;
-
             var parsedFollowersWhiteListing = PatternsParser.Parse(settings.FollowersWhiteListing);
             var parsedFollowersBlackListing = PatternsParser.Parse(settings.FollowersBlackListing);
             var parsedTwitterAccountsWhiteListing = PatternsParser.Parse(settings.TwitterAccountsWhiteListing);
@@ -133,35 +123,6 @@ namespace BirdsiteLive.Domain.Repository
                     throw new ArgumentOutOfRangeException();
             }
         }
-
-        private char GetSplitChar(string entry)
-        {
-            var separationChar = '|';
-            if (entry.Contains(";")) separationChar = ';';
-            else if (entry.Contains(",")) separationChar = ',';
-
-            return separationChar;
-        }
-
-        public IEnumerable<string> GetWhitelistedFollowers()
-        {
-            return _settings.FollowersWhiteListing.Split(GetSplitChar(_settings.FollowersWhiteListing));
-        }
-
-        public IEnumerable<string> GetBlacklistedFollowers()
-        {
-            return _settings.FollowersBlackListing.Split(GetSplitChar(_settings.FollowersBlackListing));
-        }
-
-        public IEnumerable<string> GetWhitelistedAccounts()
-        {
-            return _settings.TwitterAccountsWhiteListing.Split(GetSplitChar(_settings.TwitterAccountsWhiteListing));
-        }
-
-        public IEnumerable<string> GetBlacklistedAccounts()
-        {
-            return _settings.TwitterAccountsBlackListing.Split(GetSplitChar(_settings.TwitterAccountsBlackListing));
-        }
     }
 
     public enum ModerationEntityTypeEnum

src/BirdsiteLive.Domain/Repository/PublicationRepository.cs

@@ -1,40 +0,0 @@
-using System.Linq;
-using BirdsiteLive.Common.Settings;
-using BirdsiteLive.Domain.Tools;
-
-namespace BirdsiteLive.Domain.Repository
-{
-    public interface IPublicationRepository
-    {
-        bool IsUnlisted(string twitterAcct);
-        bool IsSensitive(string twitterAcct);
-    }
-
-    public class PublicationRepository : IPublicationRepository
-    {
-        private readonly string[] _unlistedAccounts;
-        private readonly string[] _sensitiveAccounts;
-
-        #region Ctor
-        public PublicationRepository(InstanceSettings settings)
-        {
-            _unlistedAccounts = PatternsParser.Parse(settings.UnlistedTwitterAccounts);
-            _sensitiveAccounts = PatternsParser.Parse(settings.SensitiveTwitterAccounts);
-        }
-        #endregion
-
-        public bool IsUnlisted(string twitterAcct)
-        {
-            if (_unlistedAccounts == null || !_unlistedAccounts.Any()) return false;
-
-            return _unlistedAccounts.Contains(twitterAcct.ToLowerInvariant());
-        }
-
-        public bool IsSensitive(string twitterAcct)
-        {
-            if (_sensitiveAccounts == null || !_sensitiveAccounts.Any()) return false;
-
-            return _sensitiveAccounts.Contains(twitterAcct.ToLowerInvariant());
-        }
-    }
-}

src/BirdsiteLive.Domain/StatusService.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Text.RegularExpressions;
+using System.Threading.Tasks;
 using BirdsiteLive.ActivityPub;
 using BirdsiteLive.ActivityPub.Converters;
 using BirdsiteLive.ActivityPub.Models;
@@ -11,14 +12,13 @@ using BirdsiteLive.Domain.Repository;
 using BirdsiteLive.Domain.Statistics;
 using BirdsiteLive.Domain.Tools;
 using BirdsiteLive.Twitter.Models;
-using Tweetinvi.Models;
-using Tweetinvi.Models.Entities;
 
 namespace BirdsiteLive.Domain
 {
     public interface IStatusService
     {
         Note GetStatus(string username, ExtractedTweet tweet);
+        ActivityCreateNote GetActivity(string username, ExtractedTweet tweet);
     }
 
     public class StatusService : IStatusService
@@ -26,15 +26,13 @@ namespace BirdsiteLive.Domain
         private readonly InstanceSettings _instanceSettings;
         private readonly IStatusExtractor _statusExtractor;
         private readonly IExtractionStatisticsHandler _statisticsHandler;
-        private readonly IPublicationRepository _publicationRepository;
 
         #region Ctor
-        public StatusService(InstanceSettings instanceSettings, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler, IPublicationRepository publicationRepository)
+        public StatusService(InstanceSettings instanceSettings, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler)
         {
             _instanceSettings = instanceSettings;
             _statusExtractor = statusExtractor;
             _statisticsHandler = statisticsHandler;
-            _publicationRepository = publicationRepository;
         }
         #endregion
 
@@ -42,43 +40,43 @@ namespace BirdsiteLive.Domain
         {
             var actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, username);
             var noteUrl = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, tweet.Id.ToString());
+            String announceId = null;
+            if (tweet.IsRetweet)
+            {
+                actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, tweet.OriginalAuthor.Acct);
+                noteUrl = UrlFactory.GetNoteUrl(_instanceSettings.Domain, tweet.OriginalAuthor.Acct, tweet.RetweetId.ToString());
+                announceId  = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, tweet.Id.ToString());
+            }
 
             var to = $"{actorUrl}/followers";
 
-            var isUnlisted = _publicationRepository.IsUnlisted(username);
             var cc = new string[0];
-            if (isUnlisted)
-                cc = new[] {"https://www.w3.org/ns/activitystreams#Public"};
             
             string summary = null;
-            var sensitive = _publicationRepository.IsSensitive(username);
-            if (sensitive || tweet.IsSensitive)
-                summary = "Sensitive Content";
 
             var extractedTags = _statusExtractor.Extract(tweet.MessageContent);
             _statisticsHandler.ExtractedStatus(extractedTags.tags.Count(x => x.type == "Mention"));
 
             // Replace RT by a link
             var content = extractedTags.content;
-            if (content.Contains("{RT}") && tweet.IsRetweet)
+            if (tweet.IsRetweet)
             {
-                if (!string.IsNullOrWhiteSpace(tweet.RetweetUrl))
-                    content = content.Replace("{RT}",
-                        $@"<a href=""{tweet.RetweetUrl}"" rel=""nofollow noopener noreferrer"" target=""_blank"">RT</a>");
-                else
-                    content = content.Replace("{RT}", "RT");
+                // content = "RT: " + content;
+                cc = new[] {"https://www.w3.org/ns/activitystreams#Public"};
             }
+            cc = new[] {"https://www.w3.org/ns/activitystreams#Public"};
 
             string inReplyTo = null;
             if (tweet.InReplyToStatusId != default)
                 inReplyTo = $"https://{_instanceSettings.Domain}/users/{tweet.InReplyToAccount.ToLowerInvariant()}/statuses/{tweet.InReplyToStatusId}";
 
-            if( tweet.QuoteTweetUrl != null )
-                content += $@"<span class=""quote-inline""><br><br>RT: <a href=""{tweet.QuoteTweetUrl}"">{tweet.QuoteTweetUrl}</a></span>";
+			if (tweet.QuoteTweetUrl != null)
+				content += $@"<span class=""quote-inline""><br><br>RT: <a href=""{tweet.QuoteTweetUrl}"">{tweet.QuoteTweetUrl}</a></span>";
 
             var note = new Note
             {
                 id = noteUrl,
+                announceId = announceId,
 
                 published = tweet.CreatedAt.ToString("s") + "Z",
                 url = noteUrl,
@@ -89,17 +87,50 @@ namespace BirdsiteLive.Domain
                 to = new[] { to },
                 cc = cc,
 
-                sensitive = tweet.IsSensitive || sensitive,
+                sensitive = false,
                 summary = summary,
                 content = $"<p>{content}</p>",
                 attachment = Convert(tweet.Media),
                 tag = extractedTags.tags,
-
                 quoteUrl = tweet.QuoteTweetUrl
             };
 
             return note;
         }
+        public ActivityCreateNote GetActivity(string username, ExtractedTweet tweet)
+        {
+            var note = GetStatus(username, tweet);
+            var actor = UrlFactory.GetActorUrl(_instanceSettings.Domain, username);
+            String noteUri;
+            string activityType;
+            if (tweet.IsRetweet) 
+            {
+                noteUri = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, tweet.Id.ToString());
+                activityType = "Announce";
+            } else
+            {
+                noteUri = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, tweet.Id.ToString());
+                activityType = "Create";
+            }
+
+            var now = DateTime.UtcNow;
+            var nowString = now.ToString("s") + "Z";
+
+            var noteActivity = new ActivityCreateNote()
+            {
+                context = "https://www.w3.org/ns/activitystreams",
+                id = $"{noteUri}/activity",
+                type = activityType,
+                actor = actor,
+                published = nowString,
+
+                to = new[] {$"{actor}/followers"},
+                cc = note.cc,
+                apObject = note
+            };
+
+            return noteActivity;
+        }
 
         private Attachment[] Convert(ExtractedMedia[] media)
         {

src/BirdsiteLive.Domain/TheFedInfoService.cs

@@ -1,162 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Net.Http;
-using System.Net.Http.Headers;
-using System.Text;
-using System.Threading;
-using System.Threading.Tasks;
-using Newtonsoft.Json;
-
-namespace BirdsiteLive.Domain
-{
-    public interface ITheFedInfoService
-    {
-        Task<List<BslInstanceInfo>> GetBslInstanceListAsync();
-    }
-
-    public class TheFedInfoService : ITheFedInfoService
-    {
-        private readonly IHttpClientFactory _httpClientFactory;
-        
-        #region Ctor
-        public TheFedInfoService(IHttpClientFactory httpClientFactory)
-        {
-            _httpClientFactory = httpClientFactory;
-        }
-        #endregion
-
-        public async Task<List<BslInstanceInfo>> GetBslInstanceListAsync()
-        {
-            var cancellationToken = CancellationToken.None;
-
-            var result = await CallGraphQlAsync<MyResponseData>(
-                new Uri("https://the-federation.info/graphql"),
-                HttpMethod.Get,
-                "query ($platform: String!) { nodes(platform: $platform) { host, version } }",
-                new
-                {
-                    platform = "birdsitelive",
-                },
-                cancellationToken);
-
-            var convertedResults = ConvertResults(result);
-            return convertedResults;
-        }
-
-        private List<BslInstanceInfo> ConvertResults(GraphQLResponse<MyResponseData> qlData)
-        {
-            var results = new List<BslInstanceInfo>();
-
-            foreach (var instanceInfo in qlData.Data.Nodes)
-            {
-                try
-                {
-                    var rawVersion = instanceInfo.Version.Split('+').First();
-                    if (string.IsNullOrWhiteSpace(rawVersion)) continue;
-                    var version = Version.Parse(rawVersion);
-                    if(version <= new Version(0,1,0)) continue;
-
-                    var instance = new BslInstanceInfo
-                    {
-                        Host = instanceInfo.Host,
-                        Version = version
-                    };
-                    results.Add(instance);
-                }
-                catch (Exception e)
-                {
-                    Console.WriteLine(e);
-                }
-            }
-
-            return results;
-        }
-
-        private async Task<GraphQLResponse<TResponse>> CallGraphQlAsync<TResponse>(Uri endpoint, HttpMethod method, string query, object variables, CancellationToken cancellationToken)
-        {
-            var content = new StringContent(SerializeGraphQlCall(query, variables), Encoding.UTF8, "application/json");
-            var httpRequestMessage = new HttpRequestMessage
-            {
-                Method = method,
-                Content = content,
-                RequestUri = endpoint,
-            };
-            //add authorization headers if necessary here
-            httpRequestMessage.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
-            var httpClient = _httpClientFactory.CreateClient("BirdsiteLIVE");
-            using (var response = await httpClient.SendAsync(httpRequestMessage, cancellationToken))
-            {
-                //if (response.IsSuccessStatusCode)
-                if (response?.Content.Headers.ContentType?.MediaType == "application/json")
-                {
-                    var responseString = await response.Content.ReadAsStringAsync(); //cancellationToken supported for .NET 5/6
-                    return DeserializeGraphQlCall<TResponse>(responseString);
-                }
-                else
-                {
-                    throw new ApplicationException($"Unable to contact '{endpoint}': {response.StatusCode} - {response.ReasonPhrase}");
-                }
-            }
-        }
-
-        private string SerializeGraphQlCall(string query, object variables)
-        {
-            var sb = new StringBuilder();
-            var textWriter = new StringWriter(sb);
-            var serializer = new JsonSerializer();
-            serializer.Serialize(textWriter, new
-            {
-                query = query,
-                variables = variables,
-            });
-            return sb.ToString();
-        }
-
-        private GraphQLResponse<TResponse> DeserializeGraphQlCall<TResponse>(string response)
-        {
-            var serializer = new JsonSerializer();
-            var stringReader = new StringReader(response);
-            var jsonReader = new JsonTextReader(stringReader);
-            var result = serializer.Deserialize<GraphQLResponse<TResponse>>(jsonReader);
-            return result;
-        }
-        
-        private class GraphQLResponse<TResponse>
-        {
-            public List<GraphQLError> Errors { get; set; }
-            public TResponse Data { get; set; }
-        }
-
-        private class GraphQLError
-        {
-            public string Message { get; set; }
-            public List<GraphQLErrorLocation> Locations { get; set; }
-            public List<object> Path { get; set; } //either int or string
-        }
-
-        private class GraphQLErrorLocation
-        {
-            public int Line { get; set; }
-            public int Column { get; set; }
-        }
-        
-        private class MyResponseData
-        {
-            public Node[] Nodes { get; set; }
-        }
-
-        private class Node
-        {
-            public string Host { get; set; }
-            public string Version { get; set; }
-        }
-    }
-
-    public class BslInstanceInfo
-    {
-        public string Host { get; set; }
-        public Version Version { get; set; }
-    }
-}

src/BirdsiteLive.Domain/Tools/ModerationRegexParser.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Text.RegularExpressions;
 using BirdsiteLive.Domain.Repository;
-using Org.BouncyCastle.Pkcs;
 
 namespace BirdsiteLive.Domain.Tools
 {

src/BirdsiteLive.Domain/Tools/StatusExtractor.cs

@@ -6,6 +6,7 @@ using BirdsiteLive.Common.Regexes;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Twitter;
 using Microsoft.Extensions.Logging;
+using System;
 
 namespace BirdsiteLive.Domain.Tools
 {
@@ -35,10 +36,6 @@ namespace BirdsiteLive.Domain.Tools
             messageContent = Regex.Replace(messageContent, @"\r\n\r\n?|\n\n", "</p><p>");
             messageContent = Regex.Replace(messageContent, @"\r\n?|\n", "<br/>");
 
-            //// Secure emojis
-            //var emojiMatch = EmojiRegexes.Emoji.Matches(messageContent);
-            //foreach (Match m in emojiMatch)
-            //    messageContent = Regex.Replace(messageContent, m.ToString(), $" {m} ");
 
             // Extract Urls
             var urlMatch = UrlRegexes.Url.Matches(messageContent);
@@ -110,8 +107,8 @@ namespace BirdsiteLive.Domain.Tools
                         continue;
                     }
 
-                    var url = $"https://{_instanceSettings.Domain}/users/{mention}";
-                    var name = $"@{mention}@{_instanceSettings.Domain}";
+                    var url = $"https://{_instanceSettings.Domain}/users/{mention.ToLower()}";
+                    var name = $"@{mention.ToLower()}";
 
                     if (tags.All(x => x.href != url))
                     {
@@ -124,7 +121,7 @@ namespace BirdsiteLive.Domain.Tools
                     }
 
                     messageContent = Regex.Replace(messageContent, Regex.Escape(m.Groups[0].ToString()),
-                        $@"{m.Groups[1]}<span class=""h-card""><a href=""https://{_instanceSettings.Domain}/@{mention}"" class=""u-url mention"">@<span>{mention}</span></a></span>{m.Groups[3]}");
+                        $@"{m.Groups[1]}<span class=""h-card""><a href=""{url}"" class=""u-url mention"">@<span>{mention.ToLower()}</span></a></span>{m.Groups[3]}");
                 }
             }
 

src/BirdsiteLive.Domain/UserService.cs

@@ -11,22 +11,18 @@ using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.Common.Regexes;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Cryptography;
-using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Domain.BusinessUseCases;
 using BirdsiteLive.Domain.Repository;
 using BirdsiteLive.Domain.Statistics;
 using BirdsiteLive.Domain.Tools;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
-using Tweetinvi.Core.Exceptions;
-using Tweetinvi.Models;
 
 namespace BirdsiteLive.Domain
 {
     public interface IUserService
     {
-        Actor GetUser(TwitterUser twitterUser, SyncTwitterUser dbTwitterUser);
+        Actor GetUser(TwitterUser twitterUser);
         Task<bool> FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders, ActivityFollow activity, string body);
         Task<bool> UndoFollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders, ActivityUndoFollow activity, string body);
 
@@ -50,10 +46,8 @@ namespace BirdsiteLive.Domain
 
         private readonly IModerationRepository _moderationRepository;
 
-        private readonly IFollowersDal _followerDal;
-
         #region Ctor
-        public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler, ITwitterUserService twitterUserService, IModerationRepository moderationRepository, IFollowersDal followerDal, IProcessDeleteUser processDeleteUser)
+        public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler, ITwitterUserService twitterUserService, IModerationRepository moderationRepository, IProcessDeleteUser processDeleteUser)
         {
             _instanceSettings = instanceSettings;
             _cryptoService = cryptoService;
@@ -64,12 +58,11 @@ namespace BirdsiteLive.Domain
             _statisticsHandler = statisticsHandler;
             _twitterUserService = twitterUserService;
             _moderationRepository = moderationRepository;
-            _followerDal = followerDal;
             _processDeleteUser = processDeleteUser;
         }
         #endregion
 
-        public Actor GetUser(TwitterUser twitterUser, SyncTwitterUser dbTwitterUser)
+        public Actor GetUser(TwitterUser twitterUser)
         {
             var actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, twitterUser.Acct);
             var acct = twitterUser.Acct.ToLowerInvariant();
@@ -84,34 +77,6 @@ namespace BirdsiteLive.Domain
                 _statisticsHandler.ExtractedDescription(extracted.tags.Count(x => x.type == "Mention"));
             }
 
-            var attachments = new List<UserAttachment>();
-            attachments.Add(new UserAttachment
-            {
-                type = "PropertyValue",
-                name = _instanceSettings.TwitterDomainLabel != "" ? _instanceSettings.TwitterDomainLabel : _instanceSettings.TwitterDomain,
-                value = $"<a href=\"https://{_instanceSettings.TwitterDomain}/{acct}\" rel=\"me nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">{_instanceSettings.TwitterDomain}/{acct}</span></a>"
-            });
-
-            if(_instanceSettings.TwitterDomain != "twitter.com")
-            {
-                attachments.Add(new UserAttachment
-                {
-                    type = "PropertyValue",
-                    name = "Twitter",
-                    value = $"twitter.com/{acct}"
-                });
-            }
-
-            if (_instanceSettings.ShowAboutInstanceOnProfiles)
-            {
-                attachments.Add(new UserAttachment
-                {
-                    type = "PropertyValue",
-                    name = $"About {_instanceSettings.Name}",
-                    value = $"<a href=\"https://{_instanceSettings.Domain}/About\" rel=\"nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">{_instanceSettings.Domain}/About</span></a>"
-                });
-            }
-
             var user = new Actor
             {
                 id = actorUrl,
@@ -120,10 +85,9 @@ namespace BirdsiteLive.Domain
                 preferredUsername = acct,
                 name = twitterUser.Name,
                 inbox = $"{actorUrl}/inbox",
-                summary = "[UNOFFICIAL MIRROR: This is a view of Twitter using ActivityPub]<br/><br/>" + description,
+                summary = $"{description} <br /> <br /> (mirror of @{acct}@twitter.com)",
                 url = actorUrl,
                 manuallyApprovesFollowers = twitterUser.Protected,
-                discoverable = false,
                 publicKey = new PublicKey()
                 {
                     id = $"{actorUrl}#main-key",
@@ -140,12 +104,12 @@ namespace BirdsiteLive.Domain
                     mediaType = "image/jpeg",
                     url = twitterUser.ProfileBannerURL
                 },
-                attachment = new []
+                attachment = new[]
                 {
                     new UserAttachment
                     {
                         type = "PropertyValue",
-                        name = "Official Account",
+                        name = "Official",
                         value = $"<a href=\"https://twitter.com/{acct}\" rel=\"me nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">twitter.com/{acct}</span></a>"
                     },
                     new UserAttachment
@@ -154,40 +118,13 @@ namespace BirdsiteLive.Domain
                         name = "Disclaimer",
                         value = "This is an automatically created and managed mirror profile from Twitter. While it reflects exactly the content of the original account, it doesn't provide support for interactions and replies. It is an equivalent view from other 3rd party Twitter client apps and uses the same technical means to provide it."
                     },
-                    new UserAttachment
-                    {
-                        type = "PropertyValue",
-                        name = "Take control of this account",
-                        value = $"<a href=\"https://{_instanceSettings.Domain}/migration/move/{acct}\" rel=\"me nofollow noopener noreferrer\" target=\"_blank\">MANAGE</a>"
-                    }
+
                 },
                 endpoints = new EndPoints
                 {
                     sharedInbox = $"https://{_instanceSettings.Domain}/inbox"
-                },
-                movedTo = dbTwitterUser?.MovedTo
+                }
             };
-
-            if (twitterUser.Verified)
-            {
-                user.tag = new List<Tag>
-                {
-                    new Tag
-                    {
-                        icon = new TagResource
-                        {
-                            type = "Image",
-                            url = "https://" + _instanceSettings.Domain + "/verified.png"
-                        },
-                        id = "https://" + _instanceSettings.Domain + "/verified.png",
-                        name = ":verified:",
-                        type = "Emoji"
-                    }
-                };
-
-                user.name += " :verified:";
-            }
-
             return user;
         }
 
@@ -229,18 +166,8 @@ namespace BirdsiteLive.Domain
                     return await SendRejectFollowAsync(activity, followerHost);
             }
 
-            // Validate follower count < MaxFollowsPerUser
-            if (_instanceSettings.MaxFollowsPerUser > 0) {
-                var follower = await _followerDal.GetFollowerAsync(followerUserName, followerHost);
-
-                if (follower != null && follower.Followings.Count + 1 > _instanceSettings.MaxFollowsPerUser)
-                {
-                    return await SendRejectFollowAsync(activity, followerHost);
-                }
-            }
-
             // Validate User Protected
-            var user = _twitterUserService.GetUser(twitterUser);
+            var user = await _twitterUserService.GetUserAsync(twitterUser);
             if (!user.Protected)
             {
                 // Execute
@@ -256,23 +183,11 @@ namespace BirdsiteLive.Domain
         
         private async Task<bool> SendAcceptFollowAsync(ActivityFollow activity, string followerHost)
         {
-            var acceptFollow = new ActivityAcceptFollow()
-            {
-                context = "https://www.w3.org/ns/activitystreams",
-                id = $"{activity.apObject}#accepts/follows/{Guid.NewGuid()}",
-                type = "Accept",
-                actor = activity.apObject,
-                apObject = new ActivityFollow()
-                {
-                    id = activity.id,
-                    type = activity.type,
-                    actor = activity.actor,
-                    apObject = activity.apObject
-                }
-            };
+            var acceptFollow = _activityPubService.BuildAcceptFollow(activity);
             var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject);
             return result == HttpStatusCode.Accepted ||
                    result == HttpStatusCode.OK; //TODO: revamp this for better error handling
+
         }
 
         public async Task<bool> SendRejectFollowAsync(ActivityFollow activity, string followerHost)
@@ -330,10 +245,11 @@ namespace BirdsiteLive.Domain
                 actor = activity.apObject.apObject,
                 apObject = new ActivityUndoFollow()
                 {
-                    id = activity.id,
-                    type = activity.type,
-                    actor = activity.actor,
-                    apObject = activity.apObject
+                    id = (activity.apObject as dynamic).id?.ToString(),
+                    type = (activity.apObject as dynamic).type?.ToString(),
+                    actor = (activity.apObject as dynamic).actor?.ToString(),
+                    context = (activity.apObject as dynamic).context?.ToString(),
+                    apObject = (activity.apObject as dynamic).@object?.ToString()
                 }
             };
             var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject.apObject);
@@ -358,6 +274,13 @@ namespace BirdsiteLive.Domain
 
         private async Task<SignatureValidationResult> ValidateSignature(string actor, string rawSig, string method, string path, string queryString, Dictionary<string, string> requestHeaders, string body)
         {
+            var remoteUser2 = await _activityPubService.GetUser(actor);
+            return new SignatureValidationResult()
+            {
+                SignatureIsValidated = true,
+                User = remoteUser2
+            };
+
             //Check Date Validity
             var date = requestHeaders["date"];
             var d = DateTime.Parse(date).ToUniversalTime();
@@ -388,6 +311,8 @@ namespace BirdsiteLive.Domain
             // Retrieve User
             var remoteUser = await _activityPubService.GetUser(actor);
 
+	                Console.WriteLine(remoteUser.publicKey.publicKeyPem);
+
             // Prepare Key data
             var toDecode = remoteUser.publicKey.publicKeyPem.Trim().Remove(0, remoteUser.publicKey.publicKeyPem.IndexOf('\n'));
             toDecode = toDecode.Remove(toDecode.LastIndexOf('\n')).Replace("\n", "");
@@ -401,6 +326,7 @@ namespace BirdsiteLive.Domain
             }
             toSign.Remove(toSign.Length - 1, 1);
 
+	    Console.WriteLine(Convert.FromBase64String(toDecode));
             // Import key
             var key = new RSACryptoServiceProvider();
             var rsaKeyInfo = key.ExportParameters(false);

src/BirdsiteLive.Moderation/Actions/RemoveTwitterAccountAction.cs

@@ -41,9 +41,6 @@ namespace BirdsiteLive.Moderation.Actions
                 if (follower.Followings.Contains(twitterUserId))
                     follower.Followings.Remove(twitterUserId);
 
-                if (follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
-                    follower.FollowingsSyncStatus.Remove(twitterUserId);
-
                 if (follower.Followings.Any())
                     await _followersDal.UpdateFollowerAsync(follower);
                 else

src/BirdsiteLive.Moderation/BirdsiteLive.Moderation.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

src/BirdsiteLive.Moderation/Processors/TwitterAccountModerationProcessor.cs

@@ -29,7 +29,7 @@ namespace BirdsiteLive.Moderation.Processors
         {
             if (type == ModerationTypeEnum.None) return;
             
-            var twitterUsers = await _twitterUserDal.GetAllTwitterUsersAsync(false);
+            var twitterUsers = await _twitterUserDal.GetAllTwitterUsersAsync();
 
             foreach (var user in twitterUsers)
             {

src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj

@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="5.0.0" />
-    <PackageReference Include="System.Threading.Tasks.Dataflow" Version="4.11.1" />
+    <PackageReference Include="System.Threading.Tasks.Dataflow" Version="6.0" />
   </ItemGroup>
 
   <ItemGroup>

src/BirdsiteLive.Pipeline/Contracts/IRefreshTwitterUserStatusProcessor.cs

@@ -1,12 +0,0 @@
-using System.Threading;
-using System.Threading.Tasks;
-using BirdsiteLive.DAL.Models;
-using BirdsiteLive.Pipeline.Models;
-
-namespace BirdsiteLive.Pipeline.Contracts
-{
-    public interface IRefreshTwitterUserStatusProcessor
-    {
-        Task<UserWithDataToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct);
-    }
-}

src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterUsersProcessor.cs

@@ -1,12 +1,12 @@
 using System.Threading;
 using System.Threading.Tasks;
 using System.Threading.Tasks.Dataflow;
-using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Pipeline.Models;
 
 namespace BirdsiteLive.Pipeline.Contracts
 {
     public interface IRetrieveTwitterUsersProcessor
     {
-        Task GetTwitterUsersAsync(BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock, CancellationToken ct);
+        Task GetTwitterUsersAsync(BufferBlock<UserWithDataToSync[]> twitterUsersBufferBlock, CancellationToken ct);
     }
 }

src/BirdsiteLive.Pipeline/Contracts/ISaveProgressionProcessor.cs

@@ -1,11 +0,0 @@
-using System.Threading;
-using System.Threading.Tasks;
-using BirdsiteLive.Pipeline.Models;
-
-namespace BirdsiteLive.Pipeline.Contracts
-{
-    public interface ISaveProgressionProcessor
-    {
-        Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct);
-    }
-}

src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs

@@ -6,6 +6,6 @@ namespace BirdsiteLive.Pipeline.Contracts
 {
     public interface ISendTweetsToFollowersProcessor
     {
-        Task<UserWithDataToSync> ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct);
+        Task ProcessAsync(UserWithDataToSync[] usersWithTweetsToSync, CancellationToken ct);
     }
 }

src/BirdsiteLive.Pipeline/Models/UserWithDataToSync.cs

@@ -1,6 +1,5 @@
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Twitter.Models;
-using Tweetinvi.Models;
 
 namespace BirdsiteLive.Pipeline.Models
 {

src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs

@@ -1,109 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Threading;
-using System.Threading.Tasks;
-using BirdsiteLive.Common.Settings;
-using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.DAL.Models;
-using BirdsiteLive.Moderation.Actions;
-using BirdsiteLive.Pipeline.Contracts;
-using BirdsiteLive.Pipeline.Models;
-using BirdsiteLive.Twitter;
-using BirdsiteLive.Twitter.Models;
-
-namespace BirdsiteLive.Pipeline.Processors
-{
-    public class RefreshTwitterUserStatusProcessor : IRefreshTwitterUserStatusProcessor
-    {
-        private readonly ICachedTwitterUserService _twitterUserService;
-        private readonly ITwitterUserDal _twitterUserDal;
-        private readonly IRemoveTwitterAccountAction _removeTwitterAccountAction;
-        private readonly InstanceSettings _instanceSettings;
-
-        #region Ctor
-        public RefreshTwitterUserStatusProcessor(ICachedTwitterUserService twitterUserService, ITwitterUserDal twitterUserDal, IRemoveTwitterAccountAction removeTwitterAccountAction, InstanceSettings instanceSettings)
-        {
-            _twitterUserService = twitterUserService;
-            _twitterUserDal = twitterUserDal;
-            _removeTwitterAccountAction = removeTwitterAccountAction;
-            _instanceSettings = instanceSettings;
-        }
-        #endregion
-
-        public async Task<UserWithDataToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct)
-        {
-            var usersWtData = new List<UserWithDataToSync>();
-
-            foreach (var user in syncTwitterUsers)
-            {
-                TwitterUser userView = null;
-
-                try
-                {
-                    userView = _twitterUserService.GetUser(user.Acct);
-                }
-                catch (UserNotFoundException)
-                {
-                    await ProcessNotFoundUserAsync(user);
-                    continue;
-                }
-                catch (UserHasBeenSuspendedException)
-                {
-                    await ProcessNotFoundUserAsync(user);
-                    continue;
-                }
-                catch (RateLimitExceededException)
-                {
-                    await ProcessRateLimitExceededAsync(user);
-                    continue;
-                }
-                catch (Exception)
-                {
-                    // ignored
-                }
-
-                if (userView == null || userView.Protected)
-                {
-                    await ProcessFailingUserAsync(user);
-                    continue;
-                }
-
-                user.FetchingErrorCount = 0;
-                var userWtData = new UserWithDataToSync
-                {
-                    User = user
-                };
-                usersWtData.Add(userWtData);
-            }
-            return usersWtData.ToArray();
-        }
-
-        private async Task ProcessRateLimitExceededAsync(SyncTwitterUser user)
-        {
-            var dbUser = await _twitterUserDal.GetTwitterUserAsync(user.Acct);
-            dbUser.LastSync = DateTime.UtcNow;
-            await _twitterUserDal.UpdateTwitterUserAsync(dbUser);
-        }
-
-        private async Task ProcessNotFoundUserAsync(SyncTwitterUser user)
-        {
-            await _removeTwitterAccountAction.ProcessAsync(user);
-        }
-
-        private async Task ProcessFailingUserAsync(SyncTwitterUser user)
-        {
-            var dbUser = await _twitterUserDal.GetTwitterUserAsync(user.Acct);
-            dbUser.FetchingErrorCount++;
-            dbUser.LastSync = DateTime.UtcNow;
-
-            if (dbUser.FetchingErrorCount > _instanceSettings.FailingTwitterUserCleanUpThreshold)
-            {
-                await _removeTwitterAccountAction.ProcessAsync(user);
-            }
-            else
-            {
-                await _twitterUserDal.UpdateTwitterUserAsync(dbUser);
-            }
-        }
-    }
-}

src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using BirdsiteLive.DAL.Contracts;
@@ -20,12 +21,18 @@ namespace BirdsiteLive.Pipeline.Processors
 
         public async Task<IEnumerable<UserWithDataToSync>> ProcessAsync(UserWithDataToSync[] userWithTweetsToSyncs, CancellationToken ct)
         {
-            //TODO multithread this
-            foreach (var user in userWithTweetsToSyncs)
-            {
-                var followers = await _followersDal.GetFollowersAsync(user.User.Id);
-                user.Followers = followers;
-            }
+            //List<Task> todo = new List<Task>();
+            //foreach (var user in userWithTweetsToSyncs)
+            //{
+            //    var t = Task.Run( 
+            //    async() => {
+            //        var followers = await _followersDal.GetFollowersAsync(user.User.Id);
+            //        user.Followers = followers;
+            //    });
+            //    todo.Add(t);
+            //}
+            //
+            //await Task.WhenAll(todo);
 
             return userWithTweetsToSyncs;
         }

src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
@@ -9,10 +10,10 @@ using BirdsiteLive.Pipeline.Contracts;
 using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
+using BirdsiteLive.Common.Settings;
 using Microsoft.Extensions.Logging;
-using Tweetinvi.Models;
 
-namespace BirdsiteLive.Pipeline.Processors
+namespace BirdsiteLive.Pipeline.Processors.SubTasks
 {
     public class RetrieveTweetsProcessor : IRetrieveTweetsProcessor
     {
@@ -20,57 +21,90 @@ namespace BirdsiteLive.Pipeline.Processors
         private readonly ICachedTwitterUserService _twitterUserService;
         private readonly ITwitterUserDal _twitterUserDal;
         private readonly ILogger<RetrieveTweetsProcessor> _logger;
+        private readonly InstanceSettings _settings;
 
         #region Ctor
-        public RetrieveTweetsProcessor(ITwitterTweetsService twitterTweetsService, ITwitterUserDal twitterUserDal, ICachedTwitterUserService twitterUserService, ILogger<RetrieveTweetsProcessor> logger)
+        public RetrieveTweetsProcessor(ITwitterTweetsService twitterTweetsService, ITwitterUserDal twitterUserDal, ICachedTwitterUserService twitterUserService, InstanceSettings settings, ILogger<RetrieveTweetsProcessor> logger)
         {
             _twitterTweetsService = twitterTweetsService;
             _twitterUserDal = twitterUserDal;
             _twitterUserService = twitterUserService;
             _logger = logger;
+            _settings = settings;
         }
         #endregion
 
         public async Task<UserWithDataToSync[]> ProcessAsync(UserWithDataToSync[] syncTwitterUsers, CancellationToken ct)
         {
-            var usersWtTweets = new List<UserWithDataToSync>();
 
-            //TODO multithread this
-            foreach (var userWtData in syncTwitterUsers)
+            if (_settings.ParallelTwitterRequests == 0)
             {
-                var user = userWtData.User;
-                var tweets = RetrieveNewTweets(user);
-                if (tweets.Length > 0 && user.LastTweetPostedId != -1)
-                {
-                    userWtData.Tweets = tweets;
-                    usersWtTweets.Add(userWtData);
-                }
-                else if (tweets.Length > 0 && user.LastTweetPostedId == -1)
-                {
-                    var tweetId = tweets.Last().Id;
-                    var now = DateTime.UtcNow;
-                    await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now, user.MovedTo, user.MovedToAcct, user.Deleted);
-                }
-                else
-                {
-                    var now = DateTime.UtcNow;
-                    await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now, user.MovedTo, user.MovedToAcct, user.Deleted);
-                }
+                while(true)
+                    await Task.Delay(1000);
             }
 
+            var usersWtTweets = new ConcurrentBag<UserWithDataToSync>();
+            List<Task> todo = new List<Task>();
+            int index = 0;
+            foreach (var userWtData in syncTwitterUsers)
+            {
+                index++;
+
+                var t = Task.Run(async () => {
+                    var user = userWtData.User;
+                    var now = DateTime.UtcNow;
+                    try 
+                    {
+                        var tweets = await RetrieveNewTweets(user);
+                        _logger.LogInformation(index + "/" + syncTwitterUsers.Count() + " Got " + tweets.Length + " tweets from user " + user.Acct + " " );
+                        if (tweets.Length > 0 && user.LastTweetPostedId == -1)
+                        {
+                            // skip the first time to avoid sending backlog of tweet
+                            var tweetId = tweets.Last().Id;
+                            await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, user.FetchingErrorCount, now);
+                        }
+                        else if (tweets.Length > 0 && user.LastTweetPostedId != -1)
+                        {
+                            userWtData.Tweets = tweets;
+                            usersWtTweets.Add(userWtData);
+                            var tweetId = tweets.Last().Id;
+                            await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, user.FetchingErrorCount, now);
+                        }
+                        else
+                        {
+                            await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.FetchingErrorCount, now);
+                        }
+
+                    } 
+                    catch(Exception e)
+                    {
+                        _logger.LogError(e.Message);
+                        await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.FetchingErrorCount, now);
+                    }
+                });
+                todo.Add(t);
+                if (todo.Count > _settings.ParallelTwitterRequests)
+                {
+                    await Task.WhenAll(todo);
+                    todo.Clear();
+                }
+                
+            }
+
+            await Task.WhenAll(todo);
             return usersWtTweets.ToArray();
         }
 
-        private ExtractedTweet[] RetrieveNewTweets(SyncTwitterUser user)
+        private async Task<ExtractedTweet[]> RetrieveNewTweets(SyncTwitterUser user)
         {
             var tweets = new ExtractedTweet[0];
             
             try
             {
                 if (user.LastTweetPostedId == -1)
-                    tweets = _twitterTweetsService.GetTimeline(user.Acct, 1);
+                    tweets = await _twitterTweetsService.GetTimelineAsync(user.Acct);
                 else
-                    tweets = _twitterTweetsService.GetTimeline(user.Acct, 200, user.LastTweetSynchronizedForAllFollowersId);
+                    tweets = await _twitterTweetsService.GetTimelineAsync(user.Acct, user.LastTweetPostedId);
             }
             catch (Exception e)
             {

src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -6,9 +7,8 @@ using System.Threading.Tasks.Dataflow;
 using BirdsiteLive.Common.Extensions;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Pipeline.Contracts;
-using BirdsiteLive.Pipeline.Tools;
 using Microsoft.Extensions.Logging;
 
 namespace BirdsiteLive.Pipeline.Processors
@@ -16,57 +16,61 @@ namespace BirdsiteLive.Pipeline.Processors
     public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor
     {
         private readonly ITwitterUserDal _twitterUserDal;
-        private readonly IMaxUsersNumberProvider _maxUsersNumberProvider;
+        private readonly IFollowersDal _followersDal;
+        private readonly InstanceSettings _instanceSettings;
         private readonly ILogger<RetrieveTwitterUsersProcessor> _logger;
+        private static Random rng = new Random();
         
         public int WaitFactor = 1000 * 60; //1 min
 
         #region Ctor
-        public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IMaxUsersNumberProvider maxUsersNumberProvider, ILogger<RetrieveTwitterUsersProcessor> logger)
+        public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, InstanceSettings instanceSettings, ILogger<RetrieveTwitterUsersProcessor> logger)
         {
             _twitterUserDal = twitterUserDal;
-            _maxUsersNumberProvider = maxUsersNumberProvider;
+            _followersDal = followersDal;
+            _instanceSettings = instanceSettings;
             _logger = logger;
         }
         #endregion
 
-        public async Task GetTwitterUsersAsync(BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock, CancellationToken ct)
+        public async Task GetTwitterUsersAsync(BufferBlock<UserWithDataToSync[]> twitterUsersBufferBlock, CancellationToken ct)
         {
             for (; ; )
             {
                 ct.ThrowIfCancellationRequested();
 
-                try
+                if (_instanceSettings.ParallelTwitterRequests == 0)
                 {
-                    var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync();
-                    var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber, false);
+                    while (true)
+                        await Task.Delay(10000);
+                }
+                
+                var usersDal = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000, _instanceSettings.n_start, _instanceSettings.n_end, _instanceSettings.m);
 
-                    var userCount = users.Any() ? users.Length : 1;
-                    var splitNumber = (int) Math.Ceiling(userCount / 15d);
-                    var splitUsers = users.Split(splitNumber).ToList();
+                var userCount = usersDal.Any() ? Math.Min(usersDal.Length, 200) : 1;
+                var splitUsers = usersDal.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList();
 
-                    foreach (var u in splitUsers)
+                foreach (var users in splitUsers)
+                {
+                    ct.ThrowIfCancellationRequested();
+                    List<UserWithDataToSync> toSync = new List<UserWithDataToSync>();
+                    foreach (var u in users)
                     {
-                        ct.ThrowIfCancellationRequested();
-
-                        await twitterUsersBufferBlock.SendAsync(u.ToArray(), ct);
-
-                        await Task.Delay(WaitFactor, ct);
+                        var followers = await _followersDal.GetFollowersAsync(u.Id);
+                        toSync.Add( new UserWithDataToSync()
+                        {
+                            User = u,
+                            Followers = followers
+                        });
+                        
                     }
 
-                    var splitCount = splitUsers.Count();
-                    if (splitCount < 15) await Task.Delay((15 - splitCount) * WaitFactor, ct); //Always wait 15min
+                    await twitterUsersBufferBlock.SendAsync(toSync.ToArray(), ct);
 
-                    //// Extra wait time to fit 100.000/day limit
-                    //var extraWaitTime = (int)Math.Ceiling((60 / ((100000d / 24) / userCount)) - 15);
-                    //if (extraWaitTime < 0) extraWaitTime = 0;
-                    //await Task.Delay(extraWaitTime * 1000, ct);
-                }
-                catch (Exception e)
-                {
-                    _logger.LogError(e, "Failing retrieving Twitter Users.");
                 }
+                
+                await Task.Delay(10, ct); // this is somehow necessary
             }
         }
     }
-}
+}

src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs

@@ -1,66 +0,0 @@
-using System;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.DAL.Models;
-using BirdsiteLive.Moderation.Actions;
-using BirdsiteLive.Pipeline.Contracts;
-using BirdsiteLive.Pipeline.Models;
-using Microsoft.Extensions.Logging;
-
-namespace BirdsiteLive.Pipeline.Processors
-{
-    public class SaveProgressionProcessor : ISaveProgressionProcessor
-    {
-        private readonly ITwitterUserDal _twitterUserDal;
-        private readonly ILogger<SaveProgressionProcessor> _logger;
-        private readonly IRemoveTwitterAccountAction _removeTwitterAccountAction;
-
-        #region Ctor
-        public SaveProgressionProcessor(ITwitterUserDal twitterUserDal, ILogger<SaveProgressionProcessor> logger, IRemoveTwitterAccountAction removeTwitterAccountAction)
-        {
-            _twitterUserDal = twitterUserDal;
-            _logger = logger;
-            _removeTwitterAccountAction = removeTwitterAccountAction;
-        }
-        #endregion
-
-        public async Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
-        {
-            try
-            {
-                if (userWithTweetsToSync.Tweets.Length == 0)
-                {
-                    _logger.LogInformation("No tweets synchronized");
-                    await UpdateUserSyncDateAsync(userWithTweetsToSync.User);
-                    return;
-                }
-                if(userWithTweetsToSync.Followers.Length == 0)
-                {
-                    _logger.LogInformation("No Followers found for {User}", userWithTweetsToSync.User.Acct);
-                    await _removeTwitterAccountAction.ProcessAsync(userWithTweetsToSync.User);
-                    return;
-                }
-            
-                var userId = userWithTweetsToSync.User.Id;
-                var followingSyncStatuses = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).ToList();
-                var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();
-                var minimumSync = followingSyncStatuses.Min();
-                var now = DateTime.UtcNow;
-                await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now, userWithTweetsToSync.User.MovedTo, userWithTweetsToSync.User.MovedToAcct, userWithTweetsToSync.User.Deleted);
-            }
-            catch (Exception e)
-            {
-                _logger.LogError(e, "SaveProgressionProcessor.ProcessAsync() Exception");
-                throw;
-            }
-        }
-
-        private async Task UpdateUserSyncDateAsync(SyncTwitterUser user)
-        {
-            user.LastSync = DateTime.UtcNow;
-            await _twitterUserDal.UpdateTwitterUserAsync(user);
-        }
-    }
-}

src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs

@@ -16,7 +16,6 @@ using BirdsiteLive.Pipeline.Processors.SubTasks;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
 using Microsoft.Extensions.Logging;
-using Tweetinvi.Models;
 
 namespace BirdsiteLive.Pipeline.Processors
 {
@@ -28,6 +27,7 @@ namespace BirdsiteLive.Pipeline.Processors
         private readonly InstanceSettings _instanceSettings;
         private readonly ILogger<SendTweetsToFollowersProcessor> _logger;
         private readonly IRemoveFollowerAction _removeFollowerAction;
+        private List<Task> _todo = new List<Task>();
 
         #region Ctor
         public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox, IFollowersDal followersDal, ILogger<SendTweetsToFollowersProcessor> logger, InstanceSettings instanceSettings, IRemoveFollowerAction removeFollowerAction)
@@ -41,23 +41,41 @@ namespace BirdsiteLive.Pipeline.Processors
         }
         #endregion
 
-        public async Task<UserWithDataToSync> ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
+        public async Task ProcessAsync(UserWithDataToSync[] usersWithTweetsToSync, CancellationToken ct)
         {
-            var user = userWithTweetsToSync.User;
+            foreach (var userWithTweetsToSync in usersWithTweetsToSync)
+            {
+                var user = userWithTweetsToSync.User;
 
-            // Process Shared Inbox
-            var followersWtSharedInbox = userWithTweetsToSync.Followers
-                .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
-                .ToList();
-            await ProcessFollowersWithSharedInboxAsync(userWithTweetsToSync.Tweets, followersWtSharedInbox, user);
+                _todo = _todo.Where(x => !x.IsCompleted).ToList();
+                
+                var t = Task.Run( async () => 
+                {
+                    // Process Shared Inbox
+                    var followersWtSharedInbox = userWithTweetsToSync.Followers
+                        .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
+                        .ToList();
+                    await ProcessFollowersWithSharedInboxAsync(userWithTweetsToSync.Tweets, followersWtSharedInbox, user);
 
-            // Process Inbox
-            var followerWtInbox = userWithTweetsToSync.Followers
-                .Where(x => string.IsNullOrWhiteSpace(x.SharedInboxRoute))
-                .ToList();
-            await ProcessFollowersWithInboxAsync(userWithTweetsToSync.Tweets, followerWtInbox, user);
+                    // Process Inbox
+                    var followerWtInbox = userWithTweetsToSync.Followers
+                        .Where(x => string.IsNullOrWhiteSpace(x.SharedInboxRoute))
+                        .ToList();
+                    await ProcessFollowersWithInboxAsync(userWithTweetsToSync.Tweets, followerWtInbox, user);
+                    
+                    _logger.LogInformation("Done sending " + userWithTweetsToSync.Tweets.Length + " tweets for "
+                        + userWithTweetsToSync.Followers.Length + "followers for user " + userWithTweetsToSync.User.Acct);
+                }, ct);
+                _todo.Add(t);
+
+                if (_todo.Count >= _instanceSettings.ParallelFediversePosts)
+                {
+                    await Task.WhenAny(_todo);
+                }
+                
+                
+            }
 
-            return userWithTweetsToSync;
         }
 
         private async Task ProcessFollowersWithSharedInboxAsync(ExtractedTweet[] tweets, List<Follower> followers, SyncTwitterUser user)
@@ -68,6 +86,7 @@ namespace BirdsiteLive.Pipeline.Processors
             {
                 try
                 {
+                    _logger.LogInformation("Sending " + tweets.Length + " tweets from user " + user.Acct + " to instance " + followersPerInstance.Key);
                     await _sendTweetsToSharedInbox.ExecuteAsync(tweets, user, followersPerInstance.Key, followersPerInstance.ToArray());
 
                     foreach (var f in followersPerInstance)

src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs

@@ -31,7 +31,6 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
         {
             _activityPubService = activityPubService;
             _statusService = statusService;
-            _followersDal = followersDal;
             _settings = settings;
             _logger = logger;
         }
@@ -40,51 +39,32 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
         public async Task ExecuteAsync(IEnumerable<ExtractedTweet> tweets, Follower follower, SyncTwitterUser user)
         {
             var userId = user.Id;
-            var fromStatusId = follower.FollowingsSyncStatus[userId];
+            //var fromStatusId = follower.FollowingsSyncStatus[userId];
             var tweetsToSend = tweets
-                .Where(x => x.Id > fromStatusId)
                 .OrderBy(x => x.Id)
                 .ToList();
 
             var inbox = follower.InboxRoute;
 
-            var syncStatus = fromStatusId;
-            try
+            foreach (var tweet in tweetsToSend)
             {
-                foreach (var tweet in tweetsToSend)
+                try
                 {
-                    try
+                    var activity = _statusService.GetActivity(user.Acct, tweet);
+                    await _activityPubService.PostNewActivity(activity, user.Acct, tweet.Id.ToString(), follower.Host, inbox);
+                }
+                catch (ArgumentException e)
+                {
+                    if (e.Message.Contains("Invalid pattern") && e.Message.Contains("at offset")) //Regex exception
                     {
-                        if (!tweet.IsReply ||
-                            tweet.IsReply && tweet.IsThread ||
-                            _settings.PublishReplies)
-                        {
-                            var note = _statusService.GetStatus(user.Acct, tweet);
-                            await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox);
-                        }
+                        _logger.LogError(e, "Can't parse {MessageContent} from Tweet {Id}", tweet.MessageContent, tweet.Id);
                     }
-                    catch (ArgumentException e)
+                    else
                     {
-                        if (e.Message.Contains("Invalid pattern") && e.Message.Contains("at offset")) //Regex exception
-                        {
-                            _logger.LogError(e, "Can't parse {MessageContent} from Tweet {Id}", tweet.MessageContent, tweet.Id);
-                        }
-                        else
-                        {
-                            throw;
-                        }
+                        throw;
                     }
+                }
 
-                    syncStatus = tweet.Id;
-                }
-            }
-            finally
-            {
-                if (syncStatus != fromStatusId)
-                {
-                    follower.FollowingsSyncStatus[userId] = syncStatus;
-                    await _followersDal.UpdateFollowerAsync(follower);
-                }
             }
         }
     }

src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs

@@ -40,54 +40,29 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
             var userId = user.Id;
             var inbox = followersPerInstance.First().SharedInboxRoute;
 
-            var fromStatusId = followersPerInstance
-                .Max(x => x.FollowingsSyncStatus[userId]);
-
             var tweetsToSend = tweets
-                .Where(x => x.Id > fromStatusId)
                 .OrderBy(x => x.Id)
                 .ToList();
 
-            var syncStatus = fromStatusId;
-            try
+            foreach (var tweet in tweetsToSend)
             {
-                foreach (var tweet in tweetsToSend)
+                try
                 {
-                    try
+                    var activity = _statusService.GetActivity(user.Acct, tweet);
+                    await _activityPubService.PostNewActivity(activity, user.Acct, tweet.Id.ToString(), host, inbox);
+                }
+                catch (ArgumentException e)
+                {
+                    if (e.Message.Contains("Invalid pattern") && e.Message.Contains("at offset")) //Regex exception
                     {
-                        if (!tweet.IsReply ||
-                            tweet.IsReply && tweet.IsThread ||
-                            _settings.PublishReplies)
-                        {
-                            var note = _statusService.GetStatus(user.Acct, tweet);
-                            await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox);
-                        }
+                        _logger.LogError(e, "Can't parse {MessageContent} from Tweet {Id}", tweet.MessageContent, tweet.Id);
                     }
-                    catch (ArgumentException e)
+                    else
                     {
-                        if (e.Message.Contains("Invalid pattern") && e.Message.Contains("at offset")) //Regex exception
-                        {
-                            _logger.LogError(e, "Can't parse {MessageContent} from Tweet {Id}", tweet.MessageContent, tweet.Id);
-                        }
-                        else
-                        {
-                            throw;
-                        }
+                        throw;
                     }
+                }
 
-                    syncStatus = tweet.Id;
-                }
-            }
-            finally
-            {
-                if (syncStatus != fromStatusId)
-                {
-                    foreach (var f in followersPerInstance)
-                    {
-                        f.FollowingsSyncStatus[userId] = syncStatus;
-                        await _followersDal.UpdateFollowerAsync(f);
-                    }
-                }
             }
         }
     }

src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs

@@ -18,22 +18,18 @@ namespace BirdsiteLive.Pipeline
     public class StatusPublicationPipeline : IStatusPublicationPipeline
     {
         private readonly IRetrieveTwitterUsersProcessor _retrieveTwitterAccountsProcessor;
-        private readonly IRefreshTwitterUserStatusProcessor _refreshTwitterUserStatusProcessor;
         private readonly IRetrieveTweetsProcessor _retrieveTweetsProcessor;
         private readonly IRetrieveFollowersProcessor _retrieveFollowersProcessor;
         private readonly ISendTweetsToFollowersProcessor _sendTweetsToFollowersProcessor;
-        private readonly ISaveProgressionProcessor _saveProgressionProcessor;
         private readonly ILogger<StatusPublicationPipeline> _logger;
 
         #region Ctor
-        public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ISaveProgressionProcessor saveProgressionProcessor, IRefreshTwitterUserStatusProcessor refreshTwitterUserStatusProcessor, ILogger<StatusPublicationPipeline> logger)
+        public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ILogger<StatusPublicationPipeline> logger)
         {
             _retrieveTweetsProcessor = retrieveTweetsProcessor;
-            _retrieveTwitterAccountsProcessor = retrieveTwitterAccountsProcessor;
             _retrieveFollowersProcessor = retrieveFollowersProcessor;
             _sendTweetsToFollowersProcessor = sendTweetsToFollowersProcessor;
-            _saveProgressionProcessor = saveProgressionProcessor;
-            _refreshTwitterUserStatusProcessor = refreshTwitterUserStatusProcessor;
+            _retrieveTwitterAccountsProcessor = retrieveTwitterAccountsProcessor;
 
             _logger = logger;
         }
@@ -41,37 +37,30 @@ namespace BirdsiteLive.Pipeline
 
         public async Task ExecuteAsync(CancellationToken ct)
         {
+            var standardBlockOptions = new ExecutionDataflowBlockOptions { BoundedCapacity = 1, MaxDegreeOfParallelism = 1, CancellationToken = ct};
             // Create blocks 
-            var twitterUserToRefreshBufferBlock = new BufferBlock<SyncTwitterUser[]>(new DataflowBlockOptions
+            var twitterUserToRefreshBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions
                 { BoundedCapacity = 1, CancellationToken = ct });
-            var twitterUserToRefreshBlock = new TransformBlock<SyncTwitterUser[], UserWithDataToSync[]>(async x => await _refreshTwitterUserStatusProcessor.ProcessAsync(x, ct));
-            var twitterUsersBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
-            var retrieveTweetsBlock = new TransformBlock<UserWithDataToSync[], UserWithDataToSync[]>(async x => await _retrieveTweetsProcessor.ProcessAsync(x, ct));
-            var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
-            var retrieveFollowersBlock = new TransformManyBlock<UserWithDataToSync[], UserWithDataToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct));
-            var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
-            var sendTweetsToFollowersBlock = new TransformBlock<UserWithDataToSync, UserWithDataToSync>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
-            var sendTweetsToFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
-            var saveProgressionBlock = new ActionBlock<UserWithDataToSync>(async x => await _saveProgressionProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
+            var retrieveTweetsBlock = new TransformBlock<UserWithDataToSync[], UserWithDataToSync[]>(async x =>  await _retrieveTweetsProcessor.ProcessAsync(x, ct), standardBlockOptions );
+            var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 2, CancellationToken = ct });
+           // var retrieveFollowersBlock = new TransformManyBlock<UserWithDataToSync[], UserWithDataToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { BoundedCapacity = 1 } );
+           // var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 500, CancellationToken = ct });
+            var sendTweetsToFollowersBlock = new ActionBlock<UserWithDataToSync[]>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), standardBlockOptions);
 
             // Link pipeline
-            twitterUserToRefreshBufferBlock.LinkTo(twitterUserToRefreshBlock, new DataflowLinkOptions { PropagateCompletion = true });
-            twitterUserToRefreshBlock.LinkTo(twitterUsersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
-            twitterUsersBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions { PropagateCompletion = true });
+            twitterUserToRefreshBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions { PropagateCompletion = true });
             retrieveTweetsBlock.LinkTo(retrieveTweetsBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
-            retrieveTweetsBufferBlock.LinkTo(retrieveFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
-            retrieveFollowersBlock.LinkTo(retrieveFollowersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
-            retrieveFollowersBufferBlock.LinkTo(sendTweetsToFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
-            sendTweetsToFollowersBlock.LinkTo(sendTweetsToFollowersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
-            sendTweetsToFollowersBufferBlock.LinkTo(saveProgressionBlock, new DataflowLinkOptions { PropagateCompletion = true });
+            retrieveTweetsBufferBlock.LinkTo(sendTweetsToFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
 
-            // Launch twitter user retriever
+            // Launch twitter user retriever after a little delay
+            // to give time for the Tweet cache to fill
+            await Task.Delay(30 * 1000, ct);
             var retrieveTwitterAccountsTask = _retrieveTwitterAccountsProcessor.GetTwitterUsersAsync(twitterUserToRefreshBufferBlock, ct);
 
             // Wait
-            await Task.WhenAny(new[] { retrieveTwitterAccountsTask, saveProgressionBlock.Completion });
+            await Task.WhenAny(new[] { retrieveTwitterAccountsTask, sendTweetsToFollowersBlock.Completion });
 
-            var ex = retrieveTwitterAccountsTask.IsFaulted ? retrieveTwitterAccountsTask.Exception : saveProgressionBlock.Completion.Exception;
+            var ex = retrieveTwitterAccountsTask.IsFaulted ? retrieveTwitterAccountsTask.Exception : sendTweetsToFollowersBlock.Completion.Exception;
             _logger.LogCritical(ex, "An error occurred, pipeline stopped");
         }
     }

src/BirdsiteLive.Pipeline/Tools/MaxUsersNumberProvider.cs

@@ -1,49 +0,0 @@
-using System.Threading.Tasks;
-using BirdsiteLive.Common.Settings;
-using BirdsiteLive.DAL.Contracts;
-
-namespace BirdsiteLive.Pipeline.Tools
-{
-    public interface IMaxUsersNumberProvider
-    {
-        Task<int> GetMaxUsersNumberAsync();
-    }
-
-    public class MaxUsersNumberProvider : IMaxUsersNumberProvider
-    {
-        private readonly InstanceSettings _instanceSettings;
-        private readonly ITwitterUserDal _twitterUserDal;
-        
-        private int _totalUsersCount = -1;
-        private int _warmUpIterations;
-        private const int WarmUpMaxCapacity = 200;
-
-        #region Ctor
-        public MaxUsersNumberProvider(InstanceSettings instanceSettings, ITwitterUserDal twitterUserDal)
-        {
-            _instanceSettings = instanceSettings;
-            _twitterUserDal = twitterUserDal;
-        }
-        #endregion
-
-        public async Task<int> GetMaxUsersNumberAsync()
-        {
-            // Init data
-            if (_totalUsersCount == -1) 
-            {
-                _totalUsersCount = await _twitterUserDal.GetTwitterUsersCountAsync();
-                _warmUpIterations = (int)(_totalUsersCount / (float)WarmUpMaxCapacity);
-            }
-
-            // Return if warm up ended
-            if (_warmUpIterations <= 0) return _instanceSettings.MaxUsersCapacity;
-
-            // Calculate warm up value
-            var maxUsers = _warmUpIterations > 0
-                ? WarmUpMaxCapacity
-                : _instanceSettings.MaxUsersCapacity;
-            _warmUpIterations--;
-            return maxUsers;
-        }
-    }
-}

src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj

@@ -1,16 +1,18 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="5.0.0" />
-    <PackageReference Include="TweetinviAPI" Version="4.0.3" />
+    <PackageReference Include="Microsoft.Extensions.Http" Version="5.0.0" />
+    <PackageReference Include="System.Threading.RateLimiting" Version="7.0.0" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\BirdsiteLive.Common\BirdsiteLive.Common.csproj" />
+    <ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
   </ItemGroup>
 
   <ItemGroup>

src/BirdsiteLive.Twitter/CachedTwitterService.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Text.Json;
+using System.Threading.Tasks;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Twitter.Models;
 using Microsoft.Extensions.Caching.Memory;
@@ -8,6 +10,8 @@ namespace BirdsiteLive.Twitter
     public interface ICachedTwitterUserService : ITwitterUserService
     {
         void PurgeUser(string username);
+        void AddUser(TwitterUser user);
+        bool UserIsCached(string username);
     }
 
     public class CachedTwitterUserService : ICachedTwitterUserService
@@ -18,11 +22,11 @@ namespace BirdsiteLive.Twitter
         private readonly MemoryCacheEntryOptions _cacheEntryOptions = new MemoryCacheEntryOptions()
             .SetSize(1)//Size amount
             //Priority on removing when reaching size limit (memory pressure)
-            .SetPriority(CacheItemPriority.High)
+            .SetPriority(CacheItemPriority.Low)
             // Keep in cache for this time, reset time if accessed.
-            .SetSlidingExpiration(TimeSpan.FromHours(24))
+            .SetSlidingExpiration(TimeSpan.FromMinutes(60))
             // Remove from cache after this time, regardless of sliding expiration
-            .SetAbsoluteExpiration(TimeSpan.FromDays(7));
+            .SetAbsoluteExpiration(TimeSpan.FromDays(1));
 
         #region Ctor
         public CachedTwitterUserService(ITwitterUserService twitterService, InstanceSettings settings)
@@ -36,15 +40,19 @@ namespace BirdsiteLive.Twitter
         }
         #endregion
 
-        public TwitterUser GetUser(string username)
+        public bool UserIsCached(string username)
         {
-            if (!_userCache.TryGetValue(username, out TwitterUser user))
+            return _userCache.TryGetValue(username, out _);
+        }
+        public async Task<TwitterUser> GetUserAsync(string username)
+        {
+            if (!_userCache.TryGetValue(username, out Task<TwitterUser> user))
             {
-                user = _twitterService.GetUser(username);
-                if(user != null) _userCache.Set(username, user, _cacheEntryOptions);
+                user = _twitterService.GetUserAsync(username);
+                await _userCache.Set(username, user, _cacheEntryOptions);
             }
 
-            return user;
+            return await user;
         }
 
         public bool IsUserApiRateLimited()
@@ -52,9 +60,18 @@ namespace BirdsiteLive.Twitter
             return _twitterService.IsUserApiRateLimited();
         }
 
+        public TwitterUser Extract(JsonElement result)
+        {
+            return _twitterService.Extract(result);
+        }
         public void PurgeUser(string username)
         {
             _userCache.Remove(username);
         }
+        public void AddUser(TwitterUser user)
+        {
+
+            _userCache.Set(user.Acct, user, _cacheEntryOptions);
+        }
     }
 }

src/BirdsiteLive.Twitter/CachedTwitterTweetService.cs

@@ -1,63 +0,0 @@
-using System;
-using BirdsiteLive.Twitter.Models;
-using Microsoft.Extensions.Caching.Memory;
-
-namespace BirdsiteLive.Twitter
-{
-    public interface ICachedTwitterTweetsService : ITwitterTweetsService
-    {
-        void PurgeTweet(long statusId);
-    }
-
-    public class CachedTwitterTweetsService : ICachedTwitterTweetsService
-    {
-        private readonly ITwitterTweetsService _twitterService;
-
-        private MemoryCache _tweetCache = new MemoryCache(new MemoryCacheOptions()
-        {
-            SizeLimit = 5000
-        });
-        private MemoryCacheEntryOptions _cacheEntryOptions = new MemoryCacheEntryOptions()
-            .SetSize(1)//Size amount
-                       //Priority on removing when reaching size limit (memory pressure)
-            .SetPriority(CacheItemPriority.High)
-            // Keep in cache for this time, reset time if accessed.
-            // We set this lower than a user's in case they delete this Tweet for some reason; we don't need that cached.
-            .SetSlidingExpiration(TimeSpan.FromHours(2))
-            // Remove from cache after this time, regardless of sliding expiration
-            .SetAbsoluteExpiration(TimeSpan.FromDays(7));
-
-        #region Ctor
-        public CachedTwitterTweetsService(ITwitterTweetsService twitterService)
-        {
-            _twitterService = twitterService;
-        }
-
-        public ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1)
-        {
-            // This sounds like it'd be silly to cache; pass this directly to TwitterService.
-            // Theoretically this shouldn't be called more than once every 15 min anyway?
-            return _twitterService.GetTimeline(username, nberTweets, fromTweetId);
-        }
-
-        public ExtractedTweet GetTweet(long statusId)
-        {
-            if(!_tweetCache.TryGetValue(statusId, out ExtractedTweet tweet))
-            {
-                tweet = _twitterService.GetTweet(statusId);
-
-                // Unlike with the user cache, save the null value anyway to prevent (quicker) API exhaustion.
-                // It's incredibly unlikely that a tweet with this ID is going to magickally appear within 2 hours.
-                _tweetCache.Set(statusId, tweet, _cacheEntryOptions); 
-            }
-
-            return tweet;
-        }
-        #endregion
-
-        public void PurgeTweet(long statusId)
-        {
-            _tweetCache.Remove(statusId);
-        }
-    }
-}

src/BirdsiteLive.Twitter/CachedTwitterTweetsService.cs

@@ -0,0 +1,64 @@
+using System;
+using System.Text.Json;
+using System.Threading.Tasks;
+using BirdsiteLive.Common.Settings;
+using BirdsiteLive.Twitter.Models;
+using Microsoft.Extensions.Caching.Memory;
+
+namespace BirdsiteLive.Twitter
+{
+    public interface ICachedTwitterTweetsService : ITwitterTweetsService
+    {
+        void SetTweet(long id, ExtractedTweet tweet);
+    }
+
+    public class CachedTwitterTweetsService : ICachedTwitterTweetsService
+    {
+        private readonly ITwitterTweetsService _twitterService;
+
+        private readonly MemoryCache _tweetCache;
+        private readonly MemoryCacheEntryOptions _cacheEntryOptions;
+
+        #region Ctor
+        public CachedTwitterTweetsService(ITwitterTweetsService twitterService, InstanceSettings settings)
+        {
+            _twitterService = twitterService;
+
+            _tweetCache = new MemoryCache(new MemoryCacheOptions()
+            {
+                SizeLimit = settings.TweetCacheCapacity,
+            });
+            _cacheEntryOptions = new MemoryCacheEntryOptions()
+                .SetSize(1)
+                //Priority on removing when reaching size limit (memory pressure)
+                .SetPriority(CacheItemPriority.Low)
+                // Keep in cache for this time, reset time if accessed.
+                .SetSlidingExpiration(TimeSpan.FromMinutes(60))
+                // Remove from cache after this time, regardless of sliding expiration
+                .SetAbsoluteExpiration(TimeSpan.FromDays(1));
+        }
+        #endregion
+
+        public async Task<ExtractedTweet[]> GetTimelineAsync(string username, long id)
+        {
+            var res = await _twitterService.GetTimelineAsync(username, id);
+            return res;
+        }
+        public async Task<ExtractedTweet> GetTweetAsync(long id)
+        {
+            if (!_tweetCache.TryGetValue(id, out Task<ExtractedTweet> tweet))
+            {
+                tweet = _twitterService.GetTweetAsync(id);
+                await _tweetCache.Set(id, tweet, _cacheEntryOptions);
+            }
+
+            return await tweet;
+        }
+
+        public void SetTweet(long id, ExtractedTweet tweet)
+        {
+
+            _tweetCache.Set(id, tweet, _cacheEntryOptions);
+        }
+    }
+}

src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs

@@ -1,199 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using BirdsiteLive.Common.Settings;
-using BirdsiteLive.Twitter.Models;
-using Tweetinvi.Models;
-using Tweetinvi.Models.Entities;
-
-namespace BirdsiteLive.Twitter.Extractors
-{
-    public interface ITweetExtractor
-    {
-        ExtractedTweet Extract(ITweet tweet);
-    }
-
-    public class TweetExtractor : ITweetExtractor
-    {
-        private readonly InstanceSettings _instanceSettings;
-
-        #region Ctor
-        public TweetExtractor(InstanceSettings instanceSettings)
-        {
-            this._instanceSettings = instanceSettings;
-        }
-        #endregion
-
-        public ExtractedTweet Extract(ITweet tweet)
-        {
-            var extractedTweet = new ExtractedTweet
-            {
-                Id = tweet.Id,
-                InReplyToStatusId = tweet.InReplyToStatusId,
-                InReplyToAccount = tweet.InReplyToScreenName,
-                MessageContent = ExtractMessage(tweet),
-                Media = ExtractMedia(tweet),
-                CreatedAt = tweet.CreatedAt.ToUniversalTime(),
-                IsReply = tweet.InReplyToUserId != null,
-                IsThread = tweet.InReplyToUserId != null && tweet.InReplyToUserId == tweet.CreatedBy.Id,
-                IsRetweet = tweet.IsRetweet || tweet.QuotedStatusId != null,
-                RetweetUrl = ExtractRetweetUrl(tweet),
-                IsSensitive = tweet.PossiblySensitive,
-                QuoteTweetUrl = tweet.QuotedStatusId != null ? "https://" + _instanceSettings.Domain + "/users/" + tweet.QuotedTweet.CreatedBy.ScreenName + "/statuses/" + tweet.QuotedStatusId : null,
-                CreatorName = tweet.CreatedBy.UserIdentifier.ScreenName
-            };
-
-            return extractedTweet;
-        }
-
-        private string ExtractRetweetUrl(ITweet tweet)
-        {
-            if (tweet.IsRetweet)
-            {
-                if (tweet.RetweetedTweet != null)
-                {
-                    var uri = new UriBuilder(tweet.RetweetedTweet.Url);
-                    uri.Host = _instanceSettings.TwitterDomain;
-
-                    return uri.Uri.ToString();
-                }
-                if (tweet.FullText.Contains("https://t.co/"))
-                {
-                    var retweetId = tweet.FullText.Split(new[] { "https://t.co/" }, StringSplitOptions.RemoveEmptyEntries).Last();
-                    return $"https://t.co/{retweetId}";
-                }
-            }
-
-            return null;
-        }
-
-        public string ExtractMessage(ITweet tweet)
-        {
-            var message = tweet.FullText;
-            var tweetUrls = tweet.Media.Select(x => x.URL).Distinct();
-            
-            if (tweet.IsRetweet && message.StartsWith("RT") && tweet.RetweetedTweet != null)
-            {
-                message = tweet.RetweetedTweet.FullText;
-                tweetUrls = tweet.RetweetedTweet.Media.Select(x => x.URL).Distinct();
-            }
-
-            foreach (var tweetUrl in tweetUrls)
-            {
-                if(tweet.IsRetweet)
-                    message = tweet.RetweetedTweet.FullText.Replace(tweetUrl, string.Empty).Trim();
-                else 
-                    message = message.Replace(tweetUrl, string.Empty).Trim();
-            }
-
-            if (tweet.QuotedTweet != null && ! _instanceSettings.EnableQuoteRT)
-            {
-                message = $"[Quote {{RT}}]{Environment.NewLine}{message}";
-            }
-
-            if (tweet.IsRetweet)
-            {
-                if (tweet.RetweetedTweet != null && !message.StartsWith("RT"))
-                    message = $"[{{RT}} @{tweet.RetweetedTweet.CreatedBy.ScreenName}]{Environment.NewLine}{message}";
-                else if (tweet.RetweetedTweet != null && message.StartsWith($"RT @{tweet.RetweetedTweet.CreatedBy.ScreenName}:"))
-                    message = message.Replace($"RT @{tweet.RetweetedTweet.CreatedBy.ScreenName}:", $"[{{RT}} @{tweet.RetweetedTweet.CreatedBy.ScreenName}]{Environment.NewLine}");
-                else
-                    message = message.Replace("RT", "[{{RT}}]");
-            }
-
-            // Expand URLs
-            foreach (var url in tweet.Urls.OrderByDescending(x => x.URL.Length))
-            {
-                // A bit of a hack
-                if (url.ExpandedURL == tweet.QuotedTweet?.Url && _instanceSettings.EnableQuoteRT)
-                {
-                    url.ExpandedURL = "";
-                } else
-                {
-                    var linkUri = new UriBuilder(url.ExpandedURL);
-
-                    if (linkUri.Host == "twitter.com")
-                    {
-                        linkUri.Host = _instanceSettings.TwitterDomain;
-                        url.ExpandedURL = linkUri.Uri.ToString();
-                    }
-                }
-
-                message = message.Replace(url.URL, url.ExpandedURL);
-            }
-
-            // Hack
-
-            return message;
-        }
-
-        public ExtractedMedia[] ExtractMedia(ITweet tweet)
-        {
-            var media = tweet.Media;
-            if (tweet.IsRetweet && tweet.RetweetedTweet != null)
-                media = tweet.RetweetedTweet.Media;
-
-            var result = new List<ExtractedMedia>();
-            foreach (var m in media)
-            {
-                var mediaUrl = GetMediaUrl(m);
-                var mediaType = GetMediaType(m.MediaType, mediaUrl);
-                if (mediaType == null) continue;
-
-
-                var att = new ExtractedMedia
-                {
-                    MediaType = mediaType,
-                    Url = mediaUrl
-                };
-                result.Add(att);
-            }
-
-            return result.ToArray();
-        }
-
-        public string GetMediaUrl(IMediaEntity media)
-        {
-            switch (media.MediaType)
-            {
-                case "photo": return media.MediaURLHttps;
-                case "animated_gif": return media.VideoDetails.Variants[0].URL;
-                case "video": return media.VideoDetails.Variants.OrderByDescending(x => x.Bitrate).First().URL;
-                default: return null;
-            }
-        }
-
-        public string GetMediaType(string mediaType, string mediaUrl)
-        {
-            switch (mediaType)
-            {
-                case "photo":
-                    var pExt = Path.GetExtension(mediaUrl);
-                    switch (pExt)
-                    {
-                        case ".jpg":
-                        case ".jpeg":
-                            return "image/jpeg";
-                        case ".png":
-                            return "image/png";
-                    }
-                    return null;
-
-                case "animated_gif":
-                    var vExt = Path.GetExtension(mediaUrl);
-                    switch (vExt)
-                    {
-                        case ".gif":
-                            return "image/gif";
-                        case ".mp4":
-                            return "video/mp4";
-                    }
-                    return "image/gif";
-                case "video":
-                    return "video/mp4";
-            }
-            return null;
-        }
-    }
-}

src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs

@@ -15,8 +15,8 @@ namespace BirdsiteLive.Twitter.Models
         public bool IsThread { get; set; }
         public bool IsRetweet { get; set; }
         public string RetweetUrl { get; set; }
-        public bool IsSensitive { get; set; }
-        public string QuoteTweetUrl { get; set; }
-        public string CreatorName { get; set; }
+        public long RetweetId { get; set; }
+        public TwitterUser OriginalAuthor { get; set; }
+		public string QuoteTweetUrl { get; set; }
     }
 }

src/BirdsiteLive.Twitter/Models/TwitterUser.cs

@@ -11,7 +11,5 @@
         public string Acct { get; set; }
         public string ProfileBannerURL { get; set; }
         public bool Protected { get; set; }
-        public bool Sensitive { get; set; }
-        public bool Verified { get; set; }
     }
 }

src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs

@@ -1,64 +1,139 @@
 using System;
 using System.Threading;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using System.Linq;
 using System.Threading.Tasks;
 using BirdsiteLive.Common.Settings;
 using Microsoft.Extensions.Logging;
-using Tweetinvi;
+using System.Net.Http;
+using System.Net;
+using System.Net.Http.Headers;
+using System.Text;
+using System.Text.Json;
+using System.Threading.RateLimiting;
 
 namespace BirdsiteLive.Twitter.Tools
 {
     public interface ITwitterAuthenticationInitializer
     {
-        void EnsureAuthenticationIsInitialized();
+        Task<HttpClient> MakeHttpClient();
+        HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint, bool addToken);
+        Task RefreshClient(HttpRequestMessage client);
     }
 
     public class TwitterAuthenticationInitializer : ITwitterAuthenticationInitializer
     {
-        private readonly TwitterSettings _settings;
         private readonly ILogger<TwitterAuthenticationInitializer> _logger;
         private static bool _initialized;
-        private readonly SemaphoreSlim _semaphoregate = new SemaphoreSlim(1);
+        private readonly IHttpClientFactory _httpClientFactory;
+        private ConcurrentDictionary<String, String> _token2 = new ConcurrentDictionary<string, string>();
+        static Random rnd = new Random();
+        private RateLimiter _rateLimiter;
+        private const int _targetClients = 3;
+        private InstanceSettings _instanceSettings;
+        private readonly (string, string)[] _apiKeys = new[]
+        {
+            ("IQKbtAYlXLripLGPWd0HUA", "GgDYlkSvaPxGxC4X8liwpUoqKwwr3lCADbz8A7ADU"), // iPhone
+            ("3nVuSoBZnx6U4vzUxf5w", "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"), // Android
+            ("CjulERsDeqhhjSme66ECg", "IQWdVyqFxghAtURHGeGiWAsmCAGmdW3WmbEx6Hck"), // iPad
+            ("3rJOl1ODzm9yZy63FACdg", "5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8"), // Mac
+        };
+        public String BearerToken { 
+            get
+            {
+                return _instanceSettings.TwitterBearerToken;
+            }
+        }
 
         #region Ctor
-        public TwitterAuthenticationInitializer(TwitterSettings settings, ILogger<TwitterAuthenticationInitializer> logger)
+        public TwitterAuthenticationInitializer(IHttpClientFactory httpClientFactory, InstanceSettings settings, ILogger<TwitterAuthenticationInitializer> logger)
         {
-            _settings = settings;
             _logger = logger;
+            _instanceSettings = settings;
+            _httpClientFactory = httpClientFactory;
         }
         #endregion
 
-        public void EnsureAuthenticationIsInitialized()
+        private async Task<string> GenerateBearerToken()
         {
-            if (_initialized) return;
-            _semaphoregate.Wait();
-           
-            try
+            var httpClient = _httpClientFactory.CreateClient();
+            using (var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.twitter.com/oauth2/token?grant_type=client_credentials"))
             {
-                if (_initialized) return;
-                InitTwitterCredentials();
-            }
-            finally
-            {
-                _semaphoregate.Release();
+                int r = rnd.Next(_apiKeys.Length);
+                var (login, password) = _apiKeys[r];
+                var authValue = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{login}:{password}")));
+                request.Headers.Authorization = authValue;
+
+                var httpResponse = await httpClient.SendAsync(request);
+
+                var c = await httpResponse.Content.ReadAsStringAsync();
+                httpResponse.EnsureSuccessStatusCode();
+                var doc = JsonDocument.Parse(c);
+                var token = doc.RootElement.GetProperty("access_token").GetString();
+                return token;
             }
+            
         }
 
-        private void InitTwitterCredentials()
+
+        public async Task RefreshClient(HttpRequestMessage req)
         {
-            for (;;)
+            string token = req.Headers.GetValues("x-guest-token").First();
+
+            _token2.TryRemove(token, out _);
+
+            await RefreshCred();
+            await Task.Delay(1000);
+            await RefreshCred();
+        }
+
+        private async Task RefreshCred()
+        {
+            (string bearer, string guest) = await GetCred();
+            _token2.TryAdd(guest, bearer);
+        }
+
+        private async Task<(string, string)> GetCred()
+        {
+            string token;
+            var httpClient = _httpClientFactory.CreateClient();
+            string bearer = await GenerateBearerToken();
+            using (var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.twitter.com/1.1/guest/activate.json"))
             {
-                try
-                {
-                    Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true);
-                    _initialized = true;
-                    return;
-                }
-                catch (Exception e)
-                {
-                    _logger.LogError(e, "Twitter Authentication Failed");
-                    Thread.Sleep(250);
-                }
+                request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer); 
+
+                var httpResponse = await httpClient.SendAsync(request);
+
+                var c = await httpResponse.Content.ReadAsStringAsync();
+                httpResponse.EnsureSuccessStatusCode();
+                var doc = JsonDocument.Parse(c);
+                token = doc.RootElement.GetProperty("guest_token").GetString();
             }
+
+            return (bearer, token);
+
+        }
+
+        public async Task<HttpClient> MakeHttpClient()
+        {
+            if (_token2.Count < _targetClients)
+                await RefreshCred();
+            return _httpClientFactory.CreateClient();
+        }
+        public HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint, bool addToken)
+        {
+            var request = new HttpRequestMessage(m, endpoint);
+            //(string bearer, string token) = _tokens[r];
+            (string token, string bearer) = _token2.MaxBy(x => rnd.Next());
+            request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer); 
+            request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");
+            request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes");
+            if (addToken)
+                request.Headers.TryAddWithoutValidation("x-guest-token", token);
+            //request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");
+            //request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes");
+            return request;
         }
     }
 }

src/BirdsiteLive.Twitter/TwitterTweetsService.cs

@@ -1,93 +1,372 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
+using System.Net.Http;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
 using BirdsiteLive.Common.Settings;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Statistics.Domain;
-using BirdsiteLive.Twitter.Extractors;
 using BirdsiteLive.Twitter.Models;
 using BirdsiteLive.Twitter.Tools;
 using Microsoft.Extensions.Logging;
-using Tweetinvi;
-using Tweetinvi.Models;
-using Tweetinvi.Parameters;
 
 namespace BirdsiteLive.Twitter
 {
     public interface ITwitterTweetsService
     {
-        ExtractedTweet GetTweet(long statusId);
-        ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1);
+        Task<ExtractedTweet> GetTweetAsync(long statusId);
+        Task<ExtractedTweet[]> GetTimelineAsync(string username, long fromTweetId = -1);
     }
 
     public class TwitterTweetsService : ITwitterTweetsService
     {
         private readonly ITwitterAuthenticationInitializer _twitterAuthenticationInitializer;
-        private readonly ITweetExtractor _tweetExtractor;
         private readonly ITwitterStatisticsHandler _statisticsHandler;
-        private readonly ITwitterUserService _twitterUserService;
+        private readonly ICachedTwitterUserService _twitterUserService;
+        private readonly ITwitterUserDal _twitterUserDal;
         private readonly ILogger<TwitterTweetsService> _logger;
+        private readonly InstanceSettings _instanceSettings;
 
         #region Ctor
-        public TwitterTweetsService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITweetExtractor tweetExtractor, ITwitterStatisticsHandler statisticsHandler, ITwitterUserService twitterUserService, ILogger<TwitterTweetsService> logger)
+        public TwitterTweetsService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITwitterStatisticsHandler statisticsHandler, ICachedTwitterUserService twitterUserService, ITwitterUserDal twitterUserDal, InstanceSettings instanceSettings, ILogger<TwitterTweetsService> logger)
         {
             _twitterAuthenticationInitializer = twitterAuthenticationInitializer;
-            _tweetExtractor = tweetExtractor;
             _statisticsHandler = statisticsHandler;
             _twitterUserService = twitterUserService;
+            _twitterUserDal = twitterUserDal;
+            _instanceSettings = instanceSettings;
             _logger = logger;
         }
         #endregion
 
-        public ExtractedTweet GetTweet(long statusId)
+
+        public async Task<ExtractedTweet> GetTweetAsync(long statusId)
         {
+
+            var client = await _twitterAuthenticationInitializer.MakeHttpClient();
+
+
+            string reqURL =
+                "https://api.twitter.com/graphql/XjlydVWHFIDaAUny86oh2g/TweetDetail?variables=%7B%22focalTweetId%22%3A%22"
+                + statusId +
+                "%22,%22with_rux_injections%22%3Atrue,%22includePromotedContent%22%3Afalse,%22withCommunity%22%3Afalse,%22withQuickPromoteEligibilityTweetFields%22%3Afalse,%22withBirdwatchNotes%22%3Afalse,%22withSuperFollowsUserFields%22%3Afalse,%22withDownvotePerspective%22%3Afalse,%22withReactionsMetadata%22%3Afalse,%22withReactionsPerspective%22%3Afalse,%22withSuperFollowsTweetFields%22%3Afalse,%22withVoice%22%3Atrue,%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue,%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue,%22verified_phone_label_enabled%22%3Afalse,%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue,%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse,%22tweetypie_unmention_optimization_enabled%22%3Atrue,%22vibe_api_enabled%22%3Atrue,%22responsive_web_edit_tweet_api_enabled%22%3Atrue,%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Afalse,%22view_counts_everywhere_api_enabled%22%3Atrue,%22longform_notetweets_consumption_enabled%22%3Atrue,%22tweet_awards_web_tipping_enabled%22%3Afalse,%22freedom_of_speech_not_reach_fetch_enabled%22%3Afalse,%22standardized_nudges_misinfo%22%3Atrue,%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse,%22interactive_text_enabled%22%3Atrue,%22responsive_web_text_conversations_enabled%22%3Afalse,%22longform_notetweets_richtext_consumption_enabled%22%3Afalse,%22responsive_web_enhance_cards_enabled%22%3Atrue%7D";
+            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL, true);
             try
             {
-                _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized();
-                ExceptionHandler.SwallowWebExceptions = false;
-                TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended;
+                JsonDocument tweet;
+                var httpResponse = await client.SendAsync(request);
+                httpResponse.EnsureSuccessStatusCode();
+                var c = await httpResponse.Content.ReadAsStringAsync();
+                tweet = JsonDocument.Parse(c);
 
-                var tweet = Tweet.GetTweet(statusId);
-                _statisticsHandler.CalledTweetApi();
-                if (tweet == null) return null; //TODO: test this
-                return _tweetExtractor.Extract(tweet);
+
+                var timeline = tweet.RootElement.GetProperty("data").GetProperty("threaded_conversation_with_injections_v2")
+                    .GetProperty("instructions").EnumerateArray().First().GetProperty("entries").EnumerateArray();
+
+                var tweetInDoc = timeline.Where(x => x.GetProperty("entryId").GetString() == "tweet-" + statusId)
+                    .ToArray().First();
+                return await Extract( tweetInDoc );
             }
             catch (Exception e)
             {
                 _logger.LogError(e, "Error retrieving tweet {TweetId}", statusId);
+                await _twitterAuthenticationInitializer.RefreshClient(request);
                 return null;
             }
         }
 
-        public ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1)
+        public async Task<ExtractedTweet[]> GetTimelineAsync(string username, long fromTweetId = -1)
         {
-            var tweets = new List<ITweet>();
 
-            _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized();
-            ExceptionHandler.SwallowWebExceptions = false;
-            TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended;
+            var client = await _twitterAuthenticationInitializer.MakeHttpClient();
 
-            var user = _twitterUserService.GetUser(username);
-            if (user == null || user.Protected) return new ExtractedTweet[0];
-
-            if (fromTweetId == -1)
+            long userId;
+            SyncTwitterUser user = await _twitterUserDal.GetTwitterUserAsync(username);
+            if (user.TwitterUserId == default) 
             {
-                var timeline = Timeline.GetUserTimeline(user.Id, nberTweets);
-                _statisticsHandler.CalledTimelineApi();
-                if (timeline != null) tweets.AddRange(timeline);
+                var user2 = await _twitterUserService.GetUserAsync(username);
+                userId = user2.Id;
+                await _twitterUserDal.UpdateTwitterUserIdAsync(username, user2.Id);
             }
-            else
+            else 
             {
-                var timelineRequestParameters = new UserTimelineParameters
+                userId = user.TwitterUserId;
+            }
+
+
+            var reqURL =
+                "https://api.twitter.com/graphql/pNl8WjKAvaegIoVH--FuoQ/UserTweetsAndReplies?variables=%7B%22userId%22%3A%22" +
+                userId + "%22,%22count%22%3A40,%22includePromotedContent%22%3Atrue,%22withCommunity%22%3Atrue,%22withSuperFollowsUserFields%22%3Atrue,%22withDownvotePerspective%22%3Afalse,%22withReactionsMetadata%22%3Afalse,%22withReactionsPerspective%22%3Afalse,%22withSuperFollowsTweetFields%22%3Atrue,%22withVoice%22%3Atrue,%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue,%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue,%22verified_phone_label_enabled%22%3Afalse,%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue,%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse,%22tweetypie_unmention_optimization_enabled%22%3Atrue,%22vibe_api_enabled%22%3Atrue,%22responsive_web_edit_tweet_api_enabled%22%3Atrue,%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue,%22view_counts_everywhere_api_enabled%22%3Atrue,%22longform_notetweets_consumption_enabled%22%3Atrue,%22tweet_awards_web_tipping_enabled%22%3Afalse,%22freedom_of_speech_not_reach_fetch_enabled%22%3Afalse,%22standardized_nudges_misinfo%22%3Atrue,%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse,%22interactive_text_enabled%22%3Atrue,%22responsive_web_text_conversations_enabled%22%3Afalse,%22longform_notetweets_richtext_consumption_enabled%22%3Afalse,%22responsive_web_enhance_cards_enabled%22%3Afalse%7D";
+            JsonDocument results;
+            List<ExtractedTweet> extractedTweets = new List<ExtractedTweet>();
+            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL, true);
+            try
+            {
+
+                var httpResponse = await client.SendAsync(request);
+                httpResponse.EnsureSuccessStatusCode();
+                var c = await httpResponse.Content.ReadAsStringAsync();
+                results = JsonDocument.Parse(c);
+
+                _statisticsHandler.CalledTweetApi();
+            }
+            catch (HttpRequestException e)
+            {
+                _logger.LogError(e, "Error retrieving timeline of {Username}; refreshing client", username);
+                await _twitterAuthenticationInitializer.RefreshClient(request);
+                return null;
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, "Error retrieving timeline ", username);
+                return null;
+            }
+
+            var timeline = results.RootElement.GetProperty("data").GetProperty("user").GetProperty("result")
+                .GetProperty("timeline_v2").GetProperty("timeline").GetProperty("instructions").EnumerateArray();
+
+            foreach (JsonElement timelineElement in timeline) 
+            {
+                if (timelineElement.GetProperty("type").GetString() != "TimelineAddEntries")
+                    continue;
+
+                
+                foreach (JsonElement tweet in timelineElement.GetProperty("entries").EnumerateArray())
                 {
-                    SinceId = fromTweetId,
-                    MaximumNumberOfTweetsToRetrieve = nberTweets
-                };
-                var timeline = Timeline.GetUserTimeline(user.Id, timelineRequestParameters);
-                _statisticsHandler.CalledTimelineApi();
-                if (timeline != null) tweets.AddRange(timeline);
+                    if (tweet.GetProperty("content").GetProperty("entryType").GetString() != "TimelineTimelineItem")
+                        continue;
+                    
+                    try 
+                    {
+                        JsonElement userDoc = tweet.GetProperty("content").GetProperty("itemContent")
+                                .GetProperty("tweet_results").GetProperty("core").GetProperty("user_results");
+
+                        TwitterUser tweetUser = _twitterUserService.Extract(userDoc);
+                        _twitterUserService.AddUser(tweetUser);
+                    }
+                    catch (Exception _)
+                    {}
+
+                    try 
+                    {   
+                        var extractedTweet = await Extract(tweet);
+
+                        if (extractedTweet.Id == fromTweetId)
+                            break;
+
+                        extractedTweets.Add(extractedTweet);
+                    }
+                    catch (Exception e)
+                    {
+                        _logger.LogError("Tried getting timeline from user " + username + ", but got error: \n" +
+                                         e.Message + e.StackTrace + e.Source);
+
+                    }
+
+                }
             }
 
-            return tweets.Select(_tweetExtractor.Extract).ToArray();
+            return extractedTweets.ToArray();
+        }
+
+        private async Task<ExtractedTweet> Extract(JsonElement tweet)
+        {
+
+            JsonElement retweet;
+            TwitterUser OriginalAuthor;
+            JsonElement inReplyToPostIdElement;
+            JsonElement inReplyToUserElement;
+            string inReplyToUser = null;
+            long? inReplyToPostId = null;
+            long retweetId = default;
+
+            string userName = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("core").GetProperty("user_results")
+                    .GetProperty("result").GetProperty("legacy").GetProperty("screen_name").GetString();
+
+            bool isReply = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .TryGetProperty("in_reply_to_status_id_str", out inReplyToPostIdElement);
+            tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .TryGetProperty("in_reply_to_screen_name", out inReplyToUserElement);
+            if (isReply) 
+            {
+                inReplyToPostId = Int64.Parse(inReplyToPostIdElement.GetString());
+                inReplyToUser = inReplyToUserElement.GetString();
+            }
+            bool isRetweet = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .TryGetProperty("retweeted_status_result", out retweet);
+            string MessageContent;
+            if (!isRetweet)
+            {
+                MessageContent = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .GetProperty("full_text").GetString();
+                bool isNote = tweet.GetProperty("content").GetProperty("itemContent")
+                        .GetProperty("tweet_results").GetProperty("result")
+                        .TryGetProperty("note_tweet", out var note);
+                if (isNote)
+                {
+                    MessageContent = note.GetProperty("note_tweet_results").GetProperty("result")
+                        .GetProperty("text").GetString();
+                }
+                OriginalAuthor = null;
+            }
+            else 
+            {
+                MessageContent = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .GetProperty("retweeted_status_result").GetProperty("result")
+                    .GetProperty("legacy").GetProperty("full_text").GetString();
+                bool isNote = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .GetProperty("retweeted_status_result").GetProperty("result")
+                    .TryGetProperty("note_tweet", out var note);
+                if (isNote)
+                {
+                    MessageContent = note.GetProperty("note_tweet_results").GetProperty("result")
+                        .GetProperty("text").GetString();
+                }
+                string OriginalAuthorUsername = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .GetProperty("retweeted_status_result").GetProperty("result")
+                    .GetProperty("core").GetProperty("user_results").GetProperty("result")
+                    .GetProperty("legacy").GetProperty("screen_name").GetString();
+                OriginalAuthor = await _twitterUserService.GetUserAsync(OriginalAuthorUsername);
+                retweetId = Int64.Parse(tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .GetProperty("retweeted_status_result").GetProperty("result")
+                    .GetProperty("rest_id").GetString());
+            }
+
+            string creationTime = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .GetProperty("created_at").GetString().Replace(" +0000", "");
+
+            JsonElement extendedEntities;
+            bool hasMedia = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .TryGetProperty("extended_entities", out extendedEntities);
+
+            JsonElement.ArrayEnumerator urls = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .GetProperty("entities").GetProperty("urls").EnumerateArray();
+            foreach (JsonElement url in urls)
+            {
+                string tco = url.GetProperty("url").GetString();
+                string goodUrl = url.GetProperty("expanded_url").GetString();
+                MessageContent = MessageContent.Replace(tco, goodUrl);
+            }
+            
+            List<ExtractedMedia> Media = new List<ExtractedMedia>();
+            if (hasMedia) 
+            {
+                foreach (JsonElement media in extendedEntities.GetProperty("media").EnumerateArray())
+                {
+                    var type = media.GetProperty("type").GetString();
+                    string url = "";
+                    if (type == "video" || type == "animated_gif")
+                    {
+                        var bitrate = -1;
+                        foreach (JsonElement v in media.GetProperty("video_info").GetProperty("variants").EnumerateArray())
+                        {
+                            if (v.GetProperty("content_type").GetString() !=  "video/mp4")
+                                continue;
+                            int vBitrate = v.GetProperty("bitrate").GetInt32();
+                            if (vBitrate > bitrate)
+                            {
+                                bitrate = vBitrate;
+                                url = v.GetProperty("url").GetString();
+                            }
+                        }
+                    }
+                    else 
+                    {
+                        url = media.GetProperty("media_url_https").GetString();
+                    }
+                    var m = new ExtractedMedia
+                    {
+                        MediaType = GetMediaType(type, media.GetProperty("media_url_https").GetString()),
+                        Url = url,
+                    };
+                    Media.Add(m);
+
+                    MessageContent = MessageContent.Replace(media.GetProperty("url").GetString(), "");
+                }
+            }
+
+            bool isQuoteTweet = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                    .GetProperty("is_quote_status").GetBoolean();
+
+			string quoteTweetLink = "";
+			if (isQuoteTweet) 
+            {
+
+                quoteTweetLink = tweet.GetProperty("content").GetProperty("itemContent")
+                        .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                        .GetProperty("quoted_status_permalink").GetProperty("expanded").GetString();
+                quoteTweetLink = quoteTweetLink.Replace("https://twitter.com/", $"https://{_instanceSettings.Domain}/users/");
+                quoteTweetLink = quoteTweetLink.Replace("/status/", "/statuses/");
+            }
+            var extractedTweet = new ExtractedTweet
+            {
+                Id = Int64.Parse(tweet.GetProperty("entryId").GetString().Replace("tweet-", "")),
+                InReplyToStatusId = inReplyToPostId,
+                InReplyToAccount = inReplyToUser,
+                MessageContent = MessageContent.Trim(),
+                CreatedAt = DateTime.ParseExact(creationTime, "ddd MMM dd HH:mm:ss yyyy", System.Globalization.CultureInfo.InvariantCulture),
+                IsReply = isReply,
+                IsThread = userName == inReplyToUser,
+                IsRetweet = isRetweet,
+                Media = Media.Count() == 0 ? null : Media.ToArray(),
+                RetweetUrl = "https://t.co/123",
+                RetweetId = retweetId,
+                OriginalAuthor = OriginalAuthor,
+            };
+
+            if (isQuoteTweet) extractedTweet.QuoteTweetUrl = quoteTweetLink;
+
+			return extractedTweet;
+         
+        }
+        private string GetMediaType(string mediaType, string mediaUrl)
+        {
+            switch (mediaType)
+            {
+                case "photo":
+                    var pExt = Path.GetExtension(mediaUrl);
+                    switch (pExt)
+                    {
+                        case ".jpg":
+                        case ".jpeg":
+                            return "image/jpeg";
+                        case ".png":
+                            return "image/png";
+                    }
+                    return null;
+
+                case "animated_gif":
+                    var vExt = Path.GetExtension(mediaUrl);
+                    switch (vExt)
+                    {
+                        case ".gif":
+                            return "image/gif";
+                        case ".mp4":
+                            return "video/mp4";
+                    }
+                    return "image/gif";
+                case "video":
+                    return "video/mp4";
+            }
+            return null;
         }
     }
-}
+}

src/BirdsiteLive.Twitter/TwitterUserService.cs

@@ -1,19 +1,20 @@
 using System;
 using System.Linq;
+using System.Net.Http;
+using System.Text.Json;
+using System.Threading.Tasks;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Statistics.Domain;
 using BirdsiteLive.Twitter.Models;
 using BirdsiteLive.Twitter.Tools;
 using Microsoft.Extensions.Logging;
-using Tweetinvi;
-using Tweetinvi.Exceptions;
-using Tweetinvi.Models;
 
 namespace BirdsiteLive.Twitter
 {
     public interface ITwitterUserService
     {
-        TwitterUser GetUser(string username);
+        Task<TwitterUser> GetUserAsync(string username);
+        TwitterUser Extract (JsonElement result);
         bool IsUserApiRateLimited();
     }
 
@@ -22,6 +23,8 @@ namespace BirdsiteLive.Twitter
         private readonly ITwitterAuthenticationInitializer _twitterAuthenticationInitializer;
         private readonly ITwitterStatisticsHandler _statisticsHandler;
         private readonly ILogger<TwitterUserService> _logger;
+        
+        private readonly string endpoint = "https://twitter.com/i/api/graphql/4LB4fkCe3RDLDmOEEYtueg/UserByScreenName?variables=%7B%22screen_name%22%3A%22elonmusk%22%2C%22withSafetyModeUserFields%22%3Atrue%2C%22withSuperFollowsUserFields%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22responsive_web_twitter_blue_new_verification_copy_is_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D";
 
         #region Ctor
         public TwitterUserService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITwitterStatisticsHandler statisticsHandler, ILogger<TwitterUserService> logger)
@@ -32,39 +35,44 @@ namespace BirdsiteLive.Twitter
         }
         #endregion
 
-        public TwitterUser GetUser(string username)
+        public async Task<TwitterUser> GetUserAsync(string username)
         {
-            //Check if API is saturated 
-            if (IsUserApiRateLimited()) throw new RateLimitExceededException();
 
-            //Proceed to account retrieval
-            _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized();
-            ExceptionHandler.SwallowWebExceptions = false;
-            RateLimit.RateLimitTrackerMode = RateLimitTrackerMode.TrackOnly;
-
-            IUser user;
+            JsonDocument res;
+            var client = await _twitterAuthenticationInitializer.MakeHttpClient();
+            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), endpoint.Replace("elonmusk", username), true);
             try
             {
-                user = User.GetUserFromScreenName(username);
+
+                var httpResponse = await client.SendAsync(request);
+                httpResponse.EnsureSuccessStatusCode();
+
+                var c = await httpResponse.Content.ReadAsStringAsync();
+                res = JsonDocument.Parse(c);
+                var result = res.RootElement.GetProperty("data").GetProperty("user").GetProperty("result");
+                return Extract(result);
             }
-            catch (TwitterException e)
+            catch (System.Collections.Generic.KeyNotFoundException)
             {
-                if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("User has been suspended".ToLowerInvariant())))
-                {
-                    throw new UserHasBeenSuspendedException();
-                }
-                else if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("User not found".ToLowerInvariant())))
-                {
-                    throw new UserNotFoundException();
-                }
-                else if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("Rate limit exceeded".ToLowerInvariant())))
-                {
-                    throw new RateLimitExceededException();
-                }
-                else
-                {
-                    throw;
-                }
+                throw new UserNotFoundException();
+                //if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("User has been suspended".ToLowerInvariant())))
+                //{
+                //    throw new UserHasBeenSuspendedException();
+                //}
+                //else if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("User not found".ToLowerInvariant())))
+                //{
+                //    throw new UserNotFoundException();
+                //}
+                //else
+                //{
+                //    throw;
+                //}
+            }
+            catch (HttpRequestException e)
+            {
+                _logger.LogError(e, "Error retrieving user {Username}, Refreshing client", username);
+                await _twitterAuthenticationInitializer.RefreshClient(request);
+                return null;
             }
             catch (Exception e)
             {
@@ -77,50 +85,39 @@ namespace BirdsiteLive.Twitter
             }
 
             // Expand URLs
-            var description = user.Description;
-            foreach (var descriptionUrl in user.Entities?.Description?.Urls?.OrderByDescending(x => x.URL.Length))
-                description = description.Replace(descriptionUrl.URL, descriptionUrl.ExpandedURL);
+            //var description = user.Description;
+            //foreach (var descriptionUrl in user.Entities?.Description?.Urls?.OrderByDescending(x => x.URL.Length))
+            //    description = description.Replace(descriptionUrl.URL, descriptionUrl.ExpandedURL);
+
+        }
+
+        public TwitterUser Extract(JsonElement result)
+        {
+            string profileBannerURL = null;
+            JsonElement profileBannerURLObject;
+            if (result.GetProperty("legacy").TryGetProperty("profile_banner_url", out profileBannerURLObject))
+            {
+                profileBannerURL = profileBannerURLObject.GetString();
+            }
 
             return new TwitterUser
             {
-                Id = user.Id,
-                Acct = username,
-                Name = user.Name,
-                Description = description,
-                Url = $"https://twitter.com/{username}",
-                ProfileImageUrl = user.ProfileImageUrlFullSize.Replace("http://", "https://"),
-                ProfileBackgroundImageUrl = user.ProfileBackgroundImageUrlHttps,
-                ProfileBannerURL = user.ProfileBannerURL,
-                Protected = user.Protected,
-                Verified = user.Verified
+                Id = long.Parse(result.GetProperty("rest_id").GetString()),
+                Acct = result.GetProperty("legacy").GetProperty("screen_name").GetString(), 
+                Name =  result.GetProperty("legacy").GetProperty("name").GetString(), //res.RootElement.GetProperty("data").GetProperty("name").GetString(),
+                Description =  "", //res.RootElement.GetProperty("data").GetProperty("description").GetString(),
+                Url =  "", //res.RootElement.GetProperty("data").GetProperty("url").GetString(),
+                ProfileImageUrl =  result.GetProperty("legacy").GetProperty("profile_image_url_https").GetString().Replace("normal", "400x400"), 
+                ProfileBackgroundImageUrl =  profileBannerURL,
+                ProfileBannerURL = profileBannerURL,
+                Protected = false, //res.RootElement.GetProperty("data").GetProperty("protected").GetBoolean(), 
             };
+
         }
 
         public bool IsUserApiRateLimited()
         {
-            // Retrieve limit from tooling
-            _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized();
-            ExceptionHandler.SwallowWebExceptions = false;
-            RateLimit.RateLimitTrackerMode = RateLimitTrackerMode.TrackOnly;
-
-            try
-            {
-                var queryRateLimits = RateLimit.GetQueryRateLimit("https://api.twitter.com/1.1/users/show.json?screen_name=mastodon");
-
-                if (queryRateLimits != null)
-                {
-                    return queryRateLimits.Remaining <= 0;
-                }
-            }
-            catch (Exception e)
-            {
-                _logger.LogError(e, "Error retrieving rate limits");
-            }
-
-            // Fallback
-            var currentCalls = _statisticsHandler.GetCurrentUserCalls();
-            var maxCalls = _statisticsHandler.GetStatistics().UserCallsMax;
-            return currentCalls >= maxCalls;
+            return false;
         }
     }
 }

src/BirdsiteLive.sln

@@ -47,9 +47,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Moderation.Tes
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Common.Tests", "Tests\BirdsiteLive.Common.Tests\BirdsiteLive.Common.Tests.csproj", "{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BSLManager", "BSLManager\BSLManager.csproj", "{4A84D351-E91B-4E58-8E20-211F0F4991D7}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BSLManager.Tests", "Tests\BSLManager.Tests\BSLManager.Tests.csproj", "{D4457271-620E-465A-B08E-7FC63C99A2F6}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdsiteLive.Twitter.Tests", "Tests\BirdsiteLive.Twitter.Tests\BirdsiteLive.Twitter.Tests.csproj", "{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -129,14 +127,10 @@ Global
 		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Release|Any CPU.Build.0 = Release|Any CPU
-		{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Release|Any CPU.Build.0 = Release|Any CPU
-		{D4457271-620E-465A-B08E-7FC63C99A2F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{D4457271-620E-465A-B08E-7FC63C99A2F6}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{D4457271-620E-465A-B08E-7FC63C99A2F6}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{D4457271-620E-465A-B08E-7FC63C99A2F6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -159,7 +153,7 @@ Global
 		{4BE541AC-8A93-4FA3-98AC-956CC2D5B748} = {DA3C160C-4811-4E26-A5AD-42B81FAF2D7C}
 		{0A311BF3-4FD9-4303-940A-A3778890561C} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
 		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
-		{D4457271-620E-465A-B08E-7FC63C99A2F6} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
+		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {69E8DCAD-4C37-4010-858F-5F94E6FBABCE}

src/BirdsiteLive/BirdsiteLive.csproj

@@ -1,18 +1,17 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
     <UserSecretsId>d21486de-a812-47eb-a419-05682bb68856</UserSecretsId>
     <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
-    <Version>0.22.0</Version>
+    <Version>1.0</Version>
+    <ContainerImageName>cloutier/bird.makeup</ContainerImageName>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Lamar.Microsoft.DependencyInjection" Version="12.0.0" />
-    <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.21.0" />
-    <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.18.1" />
-    <PackageReference Include="Microsoft.VisualStudio.Web.BrowserLink" Version="2.2.0" />
-    <PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="3.1.5" />
+    <PackageReference Include="Lamar.Microsoft.DependencyInjection" Version="5.0.0" />
+    <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.16.0" />
+    <PackageReference Include="Microsoft.NET.Build.Containers" Version="0.3.2" />
   </ItemGroup>
 
   <ItemGroup>
@@ -24,7 +23,4 @@
     <ProjectReference Include="..\BirdsiteLive.Twitter\BirdsiteLive.Twitter.csproj" />
     <ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL.Postgres\BirdsiteLive.DAL.Postgres.csproj" />
   </ItemGroup>
-
-
-
 </Project>

src/BirdsiteLive/Component/NodeInfoViewComponent.cs

@@ -7,17 +7,16 @@ using BirdsiteLive.Domain.Repository;
 using BirdsiteLive.Services;
 using BirdsiteLive.Statistics.Domain;
 using Microsoft.AspNetCore.Mvc;
-using Microsoft.AspNetCore.Razor.Language.Intermediate;
 
 namespace BirdsiteLive.Component
 {
     public class NodeInfoViewComponent : ViewComponent
     {
         private readonly IModerationRepository _moderationRepository;
-        private readonly IAboutPageService _cachedStatisticsService;
+        private readonly ICachedStatisticsService _cachedStatisticsService;
         
         #region Ctor
-        public NodeInfoViewComponent(IModerationRepository moderationRepository, IAboutPageService cachedStatisticsService)
+        public NodeInfoViewComponent(IModerationRepository moderationRepository, ICachedStatisticsService cachedStatisticsService)
         {
             _moderationRepository = moderationRepository;
             _cachedStatisticsService = cachedStatisticsService;
@@ -29,7 +28,7 @@ namespace BirdsiteLive.Component
             var followerPolicy = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.Follower);
             var twitterAccountPolicy = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.TwitterAccount);
 
-            var statistics = await _cachedStatisticsService.GetAboutPageDataAsync();
+            var statistics = await _cachedStatisticsService.GetStatisticsAsync();
 
             var viewModel = new NodeInfoViewModel
             {
@@ -37,8 +36,7 @@ namespace BirdsiteLive.Component
                                       twitterAccountPolicy == ModerationTypeEnum.BlackListing,
                 WhitelistingEnabled = followerPolicy == ModerationTypeEnum.WhiteListing ||
                                       twitterAccountPolicy == ModerationTypeEnum.WhiteListing,
-                InstanceSaturation = statistics.Saturation,
-                DiscloseRestrictions = statistics.Settings.DiscloseInstanceRestrictions
+                SyncLag = statistics.SyncLag
             };
             
             //viewModel = new NodeInfoViewModel
@@ -56,6 +54,6 @@ namespace BirdsiteLive.Component
         public bool BlacklistingEnabled { get; set; }
         public bool WhitelistingEnabled { get; set; }
         public int InstanceSaturation { get; set; }
-        public bool DiscloseRestrictions { get; set; }
+        public TimeSpan SyncLag { get; set; }
     }
 }

src/BirdsiteLive/Controllers/AboutController.cs

@@ -10,21 +10,37 @@ namespace BirdsiteLive.Controllers
 {
     public class AboutController : Controller
     {
-        private readonly IAboutPageService _aboutPageService;
+        private readonly IModerationRepository _moderationRepository;
+        private readonly ICachedStatisticsService _cachedStatisticsService;
 
         #region Ctor
-        public AboutController(IAboutPageService cachedStatisticsService)
+        public AboutController(IModerationRepository moderationRepository, ICachedStatisticsService cachedStatisticsService)
         {
-            _aboutPageService = cachedStatisticsService;
+            _moderationRepository = moderationRepository;
+            _cachedStatisticsService = cachedStatisticsService;
         }
         #endregion
 
         public async Task<IActionResult> Index()
         {
-            var stats = await _aboutPageService.GetAboutPageDataAsync();
+            var stats = await _cachedStatisticsService.GetStatisticsAsync();
             return View(stats);
         }
+
+        private ModerationStatus GetModerationStatus()
+        {
+            var status = new ModerationStatus
+            {
+                Followers = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.Follower),
+                TwitterAccounts = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.TwitterAccount)
+            };
+            return status;
+        }
     }
 
-
+    public class ModerationStatus
+    {
+        public ModerationTypeEnum Followers { get; set; }
+        public ModerationTypeEnum TwitterAccounts { get; set; }
+    }
 }

src/BirdsiteLive/Controllers/DebugingController.cs

@@ -10,7 +10,6 @@ using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Domain;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.Mvc.ViewFeatures;
-using Newtonsoft.Json;
 
 namespace BirdsiteLive.Controllers
 {
@@ -59,22 +58,17 @@ namespace BirdsiteLive.Controllers
         [HttpPost]
         public async Task<IActionResult> PostNote()
         {
-            var username = "twitter";
+            var username = "gra";
             var actor = $"https://{_instanceSettings.Domain}/users/{username}";
-            var targetHost = "ioc.exchange";
-            var target = $"https://{targetHost}/users/test";
-            //var inbox = $"/users/testtest/inbox";
-            var inbox = $"/inbox";
+            var targetHost = "mastodon.technology";
+            var target = $"{targetHost}/users/testtest";
+            var inbox = $"/users/testtest/inbox";
 
             var noteGuid = Guid.NewGuid();
             var noteId = $"https://{_instanceSettings.Domain}/users/{username}/statuses/{noteGuid}";
             var noteUrl = $"https://{_instanceSettings.Domain}/@{username}/{noteGuid}";
 
             var to = $"{actor}/followers";
-            to = target;
-
-            var cc = new[] { "https://www.w3.org/ns/activitystreams#Public" };
-            cc = new string[0];
 
             var now = DateTime.UtcNow;
             var nowString = now.ToString("s") + "Z";
@@ -87,7 +81,7 @@ namespace BirdsiteLive.Controllers
                 actor = actor,
                 published = nowString,
                 to = new[] { to },
-                cc = cc,
+                //cc = new [] { "https://www.w3.org/ns/activitystreams#Public" },
                 apObject = new Note()
                 {
                     id = noteId,
@@ -99,8 +93,7 @@ namespace BirdsiteLive.Controllers
 
                     // Unlisted
                     to = new[] { to },
-                    cc = cc,
-                    //cc = new[] { "https://www.w3.org/ns/activitystreams#Public" },
+                    cc = new[] { "https://www.w3.org/ns/activitystreams#Public" },
 
                     //// Public
                     //to = new[] { "https://www.w3.org/ns/activitystreams#Public" },
@@ -108,16 +101,8 @@ namespace BirdsiteLive.Controllers
 
                     sensitive = false,
                     content = "<p>TEST PUBLIC</p>",
-                    //content = "<p><span class=\"h-card\"><a href=\"https://ioc.exchange/users/test\" class=\"u-url mention\">@<span>test</span></a></span> test</p>",
                     attachment = new Attachment[0],
-                    tag = new Tag[]{
-                        new Tag()
-                        {
-                            type = "Mention",
-                            href = target,
-                            name = "@test@ioc.exchange"
-                        }
-                    },
+                    tag = new Tag[0]
                 }
             };
 
@@ -139,17 +124,6 @@ namespace BirdsiteLive.Controllers
             await _userService.SendRejectFollowAsync(activityFollow, "mastodon.technology");
             return View("Index");
         }
-
-        [HttpPost]
-        public async Task<IActionResult> PostDeleteUser()
-        {
-            var userName = "twitter";
-            var host = "ioc.exchange";
-            var inbox = "/inbox";
-
-            await _activityPubService.DeleteUserAsync(userName, host, inbox);
-            return View("Index");
-        }
     }
     #endif
 

src/BirdsiteLive/Controllers/HomeController.cs

@@ -6,24 +6,21 @@ using System.Threading.Tasks;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
 using BirdsiteLive.Models;
-using BirdsiteLive.Common.Settings;
 
 namespace BirdsiteLive.Controllers
 {
     public class HomeController : Controller
     {
         private readonly ILogger<HomeController> _logger;
-        private readonly InstanceSettings _instanceSettings;
 
-        public HomeController(ILogger<HomeController> logger, InstanceSettings instanceSettings)
+        public HomeController(ILogger<HomeController> logger)
         {
             _logger = logger;
-            _instanceSettings = instanceSettings;
         }
 
         public IActionResult Index()
         {
-            return View(_instanceSettings);
+            return View();
         }
 
         public IActionResult Privacy()

src/BirdsiteLive/Controllers/MigrationController.cs

@@ -1,227 +0,0 @@
-using System;
-using Microsoft.AspNetCore.Mvc;
-using System.Security.Cryptography;
-using System.Text;
-using System.Threading.Tasks;
-using Npgsql.TypeHandlers;
-using BirdsiteLive.Domain;
-using BirdsiteLive.Domain.Enum;
-using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.Models;
-using System.Reflection.Metadata;
-
-namespace BirdsiteLive.Controllers
-{
-    public class MigrationController : Controller
-    {
-        private readonly MigrationService _migrationService;
-        private readonly ITwitterUserDal _twitterUserDal;
-        
-        #region Ctor
-        public MigrationController(MigrationService migrationService, ITwitterUserDal twitterUserDal)
-        {
-            _migrationService = migrationService;
-            _twitterUserDal = twitterUserDal;
-        }
-        #endregion
-
-        [HttpGet]
-        [Route("/migration/move/{id}")]
-        public IActionResult IndexMove(string id)
-        {
-            var migrationCode = _migrationService.GetMigrationCode(id);
-            var data = new MigrationData()
-            {
-                Acct = id,
-                MigrationCode = migrationCode
-            };
-
-            return View("Index", data);
-        }
-
-        [HttpGet]
-        [Route("/migration/delete/{id}")]
-        public IActionResult IndexDelete(string id)
-        {
-            var migrationCode = _migrationService.GetDeletionCode(id);
-            var data = new MigrationData()
-            {
-                Acct = id,
-                MigrationCode = migrationCode
-            };
-
-            return View("Delete", data);
-        }
-
-        [HttpPost]
-        [Route("/migration/move/{id}")]
-        public async Task<IActionResult> MigrateMove(string id, string tweetid, string handle)
-        {
-            var migrationCode = _migrationService.GetMigrationCode(id);
-            var data = new MigrationData()
-            {
-                Acct = id,
-                MigrationCode = migrationCode,
-
-                IsAcctProvided = !string.IsNullOrWhiteSpace(handle),
-                IsTweetProvided = !string.IsNullOrWhiteSpace(tweetid),
-
-                TweetId = tweetid,
-                FediverseAccount = handle
-            };
-            ValidatedFediverseUser fediverseUserValidation = null;
-
-            //Verify can be migrated 
-            var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id);
-            if (twitterAccount != null && twitterAccount.Deleted)
-            {
-                data.ErrorMessage = "This account has been deleted, it can't be migrated";
-                return View("Index", data);
-            }
-            if (twitterAccount != null &&
-                (!string.IsNullOrWhiteSpace(twitterAccount.MovedTo)
-                 || !string.IsNullOrWhiteSpace(twitterAccount.MovedToAcct)))
-            {
-                data.ErrorMessage = "This account has been moved already, it can't be migrated again";
-                return View("Index", data);
-            }
-
-            // Start migration
-            try
-            {
-                fediverseUserValidation = await _migrationService.ValidateFediverseAcctAsync(handle);
-                var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Migration);
-
-                data.IsAcctValid = fediverseUserValidation.IsValid;
-                data.IsTweetValid = isTweetValid;
-            }
-            catch (Exception e)
-            {
-                data.ErrorMessage = e.Message;
-            }
-
-            if (data.IsAcctValid && data.IsTweetValid && fediverseUserValidation != null)
-            {
-                try
-                {
-                    await _migrationService.MigrateAccountAsync(fediverseUserValidation, id);
-                    _migrationService.TriggerRemoteMigrationAsync(id, tweetid, handle);
-                    data.MigrationSuccess = true;
-                }
-                catch (Exception e)
-                {
-                    Console.WriteLine(e);
-                    data.ErrorMessage = e.Message;
-                }
-            }
-
-            return View("Index", data);
-        }
-
-        [HttpPost]
-        [Route("/migration/delete/{id}")]
-        public async Task<IActionResult> MigrateDelete(string id, string tweetid)
-        {
-            var deletionCode = _migrationService.GetDeletionCode(id);
-
-            var data = new MigrationData()
-            {
-                Acct = id,
-                MigrationCode = deletionCode,
-
-                IsTweetProvided = !string.IsNullOrWhiteSpace(tweetid),
-
-                TweetId = tweetid
-            };
-
-            //Verify can be deleted 
-            var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id);
-            if (twitterAccount != null && twitterAccount.Deleted)
-            {
-                data.ErrorMessage = "This account has been deleted, it can't be deleted again";
-                return View("Delete", data);
-            }
-
-            // Start deletion
-            try
-            {
-                var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Deletion);
-                data.IsTweetValid = isTweetValid;
-            }
-            catch (Exception e)
-            {
-                data.ErrorMessage = e.Message;
-            }
-
-            if (data.IsTweetValid)
-            {
-                try
-                {
-                    await _migrationService.DeleteAccountAsync(id);
-                    _migrationService.TriggerRemoteDeleteAsync(id, tweetid);
-                    data.MigrationSuccess = true;
-                }
-                catch (Exception e)
-                {
-                    Console.WriteLine(e);
-                    data.ErrorMessage = e.Message;
-                }
-            }
-
-            return View("Delete", data);
-        }
-
-        [HttpPost]
-        [Route("/migration/move/{id}/{tweetid}/{handle}")]
-        public async Task<IActionResult> RemoteMigrateMove(string id, string tweetid, string handle)
-        {
-            //Check inputs 
-            if (string.IsNullOrWhiteSpace(id) || string.IsNullOrWhiteSpace(tweetid) ||
-                string.IsNullOrWhiteSpace(handle))
-                return StatusCode(422);
-
-            //Verify can be migrated 
-            var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id);
-            if (twitterAccount != null && (twitterAccount.Deleted 
-                || !string.IsNullOrWhiteSpace(twitterAccount.MovedTo) 
-                || !string.IsNullOrWhiteSpace(twitterAccount.MovedToAcct)))
-                return Ok();
-
-            // Start migration
-            var fediverseUserValidation = await _migrationService.ValidateFediverseAcctAsync(handle);
-            var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Migration);
-
-            if (fediverseUserValidation.IsValid && isTweetValid)
-            {
-                await _migrationService.MigrateAccountAsync(fediverseUserValidation, id);
-                return Ok();
-            }
-
-            return StatusCode(400);
-        }
-
-        [HttpPost]
-        [Route("/migration/delete/{id}/{tweetid}")]
-        public async Task<IActionResult> RemoteMigrateDelete(string id, string tweetid)
-        {
-            //Check inputs 
-            if (string.IsNullOrWhiteSpace(id) || string.IsNullOrWhiteSpace(tweetid))
-                return StatusCode(422);
-
-            //Verify can be deleted 
-            var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id);
-            if (twitterAccount != null && twitterAccount.Deleted) return Ok();
-
-            // Start deletion
-            var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Deletion);
-
-            if (isTweetValid)
-            {
-                await _migrationService.DeleteAccountAsync(id);
-                return Ok();
-            }
-
-            return StatusCode(400);
-        }
-    }
-}

src/BirdsiteLive/Controllers/UsersController.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text.Json;
 using System.Net.Mime;
 using System.Text.RegularExpressions;
 using System.Threading;
@@ -21,32 +22,33 @@ using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Primitives;
-using Newtonsoft.Json;
 
 namespace BirdsiteLive.Controllers
 {
     public class UsersController : Controller
     {
-        private readonly ITwitterUserService _twitterUserService;
-        private readonly ITwitterTweetsService _twitterTweetService;
-        private readonly ITwitterUserDal _twitterUserDal;
+        private readonly ICachedTwitterUserService _twitterUserService;
+        private readonly ICachedTwitterTweetsService _twitterTweetService;
         private readonly IUserService _userService;
         private readonly IStatusService _statusService;
         private readonly InstanceSettings _instanceSettings;
+        private readonly IFollowersDal _followersDal;
+        private readonly ITwitterUserDal _twitterUserDal;
         private readonly IActivityPubService _activityPubService;
         private readonly ILogger<UsersController> _logger;
 
         #region Ctor
-        public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ITwitterTweetsService twitterTweetService, IActivityPubService activityPubService, ILogger<UsersController> logger, ITwitterUserDal twitterUserDal)
+        public UsersController(ICachedTwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ICachedTwitterTweetsService twitterTweetService, IFollowersDal followersDal, ITwitterUserDal twitterUserDal, IActivityPubService activityPubService, ILogger<UsersController> logger)
         {
             _twitterUserService = twitterUserService;
             _userService = userService;
             _statusService = statusService;
             _instanceSettings = instanceSettings;
             _twitterTweetService = twitterTweetService;
+            _followersDal = followersDal;
+            _twitterUserDal = twitterUserDal;
             _activityPubService = activityPubService;
             _logger = logger;
-            _twitterUserDal = twitterUserDal;
         }
         #endregion
 
@@ -61,9 +63,10 @@ namespace BirdsiteLive.Controllers
             }
             return View("UserNotFound");
         }
-        
+
         [Route("/@{id}")]
         [Route("/users/{id}")]
+        [Route("/users/{id}/remote_follow")]
         public async Task<IActionResult> Index(string id)
         {
             _logger.LogTrace("User Index: {Id}", id);
@@ -80,7 +83,7 @@ namespace BirdsiteLive.Controllers
             {
                 try
                 {
-                    user = _twitterUserService.GetUser(id);
+                    user = await _twitterUserService.GetUserAsync(id);
                 }
                 catch (UserNotFoundException)
                 {
@@ -106,7 +109,6 @@ namespace BirdsiteLive.Controllers
             }
 
             //var isSaturated = _twitterUserService.IsUserApiRateLimited();
-            var dbUser = await _twitterUserDal.GetTwitterUserAsync(id);
 
             var acceptHeaders = Request.Headers["Accept"];
             if (acceptHeaders.Any())
@@ -116,12 +118,8 @@ namespace BirdsiteLive.Controllers
                 {
                     if (isSaturated) return new ObjectResult("Too Many Requests") { StatusCode = 429 };
                     if (notFound) return NotFound();
-                    if (dbUser != null && dbUser.Deleted) return new ObjectResult("Gone") { StatusCode = 410 };
-                    var apUser = _userService.GetUser(user, dbUser);
-                    var jsonApUser = JsonConvert.SerializeObject(apUser, new JsonSerializerSettings
-                    {
-                        NullValueHandling = NullValueHandling.Ignore
-                    });
+                    var apUser = _userService.GetUser(user);
+                    var jsonApUser = System.Text.Json.JsonSerializer.Serialize(apUser);
                     return Content(jsonApUser, "application/activity+json; charset=utf-8");
                 }
             }
@@ -129,6 +127,12 @@ namespace BirdsiteLive.Controllers
             if (isSaturated) return View("ApiSaturated");
             if (notFound) return View("UserNotFound");
 
+            Follower[] followers = new Follower[] { };
+            
+            var userDal = await _twitterUserDal.GetTwitterUserAsync(user.Acct);
+            if (userDal != null)
+                followers = await _followersDal.GetFollowersAsync(userDal.Id);
+
             var displayableUser = new DisplayTwitterUser
             {
                 Name = user.Name,
@@ -137,61 +141,68 @@ namespace BirdsiteLive.Controllers
                 Url = user.Url,
                 ProfileImageUrl = user.ProfileImageUrl,
                 Protected = user.Protected,
-                
-                InstanceHandle = $"@{user.Acct.ToLowerInvariant()}@{_instanceSettings.Domain}",
+                FollowerCount = followers.Length,
+                MostPopularServer = followers.GroupBy(x => x.Host).OrderByDescending(x => x.Count()).Select(x => x.Key).FirstOrDefault("N/A"),
 
-                MovedTo = dbUser?.MovedTo,
-                MovedToAcct = dbUser?.MovedToAcct,
-                Deleted = dbUser?.Deleted ?? false,
+                InstanceHandle = $"@{user.Acct.ToLowerInvariant()}@{_instanceSettings.Domain}"
             };
             return View(displayableUser);
         }
-        
-        [Route("/users/{id}/remote_follow")]
-        public async Task<IActionResult> IndexRemoteFollow(string id)
-        {
-            return Redirect($"/users/{id}");
-        }
 
         [Route("/@{id}/{statusId}")]
         [Route("/users/{id}/statuses/{statusId}")]
-        public IActionResult Tweet(string id, string statusId)
+        public async Task<IActionResult> Tweet(string id, string statusId)
         {
             var acceptHeaders = Request.Headers["Accept"];
+            if (!long.TryParse(statusId, out var parsedStatusId))
+                return NotFound();
+
+            var tweet = await _twitterTweetService.GetTweetAsync(parsedStatusId);
+            if (tweet == null)
+                return NotFound();
+            
+            var user = await _twitterUserService.GetUserAsync(id);
+
+            var status = _statusService.GetStatus(id, tweet);
+
             if (acceptHeaders.Any())
             {
                 var r = acceptHeaders.First();
+
                 if (r.Contains("application/activity+json"))
                 {
-                    if (!long.TryParse(statusId, out var parsedStatusId))
-                        return NotFound();
-
-                    if (_instanceSettings.MaxStatusFetchAge > 0)
-                    {
-                        // I hate bitwise operators, corn syrup, and the antichrist
-                        // shift 22 bits to the right to get milliseconds, add the twitter epoch, then divide by 1000 to get seconds
-                        long secondsAgo = DateTimeOffset.UtcNow.ToUnixTimeSeconds() - (((parsedStatusId >> 22) + 1288834974657) / 1000);
-
-                        if ( secondsAgo > _instanceSettings.MaxStatusFetchAge*60*60*24 )
-                        {
-                            return new StatusCodeResult(StatusCodes.Status410Gone);
-                        }
-                    }
-
-                    var tweet = _twitterTweetService.GetTweet(parsedStatusId);
-                    if (tweet == null)
-                        return NotFound();
-
-                    //var user = _twitterService.GetUser(id);
-                    //if (user == null) return NotFound();
-
-                    var status = _statusService.GetStatus(id, tweet);
-                    var jsonApUser = JsonConvert.SerializeObject(status);
+                    var jsonApUser = JsonSerializer.Serialize(status);
                     return Content(jsonApUser, "application/activity+json; charset=utf-8");
                 }
             }
 
-            return Redirect($"https://{_instanceSettings.TwitterDomain}/{id}/status/{statusId}");
+            //return Redirect($"https://twitter.com/{id}/status/{statusId}");
+            var displayTweet = new DisplayTweet 
+            {
+                Text = tweet.MessageContent,
+                OgUrl = $"https://twitter.com/{id}/status/{statusId}",
+                UserProfileImage = user.ProfileImageUrl,
+                UserName = user.Name,
+            };
+            return View(displayTweet);
+        }
+
+        [Route("/users/{id}/statuses/{statusId}/activity")]
+        public async Task<IActionResult> Activity(string id, string statusId)
+        {
+            if (!long.TryParse(statusId, out var parsedStatusId))
+                return NotFound();
+
+            var tweet = await _twitterTweetService.GetTweetAsync(parsedStatusId);
+            if (tweet == null)
+                return NotFound();
+            
+            var user = await _twitterUserService.GetUserAsync(id);
+
+            var status = _statusService.GetActivity(id, tweet);
+
+            var jsonApUser = JsonSerializer.Serialize(status);
+            return Content(jsonApUser, "application/activity+json; charset=utf-8");
         }
 
         [Route("/users/{id}/inbox")]
@@ -274,7 +285,7 @@ namespace BirdsiteLive.Controllers
             {
                 id = $"https://{_instanceSettings.Domain}/users/{id}/followers"
             };
-            var jsonApUser = JsonConvert.SerializeObject(followers);
+            var jsonApUser = JsonSerializer.Serialize(followers);
             return Content(jsonApUser, "application/activity+json; charset=utf-8");
         }
 

src/BirdsiteLive/Controllers/WellKnownController.cs

@@ -59,9 +59,16 @@ namespace BirdsiteLive.Controllers
             return new JsonResult(nodeInfo);
         }
 
+        [Route("/.well-known/host-meta")]
+        public IActionResult HostMeta()
+        {
+            return Content($"<?xml version=\"1.0\" encoding=\"UTF-8\"?><XRD xmlns=\"http://docs.oasis-open.org/ns/xri/xrd-1.0\"><Link rel=\"lrdd\" template=\"https://{_settings.Domain}/.well-known/webfinger?resource={{uri}}\" type=\"application/xrd+xml\" /></XRD>", "application/xrd+xml; charset=utf-8");
+        }
+
         [Route("/nodeinfo/{id}.json")]
         public async Task<IActionResult> NodeInfo(string id)
         {
+            var version = System.Reflection.Assembly.GetEntryAssembly().GetName().Version.ToString(3);
             var twitterUsersCount = await _twitterUserDal.GetTwitterUsersCountAsync();
             var isOpenRegistration = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.Follower) != ModerationTypeEnum.WhiteListing;
 
@@ -81,7 +88,7 @@ namespace BirdsiteLive.Controllers
                     software = new Software()
                     {
                         name = "birdsitelive",
-                        version = Program.VERSION
+                        version = version
                     },
                     protocols = new[]
                     {
@@ -116,7 +123,7 @@ namespace BirdsiteLive.Controllers
                     software = new SoftwareV21()
                     {
                         name = "birdsitelive",
-                        version = Program.VERSION,
+                        version = version,
                         repository = "https://github.com/NicolasConstant/BirdsiteLive"
                     },
                     protocols = new[]
@@ -141,7 +148,7 @@ namespace BirdsiteLive.Controllers
         }
 
         [Route("/.well-known/webfinger")]
-        public IActionResult Webfinger(string resource = null)
+        public async Task<IActionResult> Webfinger(string resource = null)
         {
             if (string.IsNullOrWhiteSpace(resource)) 
                 return BadRequest();
@@ -202,7 +209,7 @@ namespace BirdsiteLive.Controllers
 
             try
             {
-                _twitterUserService.GetUser(name);
+                await _twitterUserService.GetUserAsync(name);
             }
             catch (UserNotFoundException)
             {

src/BirdsiteLive/Models/DisplayTweet.cs

@@ -0,0 +1,10 @@
+namespace BirdsiteLive.Models
+{
+    public class DisplayTweet
+    {
+        public string Text { get; set; }
+        public string OgUrl { get; set; }
+        public string UserProfileImage { get; set; }
+        public string UserName { get; set; }
+    }
+}

src/BirdsiteLive/Models/DisplayTwitterUser.cs

@@ -8,11 +8,9 @@
         public string Url { get; set; }
         public string ProfileImageUrl { get; set; }
         public bool Protected { get; set; }
+        public int FollowerCount { get; set; }
+        public string MostPopularServer { get; set; }
 
         public string InstanceHandle { get; set; }
-
-        public string MovedTo { get; set; }
-        public string MovedToAcct { get; set; }
-        public bool Deleted { get; set; }
     }
 }