chore(deps): update mstest monorepo to v3.0.3

Signed-off-by: Sam Therapy <sam@samtherapy.net>

.drone.yml

@@ -0,0 +1,59 @@
+kind: pipeline
+name: testing
+type: docker
+
+steps:
+  - name: Install Dependencies
+    image: mcr.microsoft.com/dotnet/sdk:6.0
+    commands: 
+      - dotnet restore ./src
+
+  - name: Build
+    image: mcr.microsoft.com/dotnet/sdk:6.0
+    commands: 
+      - dotnet build --configuration Release ./src
+
+  - name: Test
+    image: mcr.microsoft.com/dotnet/sdk:6.0
+    commands: 
+      - sed -i "s/127\.0\.0\.1/database/g" ./src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/Base/PostgresTestingBase.cs
+      - dotnet test --verbosity minimal ./src
+
+services:
+  - name: database
+    image: postgres:15
+    environment:
+      POSTGRES_USER: birdtest
+      POSTGRES_PASSWORD: mysecretpassword 
+      POSTGRES_DB: birdsitetest
+
+---
+
+kind: pipeline
+name: docker-publish
+type: docker
+
+depends_on:
+  - testing
+
+steps:
+  - name: Build & Publish
+    privileged: true
+    image: quay.io/thegeeklab/drone-docker-buildx
+    settings:
+      auto_tag: true
+      repo: git.froth.zone/sam/birdsitelive
+      registry: git.froth.zone
+      username: sam
+      password:
+        from_secret: password
+      platforms: 
+        - linux/amd64
+        - linux/arm64
+    when:
+      branch:
+        - master
+      event:
+        - push
+    depends_on:
+      - "clone"

.github/workflows/dotnet-core.yml

@@ -10,11 +10,11 @@ jobs:
       working-directory: ./src
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Launch Db for testing
       run: docker run --name postgres -e POSTGRES_DB=mytestdb -e POSTGRES_PASSWORD=mysecretpassword -d -p 5432:5432 postgres
     - name: Setup .NET Core
-      uses: actions/setup-dotnet@v1
+      uses: actions/setup-dotnet@v3
       with:
         dotnet-version: 3.1.101
     - name: Install dependencies

.gitignore

@@ -352,3 +352,5 @@ MigrationBackup/
 # Ionide (cross platform F# VS Code tools) working folder
 .ionide/
 /src/BSLManager/Properties/launchSettings.json
+
+.dccache

Dockerfile

@@ -1,14 +1,15 @@
 #See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging.
 
-FROM mcr.microsoft.com/dotnet/aspnet:3.1-buster-slim AS base
+FROM mcr.microsoft.com/dotnet/aspnet:6.0 AS base
 WORKDIR /app
 EXPOSE 80
 EXPOSE 443
 
-FROM mcr.microsoft.com/dotnet/sdk:3.1-buster AS publish
+FROM mcr.microsoft.com/dotnet/sdk:6.0 AS publish
+WORKDIR /
 COPY ./src/ ./src/
-RUN dotnet publish "/src/BirdsiteLive/BirdsiteLive.csproj" -c Release -o /app/publish
-RUN dotnet publish "/src/BSLManager/BSLManager.csproj" -r linux-x64 --self-contained true -p:PublishSingleFile=true -p:IncludeAllContentForSelfExtract=true -c Release -o /app/publish
+RUN dotnet publish "/src/BirdsiteLive/BirdsiteLive.csproj" -c Release -o /app/publish \
+	&& dotnet publish "/src/BSLManager/BSLManager.csproj" -r linux-x64 --self-contained true -p:PublishSingleFile=true -p:IncludeAllContentForSelfExtract=true -c Release -o /app/publish
 
 FROM base AS final
 WORKDIR /app

INSTALLATION.md

@@ -4,19 +4,16 @@
 
 You will need a Twitter API key to make BirdsiteLIVE working. First create an **Standalone App** in the [Twitter developer portal](https://developer.twitter.com/en/portal/projects-and-apps) and retrieve the API Key and API Secret Key. 
 
-Please make sure you are using a **Standalone App** API Key and not a **Project App** API Key (that will NOT work with BirdsiteLIVE), if you don't see the **Standalone App** section, you might need to [apply for Elevated Access](https://developer.twitter.com/en/portal/products/elevated) as described in the [API documentation](https://developer.twitter.com/en/support/twitter-api/developer-account).
-
-
 ## Server prerequisites
 
 Your instance will need [docker](https://docs.docker.com/engine/install/) and [docker-compose](https://docs.docker.com/compose/install/) installed and working. 
 
 ## Setup
 
-Download the [docker-compose file](https://github.com/NicolasConstant/BirdsiteLive/blob/master/docker-compose.yml): 
+Download the [docker-compose file](https://git.froth.zone/sam/BirdsiteLIVE/raw/branch/master/docker-compose.yml): 
 
 ```
-sudo curl -L https://raw.githubusercontent.com/NicolasConstant/BirdsiteLive/master/docker-compose.yml -o docker-compose.yml
+sudo curl -L https://git.froth.zone/sam/BirdsiteLIVE/raw/branch/master/docker-compose.yml -o docker-compose.yml
 ```
 
 Then edit file: 
@@ -29,7 +26,7 @@ sudo nano docker-compose.yml
 
 #### Personal info
 
-* `Instance:Domain` the domain name you'll be using, for example use `birdsite.live` for the URL `https://birdsite.live`
+* `Instance:Domain` the domain name you'll be using, for example use `birdsite.example.com` for the URL `https://birdsite.example.com`
 * `Instance:AdminEmail` the admin's email, will be displayed in the instance /.well-known/nodeinfo endpoint
 * `Twitter:ConsumerKey` the Twitter API key
 * `Twitter:ConsumerSecret` the Twitter API secret key
@@ -58,35 +55,14 @@ docker-compose up -d
 
 By default the app will be available on the port 5000
 
-## Nginx 
+## Nginx configuration
 
-On a Debian based distrib:
-
-```
-sudo apt update
-sudo apt install nginx
-```
-
-Check nginx status: 
-
-```
-sudo systemctl status nginx
-```
-
-### Create nginx configuration
-
-Create your nginx configuration
-
-```
-sudo nano /etc/nginx/sites-enabled/{your-domain-name.com}
-```
-
-And fill your service block as follow:
+Fill your service block as follow:
 
 ```
 server {
     listen        80;
-    server_name   {your-domain-name.com};
+    server_name   birdsite.example.com;
     location / {
         proxy_pass         http://localhost:5000;
         proxy_http_version 1.1;
@@ -114,16 +90,31 @@ After having a domain name pointing to your instance, install and setup certbot:
 
 ```
 sudo apt install certbot python3-certbot-nginx
-sudo certbot --nginx -d {your-domain-name.com}
+sudo certbot --nginx -d birdsite.example.com
 ```
 
 Make sure you're redirecting all traffic to https when asked.
 
-Finally check that the auto-revewal will work as espected:
+Finally check that the auto-renewal will work as expected:
 
 ```
 sudo certbot renew --dry-run
 ```
+## Caddy
+
+
+Or, you can use [caddy](https://caddyserver.com)
+
+```caddyfile
+birdsite.example.com {
+        encode gzip
+        header ?Cache-Control "max-age=3600"
+        reverse_proxy http://localhost:5000 {
+            header_down -Server
+        }
+}
+```
+Everything 
 
 ### Set the firewall 
 
@@ -168,11 +159,11 @@ networks:
 
 services:
     server:
-        image: nicolasconstant/birdsitelive:latest
+        image: pasture/birdsitelive:latest
         [...]
 
     db:
-        image: postgres:9.6
+        image: postgres:13
         [...]
         
 +   watchtower:

README.md

@@ -1,6 +1,15 @@
-![Test](https://github.com/NicolasConstant/BirdsiteLive/workflows/.NET%20Core/badge.svg?branch=master&event=push)
+# BirdsiteLIVE: Twitter -> ActivityPub
 
-# BirdsiteLIVE
+[![Build Status](https://ci.git.froth.zone/api/badges/sam/BirdsiteLIVE/status.svg)](https://ci.git.froth.zone/sam/BirdsiteLIVE)
+
+This project is a _fork_ of [Pasture's fork](https://git.gamers.exposed/pasture/BirdsiteLIVE) of [the original BirdsiteLIVE from NicolasConstant](https://github.com/NicolasConstant/BirdsiteLive). This fork runs in production on [bird.froth.zone](https://bird.froth.zone). Changes made in this fork include:
+
+- Rebasing the forks together.
+- (this space intentionally left blank)
+
+This fork is also available as a Docker image as `git.froth.zone/sam/birdsitelive`.
+
+The project's original README is below:
 
 ## About
 
@@ -8,24 +17,22 @@ BirdsiteLIVE is an ActivityPub bridge from Twitter, it's mostly a pet project/pl
 
 ## State of development
 
-The code is pretty messy and far from a good state, since it's a playground for me the aim was to understand some AP concepts, not provide a good state-of-the-art codebase. But I might refactor it to make it cleaner. 
+The code is pretty messy and far from a good state, since it's a playground for me the aim was to understand some AP concepts, not to provide a good state-of-the-art codebase. But I might refactor it to make it cleaner.
 
-## Official instance 
+## Official instance
 
-You can find an official (and temporary) instance here: [beta.birdsite.live](https://beta.birdsite.live). This instance can disapear at any time, if you want a long term instance you should install your own or use another one. 
+There's none! Please read [here why I've stopped it](https://write.as/nicolas-constant/closing-the-official-bsl-instance).
 
 ## Installation
 
-I'm providing a [docker build](https://hub.docker.com/r/nicolasconstant/birdsitelive). To install it on your own server, please follow [those instructions](https://github.com/NicolasConstant/BirdsiteLive/blob/master/INSTALLATION.md). More [options](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md) are also available.
+I'm providing a [docker build](https://git.froth.zone/sam/-/packages/container/birdsitelive/latest). To install it on your own server, please follow [those instructions](./INSTALLATION.md). More [options](./VARIABLES.md) are also available.
 
-Also a [CLI](https://github.com/NicolasConstant/BirdsiteLive/blob/master/BSLManager.md) is available for adminitrative tasks.
+Also a (likely broken) [CLI](./BSLManager.md) is available for administrative tasks.
 
 ## License
 
-This project is licensed under the AGPLv3 License - see [LICENSE](https://github.com/NicolasConstant/BirdsiteLive/blob/master/LICENSE) for details.
+This project is licensed under the AGPLv3 License - see [LICENSE](./LICENSE) for details.
 
 ## Contact
 
 You can contact me via ActivityPub <a rel="me" href="https://fosstodon.org/@BirdsiteLIVE">here</a>.
-
-

VARIABLES.md

@@ -46,9 +46,18 @@ If both whitelisting and blacklisting are set, only the whitelisting will be act
 * `Instance:Name` (default: BirdsiteLIVE) the name of the instance
 * `Instance:ResolveMentionsInProfiles` (default: true) to enable or disable mentions parsing in profile's description. Resolving it will consume more User's API calls since newly discovered account can also contain references to others accounts as well. On a big instance it is recommended to disable it.
 * `Instance:PublishReplies` (default: false) to enable or disable replies publishing.
+* `Instance:UnlistedTwitterAccounts` (default: null) to enable unlisted publication for selected twitter accounts, separated by `;` (please limit this to brands and other public profiles).
+* `Instance:TwitterDomain` (default: twitter.com) redirect to a different domain (i.e. a Nitter instance) instead of Twitter in most areas
+* `Instance:TwitterDomainLabel` (default: "") if TwitterDomain is set, use this label on profile pages instead of the domain itself (i.e. you can set this to "Nitter" to show that on profiles instead of "twiiit.com")
+* `Instance:InfoBanner` (default: "") text to show in a banner on the front page
+* `Instance:ShowAboutInstanceOnProfiles` (default: true) show "About [instance name]" on profiles with a link to /About
+* `Instance:MaxFollowsPerUser` (default: 0 - no limit) limit the number of follows per user - any follow count above this number will be Rejected
+* `Instance:DiscloseInstanceRestrictions` (default: false) disclose your instance's restrictions on its About page
 * `Instance:UnlistedTwitterAccounts` (default: null) to enable unlisted publication for selected twitter accounts, separated by `;` (please limit this to brands and other public profiles). 
 * `Instance:SensitiveTwitterAccounts` (default: null) mark all media from given accounts as sensitive by default, separated by `;`. 
 * `Instance:FailingTwitterUserCleanUpThreshold` (default: 700) set the max allowed errors (due to a banned/deleted/private account) from a Twitter Account retrieval before auto-removal. (by default an account is called every 15 mins)
+* `Instance:MaxStatusFetchAge` (default: 0 - no limit) statuses with a Snowflake older than this age in days will not be fetched by the service and will instead return 410 Gone
+* `Instance:EnableQuoteRT` (default: false) enable Soapbox-style quote-RTs
 * `Instance:FailingFollowerCleanUpThreshold` (default: 30000) set the max allowed errors from a Follower (Fediverse) Account before auto-removal. (often due to account suppression, instance issues, etc)
 * `Instance:UserCacheCapacity` (default: 10000) set the caching limit of the Twitter User retrieval. Must be higher than the number of synchronized accounts on the instance.
 
@@ -64,7 +73,7 @@ networks:
 
 services:
     server:
-        image: nicolasconstant/birdsitelive:latest
+        image: pasture/birdsitelive:latest
         [...]
         environment:
             - Instance:Domain=domain.name
@@ -82,12 +91,16 @@ services:
 +           - Instance:ResolveMentionsInProfiles=false
 +           - Instance:PublishReplies=true
 +           - Instance:UnlistedTwitterAccounts=cocacola;twitter
++           - Instance:TwitterDomain=twiiit.com
++           - Instance:TwitterDomainLabel=Nitter
++           - Instance:InfoBanner=This is my BirdsiteLIVE instance. There are many like it, but this one is mine.
++           - Instance:ShowAboutInstanceOnProfiles=true
 +           - Instance:SensitiveTwitterAccounts=archillect
         networks:
         [...]
 
     db:
-        image: postgres:9.6
+        image: postgres:13
         [...]
 ```
 

docker-compose.yml

@@ -6,7 +6,7 @@ networks:
 
 services:
     server:
-        image: nicolasconstant/birdsitelive:latest
+        image: git.froth.zone/birdsitelive:latest
         restart: always
         container_name: birdsitelive
         environment:
@@ -27,7 +27,7 @@ services:
             - db
 
     db:
-        image: postgres:9.6
+        image: postgres:15
         restart: always
         environment:
             - POSTGRES_USER=birdsitelive

renovate.json

@@ -0,0 +1,12 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:base",
+    ":npm",
+    ":gomod",
+    ":pinSkipCi",
+    ":docker",
+    ":enableVulnerabilityAlerts",
+    ":semanticCommits"
+  ]
+}

src/BSLManager/App.cs

@@ -29,6 +29,8 @@ namespace BSLManager
 
         public void Run()
         {
+            Application.UseSystemConsole = true;
+
             Application.Init();
             var top = Application.Top;
 

src/BSLManager/BSLManager.csproj

@@ -2,15 +2,15 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Lamar" Version="5.0.3" />
+    <PackageReference Include="Lamar" Version="5.0.4" />
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="5.0.0" />
-    <PackageReference Include="Terminal.Gui" Version="1.0.0-beta.11" />
+    <PackageReference Include="Terminal.Gui" Version="1.12.1" />
   </ItemGroup>
 
   <ItemGroup>

src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj

@@ -1,12 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="System.Text.Json" Version="4.7.2" />
   </ItemGroup>
 

src/BirdsiteLive.ActivityPub/Models/Activity.cs

@@ -1,12 +1,31 @@
-using System.Text.Json.Serialization;
+using System.Collections.Generic;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class Activity
     {
+        [Newtonsoft.Json.JsonIgnore]
+        public static readonly object[] DefaultContext = new object[] {
+            "https://www.w3.org/ns/activitystreams",
+            "https://w3id.org/security/v1",
+            new Dictionary<string, string>
+            {
+                { "Emoji", "toot:Emoji" },
+                { "Hashtag", "as:Hashtag" },
+                { "PropertyValue", "schema:PropertyValue" },
+                { "value", "schema:value" },
+                { "sensitive", "as:sensitive" },
+                { "quoteUrl", "as:quoteUrl" },
+
+                { "schema", "http://schema.org#" },
+                { "toot", "https://joinmastodon.org/ns#" }
+            }
+        };
+
         [JsonProperty("@context")]
-        public object context { get; set; }
+        public object context { get; set; } = DefaultContext;
         public string id { get; set; }
         public string type { get; set; }
         public string actor { get; set; }

src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs

@@ -4,6 +4,7 @@ namespace BirdsiteLive.ActivityPub.Models
 {
     public class ActivityDelete : Activity
     {
+        public string[] to { get; set; }
         [JsonProperty("object")]
         public object apObject { get; set; }
     }

src/BirdsiteLive.ActivityPub/Models/Actor.cs

@@ -1,5 +1,8 @@
-using System.Net;
+using System;
+using System.Collections.Generic;
+using System.Net;
 using BirdsiteLive.ActivityPub.Converters;
+using BirdsiteLive.ActivityPub.Models;
 using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub
@@ -9,7 +12,7 @@ namespace BirdsiteLive.ActivityPub
         //[JsonPropertyName("@context")]
         [JsonProperty("@context")]
         [JsonConverter(typeof(ContextArrayConverter))]
-        public string[] context { get; set; } = new[] { "https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1" };
+        public object[] context { get; set; } = Activity.DefaultContext;
         public string id { get; set; }
         public string type { get; set; }
         public string followers { get; set; }
@@ -17,6 +20,7 @@ namespace BirdsiteLive.ActivityPub
         public string name { get; set; }
         public string summary { get; set; }
         public string url { get; set; }
+        public string movedTo { get; set; }
         public bool manuallyApprovesFollowers { get; set; }
         public string inbox { get; set; }
         public bool? discoverable { get; set; } = true;
@@ -25,5 +29,6 @@ namespace BirdsiteLive.ActivityPub
         public Image image { get; set; }
         public EndPoints endpoints { get; set; }
         public UserAttachment[] attachment { get; set; }
+        public List<Tag> tag;
     }
 }

src/BirdsiteLive.ActivityPub/Models/Followers.cs

@@ -7,7 +7,7 @@ namespace BirdsiteLive.ActivityPub.Models
     {
         [JsonProperty("@context")]
         [JsonConverter(typeof(ContextArrayConverter))]
-        public string context { get; set; } = "https://www.w3.org/ns/activitystreams";
+        public object[] context { get; set; } = Activity.DefaultContext;
 
         public string id { get; set; }
         public string type { get; set; } = "OrderedCollection";

src/BirdsiteLive.ActivityPub/Models/Note.cs

@@ -1,5 +1,6 @@
 using BirdsiteLive.ActivityPub.Converters;
 using Newtonsoft.Json;
+using System.Collections.Generic;
 
 namespace BirdsiteLive.ActivityPub.Models
 {
@@ -7,7 +8,7 @@ namespace BirdsiteLive.ActivityPub.Models
     {
         [JsonProperty("@context")]
         [JsonConverter(typeof(ContextArrayConverter))]
-        public string[] context { get; set; } = new[] { "https://www.w3.org/ns/activitystreams" };
+        public object[] context { get; set; } = Activity.DefaultContext;
 
         public string id { get; set; }
         public string type { get; } = "Note";
@@ -25,5 +26,7 @@ namespace BirdsiteLive.ActivityPub.Models
         public Attachment[] attachment { get; set; }
         public Tag[] tag { get; set; }
         //public Dictionary<string, string> replies;
+
+        public string quoteUrl { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/Tag.cs

@@ -1,8 +1,19 @@
-namespace BirdsiteLive.ActivityPub.Models
+using System;
+
+namespace BirdsiteLive.ActivityPub.Models
 {
     public class Tag {
+        public TagResource icon { get; set; } = null;
+        public string id { get; set; }
         public string type { get; set; } //Hashtag
         public string href { get; set; } //https://mastodon.social/tags/app
         public string name { get; set; } //#app
+        public DateTime updated { get; set; } = default(DateTime);
+    }
+
+    public class TagResource
+    {
+        public string type { get; set; }
+        public string url { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/WebFingerData.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BirdsiteLive.ActivityPub.Models
+{
+    public class WebFingerData
+    {
+        public List<string> aliases { get; set; }
+
+        public List<WebFingerLink> links { get; set; }
+    }
+
+    public class WebFingerLink
+    {
+        public string href { get; set; }
+        public string rel { get; set; }
+        public string type { get; set; }
+        public string template { get; set; }
+    }
+}

src/BirdsiteLive.Common/BirdsiteLive.Common.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
   </PropertyGroup>
 
 </Project>

src/BirdsiteLive.Common/Regexes/UrlRegexes.cs

@@ -5,5 +5,7 @@ namespace BirdsiteLive.Common.Regexes
     public class UrlRegexes
     {
         public static readonly Regex Url = new Regex(@"(.?)(((http|ftp|https):\/\/)[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])?)");
+
+        public static readonly Regex Domain = new Regex(@"^[a-zA-Z0-9\-_]+(\.[a-zA-Z0-9\-_]+)+$");
     }
 }

src/BirdsiteLive.Common/Settings/InstanceSettings.cs

@@ -10,9 +10,26 @@
         public int MaxUsersCapacity { get; set; }
 
         public string UnlistedTwitterAccounts { get; set; }
+
+        public string TwitterDomain { get; set; }
+
+        public string InfoBanner { get; set; }
+
+        public string TwitterDomainLabel { get; set; }
+
+        public bool ShowAboutInstanceOnProfiles { get; set; }
+
+        public int MaxFollowsPerUser { get; set; }
+
+        public bool DiscloseInstanceRestrictions { get; set; }
+
         public string SensitiveTwitterAccounts { get; set; }
 
         public int FailingTwitterUserCleanUpThreshold { get; set; }
+
+        public int MaxStatusFetchAge { get; set; }
+
+        public bool EnableQuoteRT { get; set; }
         public int FailingFollowerCleanUpThreshold { get; set; } = -1;
 
         public int UserCacheCapacity { get; set; }

src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj

@@ -1,13 +1,13 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Asn1" Version="1.0.9" />
-    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
-    <PackageReference Include="Portable.BouncyCastle" Version="1.8.6.7" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="Portable.BouncyCastle" Version="1.9.0" />
   </ItemGroup>
 
 </Project>

src/BirdsiteLive.Domain/ActivityPubService.cs

@@ -2,6 +2,7 @@
 using System.Linq;
 using System.Net;
 using System.Net.Http;
+using System.Runtime.InteropServices;
 using System.Security.Cryptography;
 using System.Text;
 using System.Threading.Tasks;
@@ -16,10 +17,19 @@ namespace BirdsiteLive.Domain
 {
     public interface IActivityPubService
     {
+        Task<string> GetUserIdAsync(string acct);
         Task<Actor> GetUser(string objectId);
         Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null);
         Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost,
             string targetInbox);
+        Task DeleteUserAsync(string username, string targetHost, string targetInbox);
+		Task<WebFingerData> WebFinger(string account);
+    }
+
+    public class WebFinger
+    {
+        public string subject { get; set; }
+        public string[] aliases { get; set; }
     }
 
     public class ActivityPubService : IActivityPubService
@@ -39,9 +49,27 @@ namespace BirdsiteLive.Domain
         }
         #endregion
 
+        public async Task<string> GetUserIdAsync(string acct)
+        {
+            var splittedAcct = acct.Trim('@').Split('@');
+
+            var url = $"https://{splittedAcct[1]}/.well-known/webfinger?resource=acct:{splittedAcct[0]}@{splittedAcct[1]}";
+
+            var httpClient = _httpClientFactory.CreateClient("BirdsiteLIVE");
+            httpClient.DefaultRequestHeaders.Add("Accept", "application/json");
+            var result = await httpClient.GetAsync(url);
+
+            result.EnsureSuccessStatusCode();
+
+            var content = await result.Content.ReadAsStringAsync();
+
+            var actor = JsonConvert.DeserializeObject<WebFinger>(content);
+            return actor.aliases.FirstOrDefault();
+        }
+
         public async Task<Actor> GetUser(string objectId)
         {
-            var httpClient = _httpClientFactory.CreateClient();
+            var httpClient = _httpClientFactory.CreateClient("BirdsiteLIVE");
             httpClient.DefaultRequestHeaders.Add("Accept", "application/activity+json");
             var result = await httpClient.GetAsync(objectId);
 
@@ -57,6 +85,31 @@ namespace BirdsiteLive.Domain
             return actor;
         }
 
+        public async Task DeleteUserAsync(string username, string targetHost, string targetInbox)
+        {
+            try
+            {
+                var actor = UrlFactory.GetActorUrl(_instanceSettings.Domain, username);
+
+                var deleteUser = new ActivityDelete
+                {
+                    context = "https://www.w3.org/ns/activitystreams",
+                    id = $"{actor}#delete",
+                    type = "Delete",
+                    actor = actor,
+                    to = new [] { "https://www.w3.org/ns/activitystreams#Public" },
+                    apObject = actor
+                };
+
+                await PostDataAsync(deleteUser, targetHost, actor, targetInbox);
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, "Error deleting {Username} to {Host}{Inbox}", username, targetHost, targetInbox);
+                throw;
+            }
+        }
+
         public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox)
         {
             try
@@ -104,7 +157,7 @@ namespace BirdsiteLive.Domain
 
             var signature = _cryptoService.SignAndGetSignatureHeader(date, actorUrl, targetHost, digest, usedInbox);
 
-            var client = _httpClientFactory.CreateClient();
+            var client = _httpClientFactory.CreateClient("BirdsiteLIVE");
             var httpRequestMessage = new HttpRequestMessage
             {
                 Method = HttpMethod.Post,
@@ -123,5 +176,14 @@ namespace BirdsiteLive.Domain
             response.EnsureSuccessStatusCode();
             return response.StatusCode;
         }
+
+        public async Task<WebFingerData> WebFinger(string account)
+        {
+            var httpClient = _httpClientFactory.CreateClient("BirdsiteLIVE");
+            var result = await httpClient.GetAsync("https://" + account.Split('@')[1] + "/.well-known/webfinger?resource=acct:" + account);
+            var content = await result.Content.ReadAsStringAsync();
+
+            return JsonConvert.DeserializeObject<WebFingerData>(content);
+        }
     }
 }

src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
@@ -15,4 +15,8 @@
     <ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Folder Include="Enum\" />
+  </ItemGroup>
+
 </Project>

src/BirdsiteLive.Domain/Enum/MigrationTypeEnum.cs

@@ -0,0 +1,9 @@
+namespace BirdsiteLive.Domain.Enum
+{
+    public enum MigrationTypeEnum
+    {
+        Unknown = 0,
+        Migration = 1,
+        Deletion = 2
+    }
+}

src/BirdsiteLive.Domain/MigrationService.cs

@@ -0,0 +1,352 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using BirdsiteLive.Twitter;
+using System.Security.Cryptography;
+using System.Text;
+using System.Threading.Tasks;
+using BirdsiteLive.ActivityPub;
+using BirdsiteLive.ActivityPub.Models;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.ActivityPub.Converters;
+using BirdsiteLive.Common.Settings;
+using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Domain.Enum;
+using System.Net.Http;
+using BirdsiteLive.Common.Regexes;
+using Microsoft.Extensions.Logging;
+
+namespace BirdsiteLive.Domain
+{
+    public class MigrationService
+    {
+        private readonly InstanceSettings _instanceSettings;
+        private readonly ITheFedInfoService _theFedInfoService;
+        private readonly ITwitterTweetsService _twitterTweetsService;
+        private readonly IActivityPubService _activityPubService;
+        private readonly ITwitterUserDal _twitterUserDal;
+        private readonly IFollowersDal _followersDal;
+        private readonly IHttpClientFactory _httpClientFactory;
+        private readonly ILogger<MigrationService> _logger;
+
+        #region Ctor
+        public MigrationService(ITwitterTweetsService twitterTweetsService, IActivityPubService activityPubService, ITwitterUserDal twitterUserDal, IFollowersDal followersDal, InstanceSettings instanceSettings, ITheFedInfoService theFedInfoService, IHttpClientFactory httpClientFactory, ILogger<MigrationService> logger)
+        {
+            _twitterTweetsService = twitterTweetsService;
+            _activityPubService = activityPubService;
+            _twitterUserDal = twitterUserDal;
+            _followersDal = followersDal;
+            _instanceSettings = instanceSettings;
+            _theFedInfoService = theFedInfoService;
+            _httpClientFactory = httpClientFactory;
+            _logger = logger;
+        }
+        #endregion
+
+        public string GetMigrationCode(string acct)
+        {
+            var hash = GetHashString(acct);
+            return $"[[BirdsiteLIVE-MigrationCode|{hash.Substring(0, 10)}]]";
+        }
+
+        public string GetDeletionCode(string acct)
+        {
+            var hash = GetHashString(acct);
+            return $"[[BirdsiteLIVE-DeletionCode|{hash.Substring(0, 10)}]]";
+        }
+
+        public bool ValidateTweet(string acct, string tweetId, MigrationTypeEnum type)
+        {
+            string code;
+            if (type == MigrationTypeEnum.Migration)
+                code = GetMigrationCode(acct);
+            else if (type == MigrationTypeEnum.Deletion)
+                code = GetDeletionCode(acct);
+            else
+                throw new NotImplementedException();
+
+            var castedTweetId = ExtractedTweetId(tweetId);
+            var tweet = _twitterTweetsService.GetTweet(castedTweetId);
+
+            if (tweet == null)
+                throw new Exception("Tweet not found");
+
+            if (tweet.CreatorName.Trim().ToLowerInvariant() != acct.Trim().ToLowerInvariant())
+                throw new Exception($"Tweet not published by @{acct}");
+
+            if (!tweet.MessageContent.Contains(code))
+            {
+                var message = "Tweet don't have migration code";
+                if (type == MigrationTypeEnum.Deletion)
+                    message = "Tweet don't have deletion code";
+
+                throw new Exception(message);
+            }
+
+            return true;
+        }
+
+        private long ExtractedTweetId(string tweetId)
+        {
+            if (string.IsNullOrWhiteSpace(tweetId))
+                throw new ArgumentException("No provided Tweet ID");
+
+            long castedId;
+            if (long.TryParse(tweetId, out castedId))
+                return castedId;
+
+            var urlPart = tweetId.Split('/').LastOrDefault();
+            if (long.TryParse(urlPart, out castedId))
+                return castedId;
+
+            throw new ArgumentException("Unvalid Tweet ID");
+        }
+
+        public async Task<ValidatedFediverseUser> ValidateFediverseAcctAsync(string fediverseAcct)
+        {
+            if (string.IsNullOrWhiteSpace(fediverseAcct))
+                throw new ArgumentException("Please provide Fediverse account");
+
+            if (!fediverseAcct.Contains('@') || !fediverseAcct.StartsWith("@") || fediverseAcct.Trim('@').Split('@').Length != 2)
+                throw new ArgumentException("Please provide valid Fediverse handle");
+
+            var objectId = await _activityPubService.GetUserIdAsync(fediverseAcct);
+            var user = await _activityPubService.GetUser(objectId);
+
+            var result = new ValidatedFediverseUser
+            {
+                FediverseAcct = fediverseAcct,
+                ObjectId = objectId,
+                User = user,
+                IsValid = user != null
+            };
+
+            return result;
+        }
+
+        public async Task MigrateAccountAsync(ValidatedFediverseUser validatedUser, string acct)
+        {
+            // Apply moved to
+            var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct);
+            if (twitterAccount == null)
+            {
+                await _twitterUserDal.CreateTwitterUserAsync(acct, -1, validatedUser.ObjectId, validatedUser.FediverseAcct);
+                twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct);
+            }
+
+            twitterAccount.MovedTo = validatedUser.User.id;
+            twitterAccount.MovedToAcct = validatedUser.FediverseAcct;
+            twitterAccount.LastSync = DateTime.UtcNow;
+            await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount);
+
+            // Notify Followers
+            var message = $@"<p>[BSL MIRROR SERVICE NOTIFICATION]<br/>This bot has been disabled by its original owner.<br/>It has been redirected to {validatedUser.FediverseAcct}.</p>";
+            NotifyFollowers(acct, twitterAccount, message);
+        }
+
+        private void NotifyFollowers(string acct, SyncTwitterUser twitterAccount, string message)
+        {
+            var t = Task.Run(async () =>
+            {
+                var followers = await _followersDal.GetFollowersAsync(twitterAccount.Id);
+                foreach (var follower in followers)
+                {
+                    try
+                    {
+                        var noteId = Guid.NewGuid().ToString();
+                        var actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, acct);
+                        var noteUrl = UrlFactory.GetNoteUrl(_instanceSettings.Domain, acct, noteId);
+
+                        //var to = validatedUser.ObjectId;
+                        var to = follower.ActorId;
+                        var cc = new string[0];
+
+                        var note = new Note
+                        {
+                            id = noteUrl,
+
+                            published = DateTime.UtcNow.ToString("s") + "Z",
+                            url = noteUrl,
+                            attributedTo = actorUrl,
+
+                            to = new[] { to },
+                            cc = cc,
+
+                            content = message,
+                            tag = new Tag[]{
+                                new Tag()
+                                {
+                                    type = "Mention",
+                                    href = follower.ActorId,
+                                    name = $"@{follower.Acct}@{follower.Host}"
+                                }
+                            },
+                        };
+
+                        if (!string.IsNullOrWhiteSpace(follower.SharedInboxRoute))
+                            await _activityPubService.PostNewNoteActivity(note, acct, noteId, follower.Host, follower.SharedInboxRoute);
+                        else
+                            await _activityPubService.PostNewNoteActivity(note, acct, noteId, follower.Host, follower.InboxRoute);
+                    }
+                    catch (Exception e)
+                    {
+                        _logger.LogError(e, e.Message);
+                    }
+                }
+            });
+        }
+
+        public async Task DeleteAccountAsync(string acct)
+        {
+            // Apply deleted state
+            var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct);
+            if (twitterAccount == null)
+            {
+                await _twitterUserDal.CreateTwitterUserAsync(acct, -1);
+                twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct);
+            }
+
+            twitterAccount.Deleted = true;
+            twitterAccount.LastSync = DateTime.UtcNow;
+            await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount);
+
+            // Notify Followers
+            var message = $@"<p>[BSL MIRROR SERVICE NOTIFICATION]<br/>This bot has been deleted by its original owner.<br/></p>";
+            NotifyFollowers(acct, twitterAccount, message);
+
+            // Delete remote accounts
+            DeleteRemoteAccounts(acct);
+        }
+
+        private void DeleteRemoteAccounts(string acct)
+        {
+            var t = Task.Run(async () =>
+            {
+                var allUsers = await _followersDal.GetAllFollowersAsync();
+
+                var followersWtSharedInbox = allUsers
+                    .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
+                    .GroupBy(x => x.Host)
+                    .ToList();
+                foreach (var followerGroup in followersWtSharedInbox)
+                {
+                    var host = followerGroup.First().Host;
+                    var sharedInbox = followerGroup.First().SharedInboxRoute;
+
+                    var t1 = Task.Run(async () =>
+                    {
+                        try
+                        {
+                            await _activityPubService.DeleteUserAsync(acct, host, sharedInbox);
+                        }
+                        catch (Exception e)
+                        {
+                            _logger.LogError(e, e.Message);
+                        }
+                    });
+                }
+
+                var followerWtInbox = allUsers
+                    .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
+                    .ToList();
+                foreach (var followerGroup in followerWtInbox)
+                {
+                    var host = followerGroup.Host;
+                    var sharedInbox = followerGroup.InboxRoute;
+
+                    var t1 = Task.Run(async () =>
+                    {
+                        try
+                        {
+                            await _activityPubService.DeleteUserAsync(acct, host, sharedInbox);
+                        }
+                        catch (Exception e)
+                        {
+                            _logger.LogError(e, e.Message);
+                        }
+                    });
+                }
+            });
+        }
+
+        public async Task TriggerRemoteMigrationAsync(string id, string tweetIdStg, string handle)
+        {
+            var url = $"https://{{0}}/migration/move/{{1}}/{{2}}/{handle}";
+            await ProcessRemoteMigrationAsync(id, tweetIdStg, url);
+
+        }
+
+        public async Task TriggerRemoteDeleteAsync(string id, string tweetIdStg)
+        {
+            var url = $"https://{{0}}/migration/delete/{{1}}/{{2}}";
+            await ProcessRemoteMigrationAsync(id, tweetIdStg, url);
+        }
+
+        private async Task ProcessRemoteMigrationAsync(string id, string tweetIdStg, string urlPattern)
+        {
+            try
+            {
+                var instances = await RetrieveCompatibleBslInstancesAsync();
+                var tweetId = ExtractedTweetId(tweetIdStg);
+
+                foreach (var instance in instances)
+                {
+                    try
+                    {
+                        var host = instance.Host;
+                        if(!UrlRegexes.Domain.IsMatch(host)) continue;
+
+                        var url = string.Format(urlPattern, host, id, tweetId);
+
+                        var client = _httpClientFactory.CreateClient("BirdsiteLIVE");
+                        var result = await client.PostAsync(url, null);
+                        result.EnsureSuccessStatusCode();
+                    }
+                    catch (Exception e)
+                    {
+                        _logger.LogError(e, e.Message);
+                    }
+                }
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, e.Message);
+            }
+        }
+
+        private async Task<List<BslInstanceInfo>> RetrieveCompatibleBslInstancesAsync()
+        {
+            var instances = await _theFedInfoService.GetBslInstanceListAsync();
+            var filteredInstances = instances
+                .Where(x => x.Version >= new Version(0, 21, 0))
+                .Where(x => string.Compare(x.Host, 
+                    _instanceSettings.Domain, 
+                    StringComparison.InvariantCultureIgnoreCase) != 0)
+                .ToList();
+            return filteredInstances;
+        }
+
+        private byte[] GetHash(string inputString)
+        {
+            using (HashAlgorithm algorithm = SHA256.Create())
+                return algorithm.ComputeHash(Encoding.UTF8.GetBytes(inputString));
+        }
+
+        private string GetHashString(string inputString)
+        {
+            StringBuilder sb = new StringBuilder();
+            foreach (byte b in GetHash(inputString))
+                sb.Append(b.ToString("X2"));
+
+            return sb.ToString();
+        }
+    }
+
+    public class ValidatedFediverseUser
+    {
+        public string FediverseAcct { get; set; }
+        public string ObjectId { get; set; }
+        public Actor User { get; set; }
+        public bool IsValid { get; set; }
+    }
+}

src/BirdsiteLive.Domain/Repository/ModerationRepository.cs

@@ -11,6 +11,12 @@ namespace BirdsiteLive.Domain.Repository
     {
         ModerationTypeEnum GetModerationType(ModerationEntityTypeEnum type);
         ModeratedTypeEnum CheckStatus(ModerationEntityTypeEnum type, string entity);
+
+        IEnumerable<string> GetWhitelistedFollowers();
+        IEnumerable<string> GetBlacklistedFollowers();
+        IEnumerable<string> GetWhitelistedAccounts();
+
+        IEnumerable<string> GetBlacklistedAccounts();
     }
 
     public class ModerationRepository : IModerationRepository
@@ -23,9 +29,13 @@ namespace BirdsiteLive.Domain.Repository
         private readonly Dictionary<ModerationEntityTypeEnum, ModerationTypeEnum> _modMode =
             new Dictionary<ModerationEntityTypeEnum, ModerationTypeEnum>();
 
+        private readonly ModerationSettings _settings;
+
         #region Ctor
         public ModerationRepository(ModerationSettings settings)
         {
+            _settings = settings;
+
             var parsedFollowersWhiteListing = PatternsParser.Parse(settings.FollowersWhiteListing);
             var parsedFollowersBlackListing = PatternsParser.Parse(settings.FollowersBlackListing);
             var parsedTwitterAccountsWhiteListing = PatternsParser.Parse(settings.TwitterAccountsWhiteListing);
@@ -123,6 +133,35 @@ namespace BirdsiteLive.Domain.Repository
                     throw new ArgumentOutOfRangeException();
             }
         }
+
+        private char GetSplitChar(string entry)
+        {
+            var separationChar = '|';
+            if (entry.Contains(";")) separationChar = ';';
+            else if (entry.Contains(",")) separationChar = ',';
+
+            return separationChar;
+        }
+
+        public IEnumerable<string> GetWhitelistedFollowers()
+        {
+            return _settings.FollowersWhiteListing.Split(GetSplitChar(_settings.FollowersWhiteListing));
+        }
+
+        public IEnumerable<string> GetBlacklistedFollowers()
+        {
+            return _settings.FollowersBlackListing.Split(GetSplitChar(_settings.FollowersBlackListing));
+        }
+
+        public IEnumerable<string> GetWhitelistedAccounts()
+        {
+            return _settings.TwitterAccountsWhiteListing.Split(GetSplitChar(_settings.TwitterAccountsWhiteListing));
+        }
+
+        public IEnumerable<string> GetBlacklistedAccounts()
+        {
+            return _settings.TwitterAccountsBlackListing.Split(GetSplitChar(_settings.TwitterAccountsBlackListing));
+        }
     }
 
     public enum ModerationEntityTypeEnum

src/BirdsiteLive.Domain/StatusService.cs

@@ -52,8 +52,8 @@ namespace BirdsiteLive.Domain
             
             string summary = null;
             var sensitive = _publicationRepository.IsSensitive(username);
-            if (sensitive)
-                summary = "Potential Content Warning";
+            if (sensitive || tweet.IsSensitive)
+                summary = "Sensitive Content";
 
             var extractedTags = _statusExtractor.Extract(tweet.MessageContent);
             _statisticsHandler.ExtractedStatus(extractedTags.tags.Count(x => x.type == "Mention"));
@@ -73,6 +73,9 @@ namespace BirdsiteLive.Domain
             if (tweet.InReplyToStatusId != default)
                 inReplyTo = $"https://{_instanceSettings.Domain}/users/{tweet.InReplyToAccount.ToLowerInvariant()}/statuses/{tweet.InReplyToStatusId}";
 
+            if( tweet.QuoteTweetUrl != null )
+                content += $@"<span class=""quote-inline""><br><br>RT: <a href=""{tweet.QuoteTweetUrl}"">{tweet.QuoteTweetUrl}</a></span>";
+
             var note = new Note
             {
                 id = noteUrl,
@@ -86,11 +89,13 @@ namespace BirdsiteLive.Domain
                 to = new[] { to },
                 cc = cc,
 
-                sensitive = sensitive,
+                sensitive = tweet.IsSensitive || sensitive,
                 summary = summary,
                 content = $"<p>{content}</p>",
                 attachment = Convert(tweet.Media),
-                tag = extractedTags.tags
+                tag = extractedTags.tags,
+
+                quoteUrl = tweet.QuoteTweetUrl
             };
 
             return note;

src/BirdsiteLive.Domain/TheFedInfoService.cs

@@ -0,0 +1,162 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+
+namespace BirdsiteLive.Domain
+{
+    public interface ITheFedInfoService
+    {
+        Task<List<BslInstanceInfo>> GetBslInstanceListAsync();
+    }
+
+    public class TheFedInfoService : ITheFedInfoService
+    {
+        private readonly IHttpClientFactory _httpClientFactory;
+        
+        #region Ctor
+        public TheFedInfoService(IHttpClientFactory httpClientFactory)
+        {
+            _httpClientFactory = httpClientFactory;
+        }
+        #endregion
+
+        public async Task<List<BslInstanceInfo>> GetBslInstanceListAsync()
+        {
+            var cancellationToken = CancellationToken.None;
+
+            var result = await CallGraphQlAsync<MyResponseData>(
+                new Uri("https://the-federation.info/graphql"),
+                HttpMethod.Get,
+                "query ($platform: String!) { nodes(platform: $platform) { host, version } }",
+                new
+                {
+                    platform = "birdsitelive",
+                },
+                cancellationToken);
+
+            var convertedResults = ConvertResults(result);
+            return convertedResults;
+        }
+
+        private List<BslInstanceInfo> ConvertResults(GraphQLResponse<MyResponseData> qlData)
+        {
+            var results = new List<BslInstanceInfo>();
+
+            foreach (var instanceInfo in qlData.Data.Nodes)
+            {
+                try
+                {
+                    var rawVersion = instanceInfo.Version.Split('+').First();
+                    if (string.IsNullOrWhiteSpace(rawVersion)) continue;
+                    var version = Version.Parse(rawVersion);
+                    if(version <= new Version(0,1,0)) continue;
+
+                    var instance = new BslInstanceInfo
+                    {
+                        Host = instanceInfo.Host,
+                        Version = version
+                    };
+                    results.Add(instance);
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine(e);
+                }
+            }
+
+            return results;
+        }
+
+        private async Task<GraphQLResponse<TResponse>> CallGraphQlAsync<TResponse>(Uri endpoint, HttpMethod method, string query, object variables, CancellationToken cancellationToken)
+        {
+            var content = new StringContent(SerializeGraphQlCall(query, variables), Encoding.UTF8, "application/json");
+            var httpRequestMessage = new HttpRequestMessage
+            {
+                Method = method,
+                Content = content,
+                RequestUri = endpoint,
+            };
+            //add authorization headers if necessary here
+            httpRequestMessage.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
+            var httpClient = _httpClientFactory.CreateClient("BirdsiteLIVE");
+            using (var response = await httpClient.SendAsync(httpRequestMessage, cancellationToken))
+            {
+                //if (response.IsSuccessStatusCode)
+                if (response?.Content.Headers.ContentType?.MediaType == "application/json")
+                {
+                    var responseString = await response.Content.ReadAsStringAsync(); //cancellationToken supported for .NET 5/6
+                    return DeserializeGraphQlCall<TResponse>(responseString);
+                }
+                else
+                {
+                    throw new ApplicationException($"Unable to contact '{endpoint}': {response.StatusCode} - {response.ReasonPhrase}");
+                }
+            }
+        }
+
+        private string SerializeGraphQlCall(string query, object variables)
+        {
+            var sb = new StringBuilder();
+            var textWriter = new StringWriter(sb);
+            var serializer = new JsonSerializer();
+            serializer.Serialize(textWriter, new
+            {
+                query = query,
+                variables = variables,
+            });
+            return sb.ToString();
+        }
+
+        private GraphQLResponse<TResponse> DeserializeGraphQlCall<TResponse>(string response)
+        {
+            var serializer = new JsonSerializer();
+            var stringReader = new StringReader(response);
+            var jsonReader = new JsonTextReader(stringReader);
+            var result = serializer.Deserialize<GraphQLResponse<TResponse>>(jsonReader);
+            return result;
+        }
+        
+        private class GraphQLResponse<TResponse>
+        {
+            public List<GraphQLError> Errors { get; set; }
+            public TResponse Data { get; set; }
+        }
+
+        private class GraphQLError
+        {
+            public string Message { get; set; }
+            public List<GraphQLErrorLocation> Locations { get; set; }
+            public List<object> Path { get; set; } //either int or string
+        }
+
+        private class GraphQLErrorLocation
+        {
+            public int Line { get; set; }
+            public int Column { get; set; }
+        }
+        
+        private class MyResponseData
+        {
+            public Node[] Nodes { get; set; }
+        }
+
+        private class Node
+        {
+            public string Host { get; set; }
+            public string Version { get; set; }
+        }
+    }
+
+    public class BslInstanceInfo
+    {
+        public string Host { get; set; }
+        public Version Version { get; set; }
+    }
+}

src/BirdsiteLive.Domain/UserService.cs

@@ -11,6 +11,8 @@ using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.Common.Regexes;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Cryptography;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Domain.BusinessUseCases;
 using BirdsiteLive.Domain.Repository;
 using BirdsiteLive.Domain.Statistics;
@@ -24,7 +26,7 @@ namespace BirdsiteLive.Domain
 {
     public interface IUserService
     {
-        Actor GetUser(TwitterUser twitterUser);
+        Actor GetUser(TwitterUser twitterUser, SyncTwitterUser dbTwitterUser);
         Task<bool> FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders, ActivityFollow activity, string body);
         Task<bool> UndoFollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders, ActivityUndoFollow activity, string body);
 
@@ -48,8 +50,10 @@ namespace BirdsiteLive.Domain
 
         private readonly IModerationRepository _moderationRepository;
 
+        private readonly IFollowersDal _followerDal;
+
         #region Ctor
-        public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler, ITwitterUserService twitterUserService, IModerationRepository moderationRepository, IProcessDeleteUser processDeleteUser)
+        public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler, ITwitterUserService twitterUserService, IModerationRepository moderationRepository, IFollowersDal followerDal, IProcessDeleteUser processDeleteUser)
         {
             _instanceSettings = instanceSettings;
             _cryptoService = cryptoService;
@@ -60,11 +64,12 @@ namespace BirdsiteLive.Domain
             _statisticsHandler = statisticsHandler;
             _twitterUserService = twitterUserService;
             _moderationRepository = moderationRepository;
+            _followerDal = followerDal;
             _processDeleteUser = processDeleteUser;
         }
         #endregion
 
-        public Actor GetUser(TwitterUser twitterUser)
+        public Actor GetUser(TwitterUser twitterUser, SyncTwitterUser dbTwitterUser)
         {
             var actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, twitterUser.Acct);
             var acct = twitterUser.Acct.ToLowerInvariant();
@@ -79,6 +84,34 @@ namespace BirdsiteLive.Domain
                 _statisticsHandler.ExtractedDescription(extracted.tags.Count(x => x.type == "Mention"));
             }
 
+            var attachments = new List<UserAttachment>();
+            attachments.Add(new UserAttachment
+            {
+                type = "PropertyValue",
+                name = _instanceSettings.TwitterDomainLabel != "" ? _instanceSettings.TwitterDomainLabel : _instanceSettings.TwitterDomain,
+                value = $"<a href=\"https://{_instanceSettings.TwitterDomain}/{acct}\" rel=\"me nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">{_instanceSettings.TwitterDomain}/{acct}</span></a>"
+            });
+
+            if(_instanceSettings.TwitterDomain != "twitter.com")
+            {
+                attachments.Add(new UserAttachment
+                {
+                    type = "PropertyValue",
+                    name = "Twitter",
+                    value = $"twitter.com/{acct}"
+                });
+            }
+
+            if (_instanceSettings.ShowAboutInstanceOnProfiles)
+            {
+                attachments.Add(new UserAttachment
+                {
+                    type = "PropertyValue",
+                    name = $"About {_instanceSettings.Name}",
+                    value = $"<a href=\"https://{_instanceSettings.Domain}/About\" rel=\"nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">{_instanceSettings.Domain}/About</span></a>"
+                });
+            }
+
             var user = new Actor
             {
                 id = actorUrl,
@@ -87,9 +120,10 @@ namespace BirdsiteLive.Domain
                 preferredUsername = acct,
                 name = twitterUser.Name,
                 inbox = $"{actorUrl}/inbox",
-                summary = description,
+                summary = "[UNOFFICIAL MIRROR: This is a view of Twitter using ActivityPub]<br/><br/>" + description,
                 url = actorUrl,
                 manuallyApprovesFollowers = twitterUser.Protected,
+                discoverable = false,
                 publicKey = new PublicKey()
                 {
                     id = $"{actorUrl}#main-key",
@@ -111,15 +145,49 @@ namespace BirdsiteLive.Domain
                     new UserAttachment
                     {
                         type = "PropertyValue",
-                        name = "Official",
+                        name = "Official Account",
                         value = $"<a href=\"https://twitter.com/{acct}\" rel=\"me nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">twitter.com/{acct}</span></a>"
+                    },
+                    new UserAttachment
+                    {
+                        type = "PropertyValue",
+                        name = "Disclaimer",
+                        value = "This is an automatically created and managed mirror profile from Twitter. While it reflects exactly the content of the original account, it doesn't provide support for interactions and replies. It is an equivalent view from other 3rd party Twitter client apps and uses the same technical means to provide it."
+                    },
+                    new UserAttachment
+                    {
+                        type = "PropertyValue",
+                        name = "Take control of this account",
+                        value = $"<a href=\"https://{_instanceSettings.Domain}/migration/move/{acct}\" rel=\"me nofollow noopener noreferrer\" target=\"_blank\">MANAGE</a>"
                     }
                 },
                 endpoints = new EndPoints
                 {
                     sharedInbox = $"https://{_instanceSettings.Domain}/inbox"
-                }
+                },
+                movedTo = dbTwitterUser?.MovedTo
             };
+
+            if (twitterUser.Verified)
+            {
+                user.tag = new List<Tag>
+                {
+                    new Tag
+                    {
+                        icon = new TagResource
+                        {
+                            type = "Image",
+                            url = "https://" + _instanceSettings.Domain + "/verified.png"
+                        },
+                        id = "https://" + _instanceSettings.Domain + "/verified.png",
+                        name = ":verified:",
+                        type = "Emoji"
+                    }
+                };
+
+                user.name += " :verified:";
+            }
+
             return user;
         }
 
@@ -161,6 +229,16 @@ namespace BirdsiteLive.Domain
                     return await SendRejectFollowAsync(activity, followerHost);
             }
 
+            // Validate follower count < MaxFollowsPerUser
+            if (_instanceSettings.MaxFollowsPerUser > 0) {
+                var follower = await _followerDal.GetFollowerAsync(followerUserName, followerHost);
+
+                if (follower != null && follower.Followings.Count + 1 > _instanceSettings.MaxFollowsPerUser)
+                {
+                    return await SendRejectFollowAsync(activity, followerHost);
+                }
+            }
+
             // Validate User Protected
             var user = _twitterUserService.GetUser(twitterUser);
             if (!user.Protected)

src/BirdsiteLive.Moderation/BirdsiteLive.Moderation.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

src/BirdsiteLive.Moderation/Processors/TwitterAccountModerationProcessor.cs

@@ -29,7 +29,7 @@ namespace BirdsiteLive.Moderation.Processors
         {
             if (type == ModerationTypeEnum.None) return;
             
-            var twitterUsers = await _twitterUserDal.GetAllTwitterUsersAsync();
+            var twitterUsers = await _twitterUserDal.GetAllTwitterUsersAsync(false);
 
             foreach (var user in twitterUsers)
             {

src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
 

src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs

@@ -49,12 +49,12 @@ namespace BirdsiteLive.Pipeline.Processors
                 {
                     var tweetId = tweets.Last().Id;
                     var now = DateTime.UtcNow;
-                    await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now);
+                    await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now, user.MovedTo, user.MovedToAcct, user.Deleted);
                 }
                 else
                 {
                     var now = DateTime.UtcNow;
-                    await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now);
+                    await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now, user.MovedTo, user.MovedToAcct, user.Deleted);
                 }
             }
 

src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs

@@ -39,7 +39,7 @@ namespace BirdsiteLive.Pipeline.Processors
                 try
                 {
                     var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync();
-                    var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber);
+                    var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber, false);
 
                     var userCount = users.Any() ? users.Length : 1;
                     var splitNumber = (int) Math.Ceiling(userCount / 15d);

src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs

@@ -3,6 +3,8 @@ using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Moderation.Actions;
 using BirdsiteLive.Pipeline.Contracts;
 using BirdsiteLive.Pipeline.Models;
 using Microsoft.Extensions.Logging;
@@ -13,12 +15,14 @@ namespace BirdsiteLive.Pipeline.Processors
     {
         private readonly ITwitterUserDal _twitterUserDal;
         private readonly ILogger<SaveProgressionProcessor> _logger;
+        private readonly IRemoveTwitterAccountAction _removeTwitterAccountAction;
 
         #region Ctor
-        public SaveProgressionProcessor(ITwitterUserDal twitterUserDal, ILogger<SaveProgressionProcessor> logger)
+        public SaveProgressionProcessor(ITwitterUserDal twitterUserDal, ILogger<SaveProgressionProcessor> logger, IRemoveTwitterAccountAction removeTwitterAccountAction)
         {
             _twitterUserDal = twitterUserDal;
             _logger = logger;
+            _removeTwitterAccountAction = removeTwitterAccountAction;
         }
         #endregion
 
@@ -28,28 +32,23 @@ namespace BirdsiteLive.Pipeline.Processors
             {
                 if (userWithTweetsToSync.Tweets.Length == 0)
                 {
-                    _logger.LogWarning("No tweets synchronized");
+                    _logger.LogInformation("No tweets synchronized");
+                    await UpdateUserSyncDateAsync(userWithTweetsToSync.User);
                     return;
                 }
                 if(userWithTweetsToSync.Followers.Length == 0)
                 {
-                    _logger.LogWarning("No Followers found for {User}", userWithTweetsToSync.User.Acct);
+                    _logger.LogInformation("No Followers found for {User}", userWithTweetsToSync.User.Acct);
+                    await _removeTwitterAccountAction.ProcessAsync(userWithTweetsToSync.User);
                     return;
                 }
             
                 var userId = userWithTweetsToSync.User.Id;
                 var followingSyncStatuses = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).ToList();
-            
-                if (followingSyncStatuses.Count == 0)
-                {
-                    _logger.LogWarning("No Followers sync found for {User}, Id: {UserId}", userWithTweetsToSync.User.Acct, userId);
-                    return;
-                }
-
                 var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();
                 var minimumSync = followingSyncStatuses.Min();
                 var now = DateTime.UtcNow;
-                await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now);
+                await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now, userWithTweetsToSync.User.MovedTo, userWithTweetsToSync.User.MovedToAcct, userWithTweetsToSync.User.Deleted);
             }
             catch (Exception e)
             {
@@ -57,5 +56,11 @@ namespace BirdsiteLive.Pipeline.Processors
                 throw;
             }
         }
+
+        private async Task UpdateUserSyncDateAsync(SyncTwitterUser user)
+        {
+            user.LastSync = DateTime.UtcNow;
+            await _twitterUserDal.UpdateTwitterUserAsync(user);
+        }
     }
 }

src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

src/BirdsiteLive.Twitter/CachedTwitterTweetService.cs

@@ -0,0 +1,63 @@
+using System;
+using BirdsiteLive.Twitter.Models;
+using Microsoft.Extensions.Caching.Memory;
+
+namespace BirdsiteLive.Twitter
+{
+    public interface ICachedTwitterTweetsService : ITwitterTweetsService
+    {
+        void PurgeTweet(long statusId);
+    }
+
+    public class CachedTwitterTweetsService : ICachedTwitterTweetsService
+    {
+        private readonly ITwitterTweetsService _twitterService;
+
+        private MemoryCache _tweetCache = new MemoryCache(new MemoryCacheOptions()
+        {
+            SizeLimit = 5000
+        });
+        private MemoryCacheEntryOptions _cacheEntryOptions = new MemoryCacheEntryOptions()
+            .SetSize(1)//Size amount
+                       //Priority on removing when reaching size limit (memory pressure)
+            .SetPriority(CacheItemPriority.High)
+            // Keep in cache for this time, reset time if accessed.
+            // We set this lower than a user's in case they delete this Tweet for some reason; we don't need that cached.
+            .SetSlidingExpiration(TimeSpan.FromHours(2))
+            // Remove from cache after this time, regardless of sliding expiration
+            .SetAbsoluteExpiration(TimeSpan.FromDays(7));
+
+        #region Ctor
+        public CachedTwitterTweetsService(ITwitterTweetsService twitterService)
+        {
+            _twitterService = twitterService;
+        }
+
+        public ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1)
+        {
+            // This sounds like it'd be silly to cache; pass this directly to TwitterService.
+            // Theoretically this shouldn't be called more than once every 15 min anyway?
+            return _twitterService.GetTimeline(username, nberTweets, fromTweetId);
+        }
+
+        public ExtractedTweet GetTweet(long statusId)
+        {
+            if(!_tweetCache.TryGetValue(statusId, out ExtractedTweet tweet))
+            {
+                tweet = _twitterService.GetTweet(statusId);
+
+                // Unlike with the user cache, save the null value anyway to prevent (quicker) API exhaustion.
+                // It's incredibly unlikely that a tweet with this ID is going to magickally appear within 2 hours.
+                _tweetCache.Set(statusId, tweet, _cacheEntryOptions); 
+            }
+
+            return tweet;
+        }
+        #endregion
+
+        public void PurgeTweet(long statusId)
+        {
+            _tweetCache.Remove(statusId);
+        }
+    }
+}

src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Twitter.Models;
 using Tweetinvi.Models;
 using Tweetinvi.Models.Entities;
@@ -15,6 +16,15 @@ namespace BirdsiteLive.Twitter.Extractors
 
     public class TweetExtractor : ITweetExtractor
     {
+        private readonly InstanceSettings _instanceSettings;
+
+        #region Ctor
+        public TweetExtractor(InstanceSettings instanceSettings)
+        {
+            this._instanceSettings = instanceSettings;
+        }
+        #endregion
+
         public ExtractedTweet Extract(ITweet tweet)
         {
             var extractedTweet = new ExtractedTweet
@@ -28,7 +38,10 @@ namespace BirdsiteLive.Twitter.Extractors
                 IsReply = tweet.InReplyToUserId != null,
                 IsThread = tweet.InReplyToUserId != null && tweet.InReplyToUserId == tweet.CreatedBy.Id,
                 IsRetweet = tweet.IsRetweet || tweet.QuotedStatusId != null,
-                RetweetUrl = ExtractRetweetUrl(tweet)
+                RetweetUrl = ExtractRetweetUrl(tweet),
+                IsSensitive = tweet.PossiblySensitive,
+                QuoteTweetUrl = tweet.QuotedStatusId != null ? "https://" + _instanceSettings.Domain + "/users/" + tweet.QuotedTweet.CreatedBy.ScreenName + "/statuses/" + tweet.QuotedStatusId : null,
+                CreatorName = tweet.CreatedBy.UserIdentifier.ScreenName
             };
 
             return extractedTweet;
@@ -40,7 +53,10 @@ namespace BirdsiteLive.Twitter.Extractors
             {
                 if (tweet.RetweetedTweet != null)
                 {
-                    return tweet.RetweetedTweet.Url;
+                    var uri = new UriBuilder(tweet.RetweetedTweet.Url);
+                    uri.Host = _instanceSettings.TwitterDomain;
+
+                    return uri.Uri.ToString();
                 }
                 if (tweet.FullText.Contains("https://t.co/"))
                 {
@@ -71,7 +87,11 @@ namespace BirdsiteLive.Twitter.Extractors
                     message = message.Replace(tweetUrl, string.Empty).Trim();
             }
 
-            if (tweet.QuotedTweet != null) message = $"[Quote {{RT}}]{Environment.NewLine}{message}";
+            if (tweet.QuotedTweet != null && ! _instanceSettings.EnableQuoteRT)
+            {
+                message = $"[Quote {{RT}}]{Environment.NewLine}{message}";
+            }
+
             if (tweet.IsRetweet)
             {
                 if (tweet.RetweetedTweet != null && !message.StartsWith("RT"))
@@ -84,7 +104,26 @@ namespace BirdsiteLive.Twitter.Extractors
 
             // Expand URLs
             foreach (var url in tweet.Urls.OrderByDescending(x => x.URL.Length))
+            {
+                // A bit of a hack
+                if (url.ExpandedURL == tweet.QuotedTweet?.Url && _instanceSettings.EnableQuoteRT)
+                {
+                    url.ExpandedURL = "";
+                } else
+                {
+                    var linkUri = new UriBuilder(url.ExpandedURL);
+
+                    if (linkUri.Host == "twitter.com")
+                    {
+                        linkUri.Host = _instanceSettings.TwitterDomain;
+                        url.ExpandedURL = linkUri.Uri.ToString();
+                    }
+                }
+
                 message = message.Replace(url.URL, url.ExpandedURL);
+            }
+
+            // Hack
 
             return message;
         }
@@ -102,6 +141,7 @@ namespace BirdsiteLive.Twitter.Extractors
                 var mediaType = GetMediaType(m.MediaType, mediaUrl);
                 if (mediaType == null) continue;
 
+
                 var att = new ExtractedMedia
                 {
                     MediaType = mediaType,

src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs

@@ -15,5 +15,8 @@ namespace BirdsiteLive.Twitter.Models
         public bool IsThread { get; set; }
         public bool IsRetweet { get; set; }
         public string RetweetUrl { get; set; }
+        public bool IsSensitive { get; set; }
+        public string QuoteTweetUrl { get; set; }
+        public string CreatorName { get; set; }
     }
 }

src/BirdsiteLive.Twitter/Models/TwitterUser.cs

@@ -11,5 +11,7 @@
         public string Acct { get; set; }
         public string ProfileBannerURL { get; set; }
         public bool Protected { get; set; }
+        public bool Sensitive { get; set; }
+        public bool Verified { get; set; }
     }
 }

src/BirdsiteLive.Twitter/TwitterUserService.cs

@@ -91,7 +91,8 @@ namespace BirdsiteLive.Twitter
                 ProfileImageUrl = user.ProfileImageUrlFullSize.Replace("http://", "https://"),
                 ProfileBackgroundImageUrl = user.ProfileBackgroundImageUrlHttps,
                 ProfileBannerURL = user.ProfileBannerURL,
-                Protected = user.Protected
+                Protected = user.Protected,
+                Verified = user.Verified
             };
         }
 

src/BirdsiteLive/BirdsiteLive.csproj

@@ -1,17 +1,18 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <UserSecretsId>d21486de-a812-47eb-a419-05682bb68856</UserSecretsId>
     <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
-    <Version>0.20.0</Version>
+    <Version>0.22.0</Version>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Lamar.Microsoft.DependencyInjection" Version="5.0.0" />
-    <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.16.0" />
-    <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.10.8" />
-    <PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="3.1.3" />
+    <PackageReference Include="Lamar.Microsoft.DependencyInjection" Version="12.0.0" />
+    <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.21.0" />
+    <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.18.1" />
+    <PackageReference Include="Microsoft.VisualStudio.Web.BrowserLink" Version="2.2.0" />
+    <PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="3.1.5" />
   </ItemGroup>
 
   <ItemGroup>

src/BirdsiteLive/Component/NodeInfoViewComponent.cs

@@ -14,10 +14,10 @@ namespace BirdsiteLive.Component
     public class NodeInfoViewComponent : ViewComponent
     {
         private readonly IModerationRepository _moderationRepository;
-        private readonly ICachedStatisticsService _cachedStatisticsService;
+        private readonly IAboutPageService _cachedStatisticsService;
         
         #region Ctor
-        public NodeInfoViewComponent(IModerationRepository moderationRepository, ICachedStatisticsService cachedStatisticsService)
+        public NodeInfoViewComponent(IModerationRepository moderationRepository, IAboutPageService cachedStatisticsService)
         {
             _moderationRepository = moderationRepository;
             _cachedStatisticsService = cachedStatisticsService;
@@ -29,7 +29,7 @@ namespace BirdsiteLive.Component
             var followerPolicy = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.Follower);
             var twitterAccountPolicy = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.TwitterAccount);
 
-            var statistics = await _cachedStatisticsService.GetStatisticsAsync();
+            var statistics = await _cachedStatisticsService.GetAboutPageDataAsync();
 
             var viewModel = new NodeInfoViewModel
             {
@@ -37,7 +37,8 @@ namespace BirdsiteLive.Component
                                       twitterAccountPolicy == ModerationTypeEnum.BlackListing,
                 WhitelistingEnabled = followerPolicy == ModerationTypeEnum.WhiteListing ||
                                       twitterAccountPolicy == ModerationTypeEnum.WhiteListing,
-                InstanceSaturation = statistics.Saturation
+                InstanceSaturation = statistics.Saturation,
+                DiscloseRestrictions = statistics.Settings.DiscloseInstanceRestrictions
             };
             
             //viewModel = new NodeInfoViewModel
@@ -55,5 +56,6 @@ namespace BirdsiteLive.Component
         public bool BlacklistingEnabled { get; set; }
         public bool WhitelistingEnabled { get; set; }
         public int InstanceSaturation { get; set; }
+        public bool DiscloseRestrictions { get; set; }
     }
 }

src/BirdsiteLive/Controllers/AboutController.cs

@@ -10,49 +10,21 @@ namespace BirdsiteLive.Controllers
 {
     public class AboutController : Controller
     {
-        private readonly IModerationRepository _moderationRepository;
-        private readonly ICachedStatisticsService _cachedStatisticsService;
+        private readonly IAboutPageService _aboutPageService;
 
         #region Ctor
-        public AboutController(IModerationRepository moderationRepository, ICachedStatisticsService cachedStatisticsService)
+        public AboutController(IAboutPageService cachedStatisticsService)
         {
-            _moderationRepository = moderationRepository;
-            _cachedStatisticsService = cachedStatisticsService;
+            _aboutPageService = cachedStatisticsService;
         }
         #endregion
 
         public async Task<IActionResult> Index()
         {
-            var stats = await _cachedStatisticsService.GetStatisticsAsync();
+            var stats = await _aboutPageService.GetAboutPageDataAsync();
             return View(stats);
         }
-
-        public IActionResult Blacklisting()
-        {
-            var status = GetModerationStatus();
-            return View("Blacklisting", status);
-        }
-
-        public IActionResult Whitelisting()
-        {
-            var status = GetModerationStatus();
-            return View("Whitelisting", status);
-        }
-
-        private ModerationStatus GetModerationStatus()
-        {
-            var status = new ModerationStatus
-            {
-                Followers = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.Follower),
-                TwitterAccounts = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.TwitterAccount)
-            };
-            return status;
-        }
     }
 
-    public class ModerationStatus
-    {
-        public ModerationTypeEnum Followers { get; set; }
-        public ModerationTypeEnum TwitterAccounts { get; set; }
-    }
+
 }

src/BirdsiteLive/Controllers/DebugingController.cs

@@ -59,17 +59,22 @@ namespace BirdsiteLive.Controllers
         [HttpPost]
         public async Task<IActionResult> PostNote()
         {
-            var username = "gra";
+            var username = "twitter";
             var actor = $"https://{_instanceSettings.Domain}/users/{username}";
-            var targetHost = "mastodon.technology";
-            var target = $"{targetHost}/users/testtest";
-            var inbox = $"/users/testtest/inbox";
+            var targetHost = "ioc.exchange";
+            var target = $"https://{targetHost}/users/test";
+            //var inbox = $"/users/testtest/inbox";
+            var inbox = $"/inbox";
 
             var noteGuid = Guid.NewGuid();
             var noteId = $"https://{_instanceSettings.Domain}/users/{username}/statuses/{noteGuid}";
             var noteUrl = $"https://{_instanceSettings.Domain}/@{username}/{noteGuid}";
 
             var to = $"{actor}/followers";
+            to = target;
+
+            var cc = new[] { "https://www.w3.org/ns/activitystreams#Public" };
+            cc = new string[0];
 
             var now = DateTime.UtcNow;
             var nowString = now.ToString("s") + "Z";
@@ -82,7 +87,7 @@ namespace BirdsiteLive.Controllers
                 actor = actor,
                 published = nowString,
                 to = new[] { to },
-                //cc = new [] { "https://www.w3.org/ns/activitystreams#Public" },
+                cc = cc,
                 apObject = new Note()
                 {
                     id = noteId,
@@ -94,7 +99,8 @@ namespace BirdsiteLive.Controllers
 
                     // Unlisted
                     to = new[] { to },
-                    cc = new[] { "https://www.w3.org/ns/activitystreams#Public" },
+                    cc = cc,
+                    //cc = new[] { "https://www.w3.org/ns/activitystreams#Public" },
 
                     //// Public
                     //to = new[] { "https://www.w3.org/ns/activitystreams#Public" },
@@ -102,8 +108,16 @@ namespace BirdsiteLive.Controllers
 
                     sensitive = false,
                     content = "<p>TEST PUBLIC</p>",
+                    //content = "<p><span class=\"h-card\"><a href=\"https://ioc.exchange/users/test\" class=\"u-url mention\">@<span>test</span></a></span> test</p>",
                     attachment = new Attachment[0],
-                    tag = new Tag[0]
+                    tag = new Tag[]{
+                        new Tag()
+                        {
+                            type = "Mention",
+                            href = target,
+                            name = "@test@ioc.exchange"
+                        }
+                    },
                 }
             };
 
@@ -125,6 +139,17 @@ namespace BirdsiteLive.Controllers
             await _userService.SendRejectFollowAsync(activityFollow, "mastodon.technology");
             return View("Index");
         }
+
+        [HttpPost]
+        public async Task<IActionResult> PostDeleteUser()
+        {
+            var userName = "twitter";
+            var host = "ioc.exchange";
+            var inbox = "/inbox";
+
+            await _activityPubService.DeleteUserAsync(userName, host, inbox);
+            return View("Index");
+        }
     }
     #endif
 

src/BirdsiteLive/Controllers/HomeController.cs

@@ -6,21 +6,24 @@ using System.Threading.Tasks;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
 using BirdsiteLive.Models;
+using BirdsiteLive.Common.Settings;
 
 namespace BirdsiteLive.Controllers
 {
     public class HomeController : Controller
     {
         private readonly ILogger<HomeController> _logger;
+        private readonly InstanceSettings _instanceSettings;
 
-        public HomeController(ILogger<HomeController> logger)
+        public HomeController(ILogger<HomeController> logger, InstanceSettings instanceSettings)
         {
             _logger = logger;
+            _instanceSettings = instanceSettings;
         }
 
         public IActionResult Index()
         {
-            return View();
+            return View(_instanceSettings);
         }
 
         public IActionResult Privacy()

src/BirdsiteLive/Controllers/MigrationController.cs

@@ -0,0 +1,227 @@
+using System;
+using Microsoft.AspNetCore.Mvc;
+using System.Security.Cryptography;
+using System.Text;
+using System.Threading.Tasks;
+using Npgsql.TypeHandlers;
+using BirdsiteLive.Domain;
+using BirdsiteLive.Domain.Enum;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.Models;
+using System.Reflection.Metadata;
+
+namespace BirdsiteLive.Controllers
+{
+    public class MigrationController : Controller
+    {
+        private readonly MigrationService _migrationService;
+        private readonly ITwitterUserDal _twitterUserDal;
+        
+        #region Ctor
+        public MigrationController(MigrationService migrationService, ITwitterUserDal twitterUserDal)
+        {
+            _migrationService = migrationService;
+            _twitterUserDal = twitterUserDal;
+        }
+        #endregion
+
+        [HttpGet]
+        [Route("/migration/move/{id}")]
+        public IActionResult IndexMove(string id)
+        {
+            var migrationCode = _migrationService.GetMigrationCode(id);
+            var data = new MigrationData()
+            {
+                Acct = id,
+                MigrationCode = migrationCode
+            };
+
+            return View("Index", data);
+        }
+
+        [HttpGet]
+        [Route("/migration/delete/{id}")]
+        public IActionResult IndexDelete(string id)
+        {
+            var migrationCode = _migrationService.GetDeletionCode(id);
+            var data = new MigrationData()
+            {
+                Acct = id,
+                MigrationCode = migrationCode
+            };
+
+            return View("Delete", data);
+        }
+
+        [HttpPost]
+        [Route("/migration/move/{id}")]
+        public async Task<IActionResult> MigrateMove(string id, string tweetid, string handle)
+        {
+            var migrationCode = _migrationService.GetMigrationCode(id);
+            var data = new MigrationData()
+            {
+                Acct = id,
+                MigrationCode = migrationCode,
+
+                IsAcctProvided = !string.IsNullOrWhiteSpace(handle),
+                IsTweetProvided = !string.IsNullOrWhiteSpace(tweetid),
+
+                TweetId = tweetid,
+                FediverseAccount = handle
+            };
+            ValidatedFediverseUser fediverseUserValidation = null;
+
+            //Verify can be migrated 
+            var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id);
+            if (twitterAccount != null && twitterAccount.Deleted)
+            {
+                data.ErrorMessage = "This account has been deleted, it can't be migrated";
+                return View("Index", data);
+            }
+            if (twitterAccount != null &&
+                (!string.IsNullOrWhiteSpace(twitterAccount.MovedTo)
+                 || !string.IsNullOrWhiteSpace(twitterAccount.MovedToAcct)))
+            {
+                data.ErrorMessage = "This account has been moved already, it can't be migrated again";
+                return View("Index", data);
+            }
+
+            // Start migration
+            try
+            {
+                fediverseUserValidation = await _migrationService.ValidateFediverseAcctAsync(handle);
+                var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Migration);
+
+                data.IsAcctValid = fediverseUserValidation.IsValid;
+                data.IsTweetValid = isTweetValid;
+            }
+            catch (Exception e)
+            {
+                data.ErrorMessage = e.Message;
+            }
+
+            if (data.IsAcctValid && data.IsTweetValid && fediverseUserValidation != null)
+            {
+                try
+                {
+                    await _migrationService.MigrateAccountAsync(fediverseUserValidation, id);
+                    _migrationService.TriggerRemoteMigrationAsync(id, tweetid, handle);
+                    data.MigrationSuccess = true;
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine(e);
+                    data.ErrorMessage = e.Message;
+                }
+            }
+
+            return View("Index", data);
+        }
+
+        [HttpPost]
+        [Route("/migration/delete/{id}")]
+        public async Task<IActionResult> MigrateDelete(string id, string tweetid)
+        {
+            var deletionCode = _migrationService.GetDeletionCode(id);
+
+            var data = new MigrationData()
+            {
+                Acct = id,
+                MigrationCode = deletionCode,
+
+                IsTweetProvided = !string.IsNullOrWhiteSpace(tweetid),
+
+                TweetId = tweetid
+            };
+
+            //Verify can be deleted 
+            var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id);
+            if (twitterAccount != null && twitterAccount.Deleted)
+            {
+                data.ErrorMessage = "This account has been deleted, it can't be deleted again";
+                return View("Delete", data);
+            }
+
+            // Start deletion
+            try
+            {
+                var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Deletion);
+                data.IsTweetValid = isTweetValid;
+            }
+            catch (Exception e)
+            {
+                data.ErrorMessage = e.Message;
+            }
+
+            if (data.IsTweetValid)
+            {
+                try
+                {
+                    await _migrationService.DeleteAccountAsync(id);
+                    _migrationService.TriggerRemoteDeleteAsync(id, tweetid);
+                    data.MigrationSuccess = true;
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine(e);
+                    data.ErrorMessage = e.Message;
+                }
+            }
+
+            return View("Delete", data);
+        }
+
+        [HttpPost]
+        [Route("/migration/move/{id}/{tweetid}/{handle}")]
+        public async Task<IActionResult> RemoteMigrateMove(string id, string tweetid, string handle)
+        {
+            //Check inputs 
+            if (string.IsNullOrWhiteSpace(id) || string.IsNullOrWhiteSpace(tweetid) ||
+                string.IsNullOrWhiteSpace(handle))
+                return StatusCode(422);
+
+            //Verify can be migrated 
+            var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id);
+            if (twitterAccount != null && (twitterAccount.Deleted 
+                || !string.IsNullOrWhiteSpace(twitterAccount.MovedTo) 
+                || !string.IsNullOrWhiteSpace(twitterAccount.MovedToAcct)))
+                return Ok();
+
+            // Start migration
+            var fediverseUserValidation = await _migrationService.ValidateFediverseAcctAsync(handle);
+            var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Migration);
+
+            if (fediverseUserValidation.IsValid && isTweetValid)
+            {
+                await _migrationService.MigrateAccountAsync(fediverseUserValidation, id);
+                return Ok();
+            }
+
+            return StatusCode(400);
+        }
+
+        [HttpPost]
+        [Route("/migration/delete/{id}/{tweetid}")]
+        public async Task<IActionResult> RemoteMigrateDelete(string id, string tweetid)
+        {
+            //Check inputs 
+            if (string.IsNullOrWhiteSpace(id) || string.IsNullOrWhiteSpace(tweetid))
+                return StatusCode(422);
+
+            //Verify can be deleted 
+            var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id);
+            if (twitterAccount != null && twitterAccount.Deleted) return Ok();
+
+            // Start deletion
+            var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Deletion);
+
+            if (isTweetValid)
+            {
+                await _migrationService.DeleteAccountAsync(id);
+                return Ok();
+            }
+
+            return StatusCode(400);
+        }
+    }
+}

src/BirdsiteLive/Controllers/UsersController.cs

@@ -3,7 +3,6 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Net.Mime;
-using System.Runtime.InteropServices.WindowsRuntime;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
@@ -11,6 +10,8 @@ using BirdsiteLive.ActivityPub;
 using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.Common.Regexes;
 using BirdsiteLive.Common.Settings;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Domain;
 using BirdsiteLive.Models;
 using BirdsiteLive.Tools;
@@ -28,20 +29,24 @@ namespace BirdsiteLive.Controllers
     {
         private readonly ITwitterUserService _twitterUserService;
         private readonly ITwitterTweetsService _twitterTweetService;
+        private readonly ITwitterUserDal _twitterUserDal;
         private readonly IUserService _userService;
         private readonly IStatusService _statusService;
         private readonly InstanceSettings _instanceSettings;
+        private readonly IActivityPubService _activityPubService;
         private readonly ILogger<UsersController> _logger;
 
         #region Ctor
-        public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ITwitterTweetsService twitterTweetService, ILogger<UsersController> logger)
+        public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ITwitterTweetsService twitterTweetService, IActivityPubService activityPubService, ILogger<UsersController> logger, ITwitterUserDal twitterUserDal)
         {
             _twitterUserService = twitterUserService;
             _userService = userService;
             _statusService = statusService;
             _instanceSettings = instanceSettings;
             _twitterTweetService = twitterTweetService;
+            _activityPubService = activityPubService;
             _logger = logger;
+            _twitterUserDal = twitterUserDal;
         }
         #endregion
 
@@ -56,11 +61,10 @@ namespace BirdsiteLive.Controllers
             }
             return View("UserNotFound");
         }
-
+        
         [Route("/@{id}")]
         [Route("/users/{id}")]
-        [Route("/users/{id}/remote_follow")]
-        public IActionResult Index(string id)
+        public async Task<IActionResult> Index(string id)
         {
             _logger.LogTrace("User Index: {Id}", id);
 
@@ -102,6 +106,7 @@ namespace BirdsiteLive.Controllers
             }
 
             //var isSaturated = _twitterUserService.IsUserApiRateLimited();
+            var dbUser = await _twitterUserDal.GetTwitterUserAsync(id);
 
             var acceptHeaders = Request.Headers["Accept"];
             if (acceptHeaders.Any())
@@ -111,8 +116,12 @@ namespace BirdsiteLive.Controllers
                 {
                     if (isSaturated) return new ObjectResult("Too Many Requests") { StatusCode = 429 };
                     if (notFound) return NotFound();
-                    var apUser = _userService.GetUser(user);
-                    var jsonApUser = JsonConvert.SerializeObject(apUser);
+                    if (dbUser != null && dbUser.Deleted) return new ObjectResult("Gone") { StatusCode = 410 };
+                    var apUser = _userService.GetUser(user, dbUser);
+                    var jsonApUser = JsonConvert.SerializeObject(apUser, new JsonSerializerSettings
+                    {
+                        NullValueHandling = NullValueHandling.Ignore
+                    });
                     return Content(jsonApUser, "application/activity+json; charset=utf-8");
                 }
             }
@@ -128,11 +137,21 @@ namespace BirdsiteLive.Controllers
                 Url = user.Url,
                 ProfileImageUrl = user.ProfileImageUrl,
                 Protected = user.Protected,
+                
+                InstanceHandle = $"@{user.Acct.ToLowerInvariant()}@{_instanceSettings.Domain}",
 
-                InstanceHandle = $"@{user.Acct.ToLowerInvariant()}@{_instanceSettings.Domain}"
+                MovedTo = dbUser?.MovedTo,
+                MovedToAcct = dbUser?.MovedToAcct,
+                Deleted = dbUser?.Deleted ?? false,
             };
             return View(displayableUser);
         }
+        
+        [Route("/users/{id}/remote_follow")]
+        public async Task<IActionResult> IndexRemoteFollow(string id)
+        {
+            return Redirect($"/users/{id}");
+        }
 
         [Route("/@{id}/{statusId}")]
         [Route("/users/{id}/statuses/{statusId}")]
@@ -147,6 +166,18 @@ namespace BirdsiteLive.Controllers
                     if (!long.TryParse(statusId, out var parsedStatusId))
                         return NotFound();
 
+                    if (_instanceSettings.MaxStatusFetchAge > 0)
+                    {
+                        // I hate bitwise operators, corn syrup, and the antichrist
+                        // shift 22 bits to the right to get milliseconds, add the twitter epoch, then divide by 1000 to get seconds
+                        long secondsAgo = DateTimeOffset.UtcNow.ToUnixTimeSeconds() - (((parsedStatusId >> 22) + 1288834974657) / 1000);
+
+                        if ( secondsAgo > _instanceSettings.MaxStatusFetchAge*60*60*24 )
+                        {
+                            return new StatusCodeResult(StatusCodes.Status410Gone);
+                        }
+                    }
+
                     var tweet = _twitterTweetService.GetTweet(parsedStatusId);
                     if (tweet == null)
                         return NotFound();
@@ -160,7 +191,7 @@ namespace BirdsiteLive.Controllers
                 }
             }
 
-            return Redirect($"https://twitter.com/{id}/status/{statusId}");
+            return Redirect($"https://{_instanceSettings.TwitterDomain}/{id}/status/{statusId}");
         }
 
         [Route("/users/{id}/inbox")]
@@ -246,5 +277,51 @@ namespace BirdsiteLive.Controllers
             var jsonApUser = JsonConvert.SerializeObject(followers);
             return Content(jsonApUser, "application/activity+json; charset=utf-8");
         }
+
+        [Route("/users/{actor}/remote_follow")]
+        [HttpPost]
+        public async Task<IActionResult> RemoteFollow(string actor)
+        {
+            StringValues webfingerValues;
+
+            if (!Request.Form.TryGetValue("webfinger", out webfingerValues)) return BadRequest();
+
+            var webfinger = webfingerValues.First();
+
+            if (webfinger.Length < 1 || actor.Length < 1) return BadRequest();
+
+            if (webfinger[0] == '@') webfinger = webfinger[1..];
+
+            if (webfinger.IndexOf("@") < 0 || ! new Regex("^[A-Za-z0-9_]*$").IsMatch(webfinger.Split('@')[0]) || ! new Regex("^[A-Za-z0-9_]*$").IsMatch(actor) || Uri.CheckHostName(webfinger.Split('@')[1]) == UriHostNameType.Unknown)
+            {
+                return BadRequest();
+            }
+
+            WebFingerData webfingerData;
+
+            try
+            {
+                webfingerData = await _activityPubService.WebFinger(webfinger);
+            }
+            catch(Exception e)
+            {
+                _logger.LogError("Could not WebFinger {user}: {exception}", webfinger, e);
+                return NotFound();
+            }
+
+            string redirectLink = "";
+
+            foreach(var link in webfingerData.links)
+            {
+                if(link.rel == "http://ostatus.org/schema/1.0/subscribe" && link.template.Length > 0)
+                {
+                    redirectLink = link.template.Replace("{uri}", "https://" + _instanceSettings.Domain + "/users/" + actor);
+                }
+            }
+
+            if (redirectLink == "") return NotFound();
+
+            return Redirect(redirectLink);
+        }
     }
 }

src/BirdsiteLive/Controllers/WellKnownController.cs

@@ -62,7 +62,6 @@ namespace BirdsiteLive.Controllers
         [Route("/nodeinfo/{id}.json")]
         public async Task<IActionResult> NodeInfo(string id)
         {
-            var version = System.Reflection.Assembly.GetEntryAssembly().GetName().Version.ToString(3);
             var twitterUsersCount = await _twitterUserDal.GetTwitterUsersCountAsync();
             var isOpenRegistration = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.Follower) != ModerationTypeEnum.WhiteListing;
 
@@ -82,7 +81,7 @@ namespace BirdsiteLive.Controllers
                     software = new Software()
                     {
                         name = "birdsitelive",
-                        version = version
+                        version = Program.VERSION
                     },
                     protocols = new[]
                     {
@@ -117,7 +116,7 @@ namespace BirdsiteLive.Controllers
                     software = new SoftwareV21()
                     {
                         name = "birdsitelive",
-                        version = version,
+                        version = Program.VERSION,
                         repository = "https://github.com/NicolasConstant/BirdsiteLive"
                     },
                     protocols = new[]

src/BirdsiteLive/Models/DisplayTwitterUser.cs

@@ -10,5 +10,9 @@
         public bool Protected { get; set; }
 
         public string InstanceHandle { get; set; }
+
+        public string MovedTo { get; set; }
+        public string MovedToAcct { get; set; }
+        public bool Deleted { get; set; }
     }
 }

src/BirdsiteLive/Models/MigrationData.cs

@@ -0,0 +1,21 @@
+namespace BirdsiteLive.Models
+{
+    public class MigrationData
+    {
+        public string Acct { get; set; }
+
+        public string FediverseAccount { get; set; }
+        public string TweetId { get; set; }
+
+        public string MigrationCode { get; set; }
+
+        public bool IsTweetProvided { get; set; }
+        public bool IsAcctProvided { get; set; }
+
+        public bool IsTweetValid { get; set; }
+        public bool IsAcctValid { get; set; }
+
+        public string ErrorMessage { get; set; }
+        public bool MigrationSuccess { get; set; }
+    }
+}

src/BirdsiteLive/Program.cs

@@ -14,6 +14,8 @@ namespace BirdsiteLive
 {
     public class Program
     {
+        public static string VERSION = System.Reflection.Assembly.GetEntryAssembly().GetName().Version.ToString(3) + "-fishe";
+
         public static void Main(string[] args)
         {
             CreateHostBuilder(args).Build().Run();

src/BirdsiteLive/Services/AboutPageService.cs

@@ -0,0 +1,75 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using BirdsiteLive.Common.Settings;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.Domain.Repository;
+
+namespace BirdsiteLive.Services
+{
+    public interface IAboutPageService
+    {
+        Task<AboutPageData> GetAboutPageDataAsync();
+    }
+
+    public class AboutPageService : IAboutPageService
+    {
+        private readonly ITwitterUserDal _twitterUserDal;
+
+        private static AboutPageData _aboutPageData;
+        private readonly InstanceSettings _instanceSettings;
+        private readonly IModerationRepository _moderationRepository;
+
+        #region Ctor
+        public AboutPageService(ITwitterUserDal twitterUserDal, InstanceSettings instanceSettings, IModerationRepository moderationRepository)
+        {
+            _twitterUserDal = twitterUserDal;
+            _instanceSettings = instanceSettings;
+            _moderationRepository = moderationRepository;
+        }
+        #endregion
+
+        public async Task<AboutPageData> GetAboutPageDataAsync()
+        {
+            if (_aboutPageData == null ||
+                (DateTime.UtcNow - _aboutPageData.RefreshedTime).TotalMinutes > 15)
+            {
+                var twitterUserMax = _instanceSettings.MaxUsersCapacity;
+                var twitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync();
+                var saturation = (int)((double)twitterUserCount / twitterUserMax * 100);
+
+                _aboutPageData = new AboutPageData
+                {
+                    RefreshedTime = DateTime.UtcNow,
+                    Saturation = saturation,
+                    UnlistedUsers = _instanceSettings.UnlistedTwitterAccounts.Length > 0 ? string.Join("\n", _instanceSettings.UnlistedTwitterAccounts.Split(";").Select(i => "<li>" + i + "</li>")) : "(none)",
+                    Settings = _instanceSettings,
+                    ModerationStatus = new ModerationStatus
+                    {
+                        Followers = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.Follower),
+                        TwitterAccounts = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.TwitterAccount),
+                        Repository = _moderationRepository
+                    }
+                };
+            }
+
+            return _aboutPageData;
+        }
+    }
+
+    public class AboutPageData
+    {
+        public DateTime RefreshedTime { get; set; }
+        public int Saturation { get; set; }
+        public string UnlistedUsers { get; set; }
+        public InstanceSettings Settings { get; set; }
+        public ModerationStatus ModerationStatus { get; set; }
+    }
+
+    public class ModerationStatus
+    {
+        public ModerationTypeEnum Followers { get; set; }
+        public ModerationTypeEnum TwitterAccounts { get; set; }
+        public IModerationRepository Repository { get; set; }
+    }
+}

src/BirdsiteLive/Services/CachedStatisticsService.cs

@@ -1,53 +0,0 @@
-using System;
-using System.Threading.Tasks;
-using BirdsiteLive.Common.Settings;
-using BirdsiteLive.DAL.Contracts;
-
-namespace BirdsiteLive.Services
-{
-    public interface ICachedStatisticsService
-    {
-        Task<CachedStatistics> GetStatisticsAsync();
-    }
-
-    public class CachedStatisticsService : ICachedStatisticsService
-    {
-        private readonly ITwitterUserDal _twitterUserDal;
-
-        private static CachedStatistics _cachedStatistics;
-        private readonly InstanceSettings _instanceSettings;
-
-        #region Ctor
-        public CachedStatisticsService(ITwitterUserDal twitterUserDal, InstanceSettings instanceSettings)
-        {
-            _twitterUserDal = twitterUserDal;
-            _instanceSettings = instanceSettings;
-        }
-        #endregion
-
-        public async Task<CachedStatistics> GetStatisticsAsync()
-        {
-            if (_cachedStatistics == null ||
-                (DateTime.UtcNow - _cachedStatistics.RefreshedTime).TotalMinutes > 15)
-            {
-                var twitterUserMax = _instanceSettings.MaxUsersCapacity;
-                var twitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync();
-                var saturation = (int)((double)twitterUserCount / twitterUserMax * 100);
-
-                _cachedStatistics = new CachedStatistics
-                {
-                    RefreshedTime = DateTime.UtcNow,
-                    Saturation = saturation
-                };
-            }
-
-            return _cachedStatistics;
-        }
-    }
-
-    public class CachedStatistics
-    {
-        public DateTime RefreshedTime { get; set; }
-        public int Saturation { get; set; }
-    }
-}

src/BirdsiteLive/Startup.cs

@@ -2,6 +2,7 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Net.Http;
+using System.Net.Http.Headers;
 using System.Threading.Tasks;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Common.Structs;
@@ -50,7 +51,12 @@ namespace BirdsiteLive
 
             services.AddControllersWithViews();
 
-            services.AddHttpClient();
+            services.AddHttpClient("BirdsiteLIVE", httpClient => {
+                ProductInfoHeaderValue product = new("BirdsiteLIVE", $"{Program.VERSION}");
+                ProductInfoHeaderValue comment = new($"(+https://{Configuration["Instance:Domain"]})");
+                httpClient.DefaultRequestHeaders.UserAgent.Add(product);
+                httpClient.DefaultRequestHeaders.UserAgent.Add(comment);
+            });
         }
 
         public void ConfigureContainer(ServiceRegistry services)
@@ -91,6 +97,9 @@ namespace BirdsiteLive
             services.For<ITwitterUserService>().DecorateAllWith<CachedTwitterUserService>();
             services.For<ITwitterUserService>().Use<TwitterUserService>().Singleton();
 
+            services.For<ITwitterTweetsService>().DecorateAllWith<CachedTwitterTweetsService>();
+            services.For<ITwitterTweetsService>().Use<TwitterTweetsService>().Singleton();
+
             services.For<ITwitterAuthenticationInitializer>().Use<TwitterAuthenticationInitializer>().Singleton();
 
             services.Scan(_ =>
@@ -118,6 +127,7 @@ namespace BirdsiteLive
             if (env.IsDevelopment())
             {
                 app.UseDeveloperExceptionPage();
+                app.UseBrowserLink();
             }
             else
             {

src/BirdsiteLive/Views/About/Blacklisting.cshtml

@@ -1,27 +0,0 @@
-@using BirdsiteLive.Domain.Repository
-@model BirdsiteLive.Controllers.ModerationStatus
-@{
-    ViewData["Title"] = "Blacklisting";
-}
-
-<div class="col-12 col-sm-12 col-md-10 col-lg-8 mx-auto">
-    <h2>Blacklisting</h2>
-
-    @if (Model.Followers == ModerationTypeEnum.BlackListing)
-    {
-        <p><br />This node is blacklisting some instances and/or Fediverse users.<br /><br /></p>
-    }
-
-    @if (Model.TwitterAccounts == ModerationTypeEnum.BlackListing)
-    {
-        <p><br />This node is blacklisting some twitter users.<br /><br /></p>
-    }
-
-    @if (Model.Followers != ModerationTypeEnum.BlackListing && Model.TwitterAccounts != ModerationTypeEnum.BlackListing)
-    {
-        <p><br />This node is not using blacklisting.<br /><br /></p>
-    }
-
-    @*<h2>FAQ</h2>
-    <p>TODO</p>*@
-</div>

src/BirdsiteLive/Views/About/Index.cshtml

@@ -1,30 +1,122 @@
-@model BirdsiteLive.Services.CachedStatistics
+@model BirdsiteLive.Services.AboutPageData
 @{
     ViewData["Title"] = "About";
 }
 
 <div class="col-12 col-sm-12 col-md-10 col-lg-8 mx-auto">
-    <h2>Node Saturation</h2>
+    <h1>About</h1>
+
+    @if (Model.Settings.MaxFollowsPerUser > 0)
+    {
+        <div class="alert alert-warning">
+            In order to keep this service available to everyone, users are only permitted to follow <b>@Model.Settings.MaxFollowsPerUser</b> account(s). Any additional follows will be rejected.
+        </div>
+    }
+
+    @if (Model.Settings.InfoBanner.Length > 0)
+    {
+        <div class="alert alert-danger">
+            @Html.Raw(Model.Settings.InfoBanner)
+        </div>
+    }
+
+    <h4>About @Model.Settings.Name</h4>
 
     <p>
-        <br/>
-        This node usage is at @Model.Saturation%<br/>
-        <br/>
+        @Model.Settings.Name runs an instance of BirdsiteLIVE, an <a href="https://activitypub.rocks" target="_blank">ActivityPub</a>-compatible <a href="https://en.wikipedia.org/wiki/Fediverse" target="_blank">Fediverse</a> server that delivers Tweets from Twitter to users on the Fediverse.
+
+        <br /><br />
+
+        BirdsiteLIVE does not make any public posts; every post is scoped appropriately using the "followers-only" or "unlisted" ActivityPub audiences.
     </p>
 
-    <h2>FAQ</h2>
-    <h4>Why is there a limit on the node?</h4>
-
-    <p>BirdsiteLIVE rely on the Twitter API to provide high quality content. This API has limitations and therefore limits node capacity.</p>
-
-    <h4>What happen when the node is saturated?</h4>
+    <h4 id="followers-only">Unlisted accounts</h4>
 
     <p>
-        When the saturation rate goes above 100% the node will no longer update all accounts every 15 minutes and instead will reduce the pooling rate to stay under the API limits, the more saturated a node is the less efficient it will be.<br />
-        The software doesn't scale, and it's by design.
+
+        The goal of this instance is <i>not</i> to make the Fediverse "Twitter 2"; it is to make the Fediverse easier to join by allowing people to follow those who will never move. Therefore, by default, Twitter posts are not shown publicly. This instance's admin may allow certain accounts to post using the unlisted audience; this allows an account's posts to be "boosted" or "repeated."
+
+        <br /><br />
+
+        Accounts that post using the "unlisted" audience are as follows:
+
+        <ul>
+            @Html.Raw(Model.UnlistedUsers)
+        </ul>
     </p>
 
-    <h4>How can I reduce the node's saturation?</h4>
+    <h4 id="saturation">Instance saturation</h4>
 
-    <p>If you're not on your own node, be reasonable and don't follow too much accounts. And if you can, host your own node. BirdsiteLIVE doesn't require a lot of resources to work and therefore is really cheap to self-host.</p>
+    <p>
+        This instance's saturation level is currently at @Model.Saturation%.
+
+        <br /><br />
+
+        This instance relies on a tool Twitter provides (the API) to fetch Tweets in a predictable and high-quality manner. However, due to limits imposed by Twitter, this instance can only fetch so many Tweets and users per hour.<br /><br />
+
+        As this instance's saturation level approaches and exceeds 100%, it will no longer update accounts every 15 minutes and reduce how often it fetches Tweets to stay under Twitter's limits. Essentially, the more saturated a node is, the less efficient it will be.<br /><br />
+
+        When possible, you should <a href="https://git.froth.zone/sam/BirdsiteLIVE/src/branch/master/INSTALLATION.md" target="_blank">start your own BirdsiteLIVE instance</a>. If you cannot, please be courteous and follow a limited number of accounts to keep the service available for everyone.
+    </p>
+
+    @if (Model.Settings.DiscloseInstanceRestrictions && (Model.ModerationStatus.Followers != BirdsiteLive.Domain.Repository.ModerationTypeEnum.None || Model.ModerationStatus.TwitterAccounts != BirdsiteLive.Domain.Repository.ModerationTypeEnum.None))
+    {
+        <h4 id="restrictions">
+            Instance restrictions
+        </h4>
+
+        <p>This instance can generally communicate with any other server following the ActivityPub protocol, with some exceptions, as listed below and configured by this server's administrators.</p>
+
+        if (Model.ModerationStatus.Followers == BirdsiteLive.Domain.Repository.ModerationTypeEnum.BlackListing)
+        {
+            <h5 id="instance-blacklist">Instance blacklist</h5>
+            <p>No data for instances on this list will be processed. Users from instances on this list are not able to follow or directly receive Tweets from accounts on this instance.</p>
+
+            <ul>
+                @foreach (var i in Model.ModerationStatus.Repository.GetBlacklistedFollowers())
+                {
+                    <li>@i</li>
+                }
+            </ul>
+        }
+
+        if (Model.ModerationStatus.Followers == BirdsiteLive.Domain.Repository.ModerationTypeEnum.WhiteListing)
+        {
+            <h5 id="instance-whitelist">Instance whitelist</h5>
+            <p>Only users from instances on this list will be able to follow or directly receive Tweets from accounts on this instance.</p>
+
+            <ul>
+                @foreach (var i in Model.ModerationStatus.Repository.GetWhitelistedFollowers())
+                {
+                    <li>@i</li>
+                }
+            </ul>
+        }
+
+        if (Model.ModerationStatus.TwitterAccounts == BirdsiteLive.Domain.Repository.ModerationTypeEnum.BlackListing)
+        {
+            <h5 id="account-blacklist">Account blacklist</h5>
+            <p>Users will not be able to follow the following Twitter accounts on this instance, and Tweets from these accounts will not be relayed.</p>
+
+            <ul>
+                @foreach (var i in Model.ModerationStatus.Repository.GetBlacklistedAccounts())
+                {
+                    <li>@i</li>
+                }
+            </ul>
+        }
+
+        if (Model.ModerationStatus.TwitterAccounts == BirdsiteLive.Domain.Repository.ModerationTypeEnum.WhiteListing)
+        {
+            <h5 id="account-whitelist">Account whitelist</h5>
+            <p>Only Twitter accounts on this list are able to be followed from this instance.</p>
+
+            <ul>
+                @foreach (var i in Model.ModerationStatus.Repository.GetWhitelistedAccounts())
+                {
+                    <li>@i</li>
+                }
+            </ul>
+        }
+    }
 </div>

src/BirdsiteLive/Views/About/Whitelisting.cshtml

@@ -1,27 +0,0 @@
-@using BirdsiteLive.Domain.Repository
-@model BirdsiteLive.Controllers.ModerationStatus
-@{
-    ViewData["Title"] = "Whitelisting";
-}
-
-<div class="col-12 col-sm-12 col-md-10 col-lg-8 mx-auto">
-    <h2>Whitelisting</h2>
-
-    @if (Model.Followers == ModerationTypeEnum.WhiteListing)
-    {
-        <p><br />This node is whitelisting some instances and/or Fediverse users.<br /><br /></p>
-    }
-
-    @if (Model.TwitterAccounts == ModerationTypeEnum.WhiteListing)
-    {
-        <p><br />This node is whitelisting some twitter users.<br /><br /></p>
-    }
-
-    @if (Model.Followers != ModerationTypeEnum.WhiteListing && Model.TwitterAccounts != ModerationTypeEnum.WhiteListing)
-    {
-        <p><br />This node is not using whitelisting.<br /><br /></p>
-    }
-
-    @*<h2>FAQ</h2>
-    <p>TODO</p>*@
-</div>

src/BirdsiteLive/Views/Debuging/Index.cshtml

@@ -23,4 +23,10 @@
     <!-- Input and Submit elements -->
 
     <button type="submit" value="Submit">Reject Follow</button>
+</form>
+
+<form asp-controller="Debuging" asp-action="PostDeleteUser" method="post">
+    <!-- Input and Submit elements -->
+
+    <button type="submit" value="Submit">Delete User</button>
 </form>

src/BirdsiteLive/Views/Home/Index.cshtml

@@ -1,32 +1,40 @@
-@using BirdsiteLive.Controllers;
-@{
-    ViewData["Title"] = "Home Page";
-}
+@using BirdsiteLive.Common.Settings;
+@model InstanceSettings
+@{ ViewData["Title"] = "Home Page"; }
 
 <div class="text-center">
     <h1 class="display-4">Welcome</h1>
     <p>
         <br />
-        BirdsiteLIVE is a Twitter to ActivityPub bridge.<br />
-        Find a Twitter account below:
+        This instance is a Twitter to ActivityPub bridge.<br />
+        <a asp-controller="About" asp-action="Index">Learn more</a> or find a Twitter account below:
     </p>
-    
+
     <form method="POST">
-        @*<div class="form-group">
-            <label for="exampleInputEmail1">Email address</label>
-            <input type="email" class="form-control" id="exampleInputEmail1" aria-describedby="emailHelp" placeholder="Enter email">
-            <small id="emailHelp" class="form-text text-muted">We'll never share your email with anyone else.</small>
-        </div>*@
         <div class="form-group">
-            @*<label for="exampleInputPassword1">Password</label>*@
             <input type="text" class="form-control col-8 col-sm-8 col-md-6 col-lg-4 mx-auto" id="handle" name="handle" autocomplete="off" placeholder="Twitter Handle">
         </div>
         <button type="submit" class="btn btn-primary">Show</button>
     </form>
 
+    <br /><br />
+
+    @if( Model.MaxFollowsPerUser > 0)
+    {
+        <div class="alert alert-warning">
+            In order to keep this service available to everyone, users are only permitted to follow <b>@Model.MaxFollowsPerUser</b> account(s). Any additional follows will be rejected. For more information, see our <a href="/About#saturation">about page</a>.
+        </div>
+    }
+
+    @if (Model.InfoBanner.Length > 0)
+    {
+    <div class="alert alert-danger">
+        @Html.Raw(Model.InfoBanner)
+    </div>
+    }
 
     @*@if (HtmlHelperExtensions.IsDebug())
-    {
-        <a class="nav-link text-dark" asp-area="" asp-controller="Debuging" asp-action="Index">Debug</a>
-    }*@
+        {
+            <a class="nav-link text-dark" asp-area="" asp-controller="Debuging" asp-action="Index">Debug</a>
+        }*@
 </div>

src/BirdsiteLive/Views/Migration/Delete.cshtml

@@ -0,0 +1,51 @@
+@model MigrationData
+@{
+    ViewData["Title"] = "Migration";
+}
+
+<div class="col-12 col-sm-10 col-md-8 col-lg-6 mx-auto">
+    @if (!string.IsNullOrWhiteSpace(ViewData.Model.ErrorMessage))
+    {
+        <div class="alert alert-danger" role="alert">
+            @ViewData.Model.ErrorMessage
+        </div>
+    }
+    
+    @if (ViewData.Model.MigrationSuccess)
+    {
+        <div class="alert alert-success" role="alert">
+            The mirror has been successfully deleted
+        </div>
+    }
+
+    <h1 class="display-4 migration__title">Delete @@@ViewData.Model.Acct mirror</h1>
+
+    @if (!ViewData.Model.IsTweetProvided)
+    {
+        <h2 class="display-4 migration__subtitle">What is needed?</h2>
+
+        <p>You'll need access to the Twitter account to provide proof of ownership.</p>
+
+        <h2 class="display-4 migration__subtitle">What will deletion do?</h2>
+
+        <p>
+            Deletion will remove all followers, delete the account and will be blacklisted so that it can't be recreated.<br />
+        </p>
+    }
+
+    <h2 class="display-4 migration__subtitle">Start the deletion!</h2>
+
+    <p>Please copy and post this string in a public Tweet (the string must be untampered, but you can write anything you want before or after it):</p>
+
+    <input type="text" name="textbox" value="@ViewData.Model.MigrationCode" onclick="this.select()" class="form-control" readonly />
+    <br />
+
+    <h2 class="display-4 migration__subtitle">Provide deletion information:</h2>
+    <form method="POST">
+        <div class="form-group">
+            <label for="tweetid">Tweet URL</label>
+            <input type="text" class="form-control" id="tweetid" name="tweetid" autocomplete="off" placeholder="https://twitter.com/<username>/status/<tweet id>" value="@ViewData.Model.TweetId">
+        </div>
+        <button type="submit" class="btn btn-primary">Delete!</button>
+    </form>
+</div>

src/BirdsiteLive/Views/Migration/Index.cshtml

@@ -0,0 +1,66 @@
+@model MigrationData
+@{
+    ViewData["Title"] = "Migration";
+}
+
+<div class="col-12 col-sm-10 col-md-8 col-lg-6 mx-auto">
+    @if (!string.IsNullOrWhiteSpace(ViewData.Model.ErrorMessage))
+    {
+        <div class="alert alert-danger" role="alert">
+            @ViewData.Model.ErrorMessage
+        </div>
+    }
+    
+    @if (ViewData.Model.MigrationSuccess)
+    {
+        <div class="alert alert-success" role="alert">
+            The mirror has been successfully migrated 
+        </div>
+    }
+
+    <h1 class="display-4 migration__title">Migrate @@@ViewData.Model.Acct mirror to my Fediverse account</h1>
+
+    @if (!ViewData.Model.IsAcctProvided && !ViewData.Model.IsTweetProvided)
+    {
+        <h2 class="display-4 migration__subtitle">What is needed?</h2>
+
+        <p>You'll need a Fediverse account and access to the Twitter account to provide proof of ownership.</p>
+
+        <h2 class="display-4 migration__subtitle">What will migration do?</h2>
+
+        <p>
+            Migration will notify followers of the migration of the mirror account to your fediverse account and will be disabled after that.<br />
+        </p>
+    }
+
+    <h2 class="display-4 migration__subtitle">Start the migration!</h2>
+
+    <p>Please copy and post this string in a public Tweet (the string must be untampered, but you can write anything you want before or after it):</p>
+
+    <input type="text" name="textbox" value="@ViewData.Model.MigrationCode" onclick="this.select()" class="form-control" readonly />
+    <br />
+
+    <h2 class="display-4 migration__subtitle">Provide migration information:</h2>
+    <form method="POST">
+        @*<div class="form-group">
+        <label for="exampleInputEmail1">Email address</label>
+        <input type="email" class="form-control" id="exampleInputEmail1" aria-describedby="emailHelp" placeholder="Enter email">
+        <small id="emailHelp" class="form-text text-muted">We'll never share your email with anyone else.</small>
+        </div>*@
+        <div class="form-group">
+            <label for="handle">Fediverse target account</label>
+            <input type="text" class="form-control" id="handle" name="handle" autocomplete="off" placeholder="@Html.Raw("@username@domain.ext")" value="@ViewData.Model.FediverseAccount">
+        </div>
+        <div class="form-group">
+            <label for="tweetid">Tweet URL</label>
+            <input type="text" class="form-control" id="tweetid" name="tweetid" autocomplete="off" placeholder="https://twitter.com/<username>/status/<tweet id>" value="@ViewData.Model.TweetId">
+        </div>
+        <button type="submit" class="btn btn-primary">Migrate!</button>
+    </form>
+    <br />
+    <br />
+    <br />
+    <div class="user-owner">
+        <a href="/migration/delete/@ViewData.Model.Acct">I don't have a fediverse account and I'd like to delete this mirror.</a>
+    </div>
+</div>

src/BirdsiteLive/Views/Shared/Components/NodeInfo/Default.cshtml

@@ -1,17 +1,13 @@
 @model BirdsiteLive.Component.NodeInfoViewModel
 
 <div>
-    @if (ViewData.Model.WhitelistingEnabled)
+    @if (ViewData.Model.DiscloseRestrictions && (ViewData.Model.WhitelistingEnabled || ViewData.Model.BlacklistingEnabled))
     {
-        <a asp-controller="About" asp-action="Whitelisting" class="badge badge-light" title="What does this mean?">Whitelisting Enabled</a>
-    }
-    @if (ViewData.Model.BlacklistingEnabled)
-    {
-        <a asp-controller="About" asp-action="Blacklisting" class="badge badge-light" title="What does this mean?">Blacklisting Enabled</a>
+        <a asp-controller="About" asp-action="Index" class="badge badge-light" title="What does this mean?" asp-fragment="restrictions">View restrictions</a>
     }
 
     <div class="node-progress-bar">
-        <div class="node-progress-bar__label"><a asp-controller="About" asp-action="Index">Instance saturation:</a></div>
+        <div class="node-progress-bar__label"><a asp-controller="About" asp-action="Index" asp-fragment="saturation">Instance saturation:</a></div>
         <div class="progress node-progress-bar__bar">
             <div class="progress-bar 
                  @((ViewData.Model.InstanceSaturation > 50 && ViewData.Model.InstanceSaturation < 75) ? "bg-warning ":"")

src/BirdsiteLive/Views/Shared/_Layout.cshtml

@@ -47,9 +47,9 @@
         </div>
         <div class="container">
 
-            <a href="https://github.com/NicolasConstant/BirdsiteLive">Github</a> @*<a asp-area="" asp-controller="Home" asp-action="Privacy">Privacy</a>*@
+            <a href="https://git.froth.zone/sam/BirdsiteLIVE">Source code</a> (AGPLv3) @*<a asp-area="" asp-controller="Home" asp-action="Privacy">Privacy</a>*@
 
-            <span style="float: right;">BirdsiteLIVE @System.Reflection.Assembly.GetEntryAssembly().GetName().Version.ToString(3)</span>
+            <span style="float: right;">BirdsiteLIVE @Program.VERSION</span>
         </div>
     </footer>
     <script src="~/lib/jquery/dist/jquery.min.js"></script>

src/BirdsiteLive/Views/Users/Index.cshtml

@@ -31,7 +31,20 @@
     <br />
     <br />
 
-    @if (ViewData.Model.Protected)
+    @if (ViewData.Model.Deleted)
+    {
+        <div class="alert alert-danger" role="alert">
+            This mirror has been deleted by its Twitter owner.
+        </div>
+    }
+    else if (!string.IsNullOrEmpty(ViewData.Model.MovedTo))
+    {
+        <div class="alert alert-danger" role="alert">
+            This account has been migrated by its Twitter owner and has been disabled.<br />
+            You can follow this user at <a href="@ViewData.Model.MovedTo">@ViewData.Model.MovedToAcct</a>.
+        </div>
+    }
+    else if (ViewData.Model.Protected)
     {
         <div class="alert alert-danger" role="alert">
             This account is protected, BirdsiteLIVE cannot fetch their tweets and will not provide follow support until it is unprotected again.
@@ -40,9 +53,18 @@
     else
     {
         <div>
-            <p>Search this handle to find it in your instance:</p>
+            <form action="/users/@ViewData.Model.Acct/remote_follow" method="post">
+                <input type="text" class="form-control mb-2" placeholder="your handle, i.e. @@lain@@pleroma.com" name="webfinger" />
+                <input type="submit" class="btn btn-primary w-100 mb-2" value="Remote follow" />
+            </form>
+
+            <p>or search this handle to find it in your instance:</p>
 
             <input type="text" name="textbox" value="@ViewData.Model.InstanceHandle" onclick="this.select()" class="form-control" readonly />
         </div>
     }
+
+    <div class="user-owner">
+        <a href="/migration/move/@ViewData.Model.Acct">I'm the owner of this account and I would like to take control of this mirror.</a>
+    </div>
 </div>

src/BirdsiteLive/appsettings.json

@@ -20,10 +20,17 @@
     "AdminEmail": "me@domain.name",
     "ResolveMentionsInProfiles": true,
     "PublishReplies": false,
-    "MaxUsersCapacity": 1000,
+    "MaxUsersCapacity": 1500,
     "UnlistedTwitterAccounts": null,
+    "TwitterDomain": "twitter.com",
+    "TwitterDomainLabel": "",
+    "InfoBanner": "",
+    "ShowAboutInstanceOnProfiles": true,
+    "MaxFollowsPerUser": 0,
+    "DiscloseInstanceRestrictions": false,
     "SensitiveTwitterAccounts": null,
     "FailingTwitterUserCleanUpThreshold": 700,
+    "MaxStatusFetchAge": 0,
     "FailingFollowerCleanUpThreshold": 30000,
     "UserCacheCapacity": 10000 
   },
@@ -43,5 +50,5 @@
     "FollowersBlackListing": null,
     "TwitterAccountsWhiteListing": null,
     "TwitterAccountsBlackListing": null
-  } 
-}
+  }
+}

src/BirdsiteLive/wwwroot/css/birdsite.css

@@ -71,3 +71,18 @@
     margin-left: 60px;
     /*font-weight: bold;*/
 }
+
+.user-owner {
+    font-size: .8em;
+    padding-top: 20px;
+}
+
+/** Migration **/
+
+.migration__title {
+    font-size: 1.8em;
+}
+
+.migration__subtitle {
+    font-size: 1.4em;
+}

src/DataAccessLayers/BirdsiteLive.DAL.Postgres/BirdsiteLive.DAL.Postgres.csproj

@@ -1,12 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Dapper" Version="2.0.35" />
-    <PackageReference Include="Npgsql" Version="4.1.3.1" />
+    <PackageReference Include="Dapper" Version="2.0.123" />
+    <PackageReference Include="Npgsql" Version="4.1.12" />
   </ItemGroup>
 
   <ItemGroup>

src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs

@@ -23,7 +23,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
     public class DbInitializerPostgresDal : PostgresBase, IDbInitializerDal
     {
         private readonly PostgresTools _tools;
-        private readonly Version _currentVersion = new Version(2, 4);
+        private readonly Version _currentVersion = new Version(2, 5);
         private const string DbVersionType = "db-version";
 
         #region Ctor
@@ -135,7 +135,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
                 new Tuple<Version, Version>(new Version(2,0), new Version(2,1)),
                 new Tuple<Version, Version>(new Version(2,1), new Version(2,2)),
                 new Tuple<Version, Version>(new Version(2,2), new Version(2,3)),
-                new Tuple<Version, Version>(new Version(2,3), new Version(2,4))
+                new Tuple<Version, Version>(new Version(2,3), new Version(2,4)),
+                new Tuple<Version, Version>(new Version(2,4), new Version(2,5))
             };
         }
 
@@ -172,6 +173,17 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
                 var alterPostingError = $@"ALTER TABLE {_settings.FollowersTableName} ALTER COLUMN postingErrorCount TYPE INTEGER";
                 await _tools.ExecuteRequestAsync(alterPostingError);
             }
+            else if (from == new Version(2, 4) && to == new Version(2, 5))
+            {
+                var addMovedTo = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD movedTo VARCHAR(2048)";
+                await _tools.ExecuteRequestAsync(addMovedTo);
+
+                var addMovedToAcct = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD movedToAcct VARCHAR(305)";
+                await _tools.ExecuteRequestAsync(addMovedToAcct);
+
+                var addDeletedToAcct = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD deleted BOOLEAN";
+                await _tools.ExecuteRequestAsync(addDeletedToAcct);
+            }
             else
             {
                 throw new NotImplementedException();

src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs

@@ -18,7 +18,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
         }
         #endregion
 
-        public async Task CreateTwitterUserAsync(string acct, long lastTweetPostedId)
+        public async Task CreateTwitterUserAsync(string acct, long lastTweetPostedId, string movedTo = null, string movedToAcct = null)
         {
             acct = acct.ToLowerInvariant();
 
@@ -27,8 +27,15 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
                 dbConnection.Open();
 
                 await dbConnection.ExecuteAsync(
-                    $"INSERT INTO {_settings.TwitterUserTableName} (acct,lastTweetPostedId,lastTweetSynchronizedForAllFollowersId) VALUES(@acct,@lastTweetPostedId,@lastTweetSynchronizedForAllFollowersId)",
-                    new { acct, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = lastTweetPostedId });
+                    $"INSERT INTO {_settings.TwitterUserTableName} (acct,lastTweetPostedId,lastTweetSynchronizedForAllFollowersId, movedTo, movedToAcct) VALUES(@acct,@lastTweetPostedId,@lastTweetSynchronizedForAllFollowersId,@movedTo,@movedToAcct)",
+                    new
+                    {
+                        acct, 
+                        lastTweetPostedId, 
+                        lastTweetSynchronizedForAllFollowersId = lastTweetPostedId,
+                        movedTo,
+                        movedToAcct
+                    });
             }
         }
 
@@ -62,7 +69,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
 
         public async Task<int> GetTwitterUsersCountAsync()
         {
-            var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName}";
+            var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE";
 
             using (var dbConnection = Connection)
             {
@@ -75,7 +82,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
 
         public async Task<int> GetFailingTwitterUsersCountAsync()
         {
-            var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE fetchingErrorCount > 0";
+            var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE fetchingErrorCount > 0 AND (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE";
 
             using (var dbConnection = Connection)
             {
@@ -86,9 +93,10 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
             }
         }
 
-        public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber)
+        public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber, bool retrieveDisabledUser)
         {
-            var query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber";
+            var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber";
+            if (retrieveDisabledUser) query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber";
 
             using (var dbConnection = Connection)
             {
@@ -99,9 +107,10 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
             }
         }
 
-        public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync()
+        public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(bool retrieveDisabledUser)
         {
-            var query = $"SELECT * FROM {_settings.TwitterUserTableName}";
+            var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE";
+            if(retrieveDisabledUser) query = $"SELECT * FROM {_settings.TwitterUserTableName}";
 
             using (var dbConnection = Connection)
             {
@@ -112,26 +121,36 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
             }
         }
 
-        public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync)
+        public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync, string movedTo, string movedToAcct, bool deleted)
         {
             if(id == default) throw new ArgumentException("id");
             if(lastTweetPostedId == default) throw new ArgumentException("lastTweetPostedId");
             if(lastTweetSynchronizedForAllFollowersId == default) throw new ArgumentException("lastTweetSynchronizedForAllFollowersId");
             if(lastSync == default) throw new ArgumentException("lastSync");
 
-            var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId, fetchingErrorCount = @fetchingErrorCount, lastSync = @lastSync WHERE id = @id";
+            var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId, fetchingErrorCount = @fetchingErrorCount, lastSync = @lastSync, movedTo = @movedTo, movedToAcct = @movedToAcct, deleted = @deleted WHERE id = @id";
 
             using (var dbConnection = Connection)
             {
                 dbConnection.Open();
 
-                await dbConnection.QueryAsync(query, new { id, lastTweetPostedId,  lastTweetSynchronizedForAllFollowersId, fetchingErrorCount, lastSync = lastSync.ToUniversalTime() });
+                await dbConnection.QueryAsync(query, new
+                {
+                    id, 
+                    lastTweetPostedId, 
+                    lastTweetSynchronizedForAllFollowersId, 
+                    fetchingErrorCount, 
+                    lastSync = lastSync.ToUniversalTime(),
+                    movedTo,
+                    movedToAcct,
+                    deleted
+                });
             }
         }
 
         public async Task UpdateTwitterUserAsync(SyncTwitterUser user)
         {
-            await UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, user.LastSync);
+            await UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, user.LastSync, user.MovedTo, user.MovedToAcct, user.Deleted);
         }
 
         public async Task DeleteTwitterUserAsync(string acct)

src/DataAccessLayers/BirdsiteLive.DAL/BirdsiteLive.DAL.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs

@@ -6,12 +6,13 @@ namespace BirdsiteLive.DAL.Contracts
 {
     public interface ITwitterUserDal
     {
-        Task CreateTwitterUserAsync(string acct, long lastTweetPostedId);
+        Task CreateTwitterUserAsync(string acct, long lastTweetPostedId, string movedTo = null,
+            string movedToAcct = null);
         Task<SyncTwitterUser> GetTwitterUserAsync(string acct);
         Task<SyncTwitterUser> GetTwitterUserAsync(int id);
-        Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber);
-        Task<SyncTwitterUser[]> GetAllTwitterUsersAsync();
-        Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync);
+        Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber, bool retrieveDisabledUser);
+        Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(bool retrieveDisabledUser);
+        Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync, string movedTo, string movedToAcct, bool deleted);
         Task UpdateTwitterUserAsync(SyncTwitterUser user);
         Task DeleteTwitterUserAsync(string acct);
         Task DeleteTwitterUserAsync(int id);

src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs

@@ -12,6 +12,11 @@ namespace BirdsiteLive.DAL.Models
 
         public DateTime LastSync { get; set; }
 
-        public int FetchingErrorCount { get; set; } //TODO: update DAL
+        public int FetchingErrorCount { get; set; }
+
+        public string MovedTo { get; set; }
+        public string MovedToAcct { get; set; }
+
+        public bool Deleted { get; set; }
     }
 }

src/Tests/BSLManager.Tests/BSLManager.Tests.csproj

@@ -1,20 +1,21 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.1.1" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.1.1" />
-    <PackageReference Include="coverlet.collector" Version="1.3.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.0" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.0.3" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.0.3" />
+    <PackageReference Include="coverlet.collector" Version="6.0.0" />
+
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\..\BSLManager\BSLManager.csproj" />
   </ItemGroup>
 
-</Project>
+</Project>

src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj

@@ -1,20 +1,20 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
-    <PackageReference Include="coverlet.collector" Version="1.2.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.0" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.0.3" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.0.3" />
+    <PackageReference Include="coverlet.collector" Version="6.0.0" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\..\BirdsiteLive.ActivityPub\BirdsiteLive.ActivityPub.csproj" />
   </ItemGroup>
 
-</Project>
+</Project>

src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj

@@ -1,20 +1,20 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.1.1" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.1.1" />
-    <PackageReference Include="coverlet.collector" Version="1.3.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.0" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.0.3" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.0.3" />
+    <PackageReference Include="coverlet.collector" Version="6.0.0" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\..\BirdsiteLive.Common\BirdsiteLive.Common.csproj" />
   </ItemGroup>
 
-</Project>
+</Project>

src/Tests/BirdsiteLive.Common.Tests/Regexes/UrlRegexesTests.cs

@@ -0,0 +1,72 @@
+using BirdsiteLive.Common.Regexes;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace BirdsiteLive.Common.Tests
+{
+    [TestClass]
+    public class UrlRegexesTests
+    {
+        [TestMethod]
+        public void Url_Test()
+        {
+            var input = "https://misskey.tdl/users/8hwf6zy2k1#main-key";
+            Assert.IsTrue(UrlRegexes.Url.IsMatch(input));
+        }
+
+        [TestMethod]
+        public void Url_Not_Test()
+        {
+            var input = "misskey.tdl/users/8hwf6zy2k1#main-key";
+            Assert.IsFalse(UrlRegexes.Url.IsMatch(input));
+        }
+
+        [TestMethod]
+        public void Domain_Test()
+        {
+            var input = "misskey-data_sq.tdl";
+            Assert.IsTrue(UrlRegexes.Domain.IsMatch(input));
+        }
+
+        [TestMethod]
+        public void Domain_Numbers_Test()
+        {
+            var input = "miss45654QAzedqskey-data_sq.tdl";
+            Assert.IsTrue(UrlRegexes.Domain.IsMatch(input));
+        }
+
+        [TestMethod]
+        public void Domain_Subdomain_Test()
+        {
+            var input = "s.sub.dqdq-_Dz9sd.tdl";
+            Assert.IsTrue(UrlRegexes.Domain.IsMatch(input));
+        }
+
+        [TestMethod]
+        public void Domain_Not_Test()
+        {
+            var input = "mis$s45654QAzedqskey-data_sq.tdl";
+            Assert.IsFalse(UrlRegexes.Domain.IsMatch(input));
+        }
+
+        [TestMethod]
+        public void Domain_Slash_Test()
+        {
+            var input = "miss45654QAz/edqskey-data_sq.tdl";
+            Assert.IsFalse(UrlRegexes.Domain.IsMatch(input));
+        }
+
+        [TestMethod]
+        public void Domain_NotSub_Test()
+        {
+            var input = ".mis$s45654QAzedqskey-data_sq.tdl";
+            Assert.IsFalse(UrlRegexes.Domain.IsMatch(input));
+        }
+
+        [TestMethod]
+        public void Domain_NotExt_Test()
+        {
+            var input = ".mis$s45654QAzedqskey-data_sq.tdl";
+            Assert.IsFalse(UrlRegexes.Domain.IsMatch(input));
+        }
+    }
+}

src/Tests/BirdsiteLive.Cryptography.Tests/BirdsiteLive.Cryptography.Tests.csproj

@@ -1,20 +1,20 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
-    <PackageReference Include="coverlet.collector" Version="1.2.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.0" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.0.3" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.0.3" />
+    <PackageReference Include="coverlet.collector" Version="6.0.0" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\..\BirdsiteLive.Cryptography\BirdsiteLive.Cryptography.csproj" />
   </ItemGroup>
 
-</Project>
+</Project>

src/Tests/BirdsiteLive.DAL.Postgres.Tests/BirdsiteLive.DAL.Postgres.Tests.csproj

@@ -1,16 +1,16 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
-    <PackageReference Include="coverlet.collector" Version="1.2.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.0" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.0.3" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.0.3" />
+    <PackageReference Include="coverlet.collector" Version="6.0.0" />
   </ItemGroup>
 
   <ItemGroup>

src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/Base/PostgresTestingBase.cs

@@ -14,7 +14,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers.Base
         {
             _settings = new PostgresSettings
             {
-                ConnString = "Host=127.0.0.1;Username=postgres;Password=mysecretpassword;Database=mytestdb",
+                ConnString = "Host=127.0.0.1;Username=birdtest;Password=mysecretpassword;Database=birdsitetest",
                 DbVersionTableName = "DbVersionTableName" + RandomGenerator.GetString(4),
                 CachedTweetsTableName = "CachedTweetsTableName" + RandomGenerator.GetString(4),
                 FollowersTableName = "FollowersTableName" + RandomGenerator.GetString(4),

src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs

@@ -71,6 +71,28 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             Assert.AreEqual(result.Id, resultById.Id);
         }
 
+        [TestMethod]
+        public async Task CreateAndGetMigratedUser_byId()
+        {
+            var acct = "myid";
+            var lastTweetId = 1548L;
+            var movedTo = "https://";
+            var movedToAcct = "@account@instance";
+
+            var dal = new TwitterUserPostgresDal(_settings);
+
+            await dal.CreateTwitterUserAsync(acct, lastTweetId, movedTo, movedToAcct);
+            var result = await dal.GetTwitterUserAsync(acct);
+            var resultById = await dal.GetTwitterUserAsync(result.Id);
+
+            Assert.AreEqual(acct, resultById.Acct);
+            Assert.AreEqual(lastTweetId, resultById.LastTweetPostedId);
+            Assert.AreEqual(lastTweetId, resultById.LastTweetSynchronizedForAllFollowersId);
+            Assert.AreEqual(result.Id, resultById.Id);
+            Assert.AreEqual(movedTo, result.MovedTo);
+            Assert.AreEqual(movedToAcct, result.MovedToAcct);
+        }
+
         [TestMethod]
         public async Task CreateUpdateAndGetUser()
         {
@@ -87,7 +109,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var updatedLastSyncId = 1550L;
             var now = DateTime.Now;
             var errors = 15;
-            await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now);
+            await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now, null, null, false);
 
             result = await dal.GetTwitterUserAsync(acct);
 
@@ -96,6 +118,68 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId);
             Assert.AreEqual(errors, result.FetchingErrorCount);
             Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100);
+            Assert.AreEqual(null, result.MovedTo);
+            Assert.AreEqual(null, result.MovedToAcct);
+        }
+
+        [TestMethod]
+        public async Task CreateUpdateAndGetMigratedUser()
+        {
+            var acct = "myid";
+            var lastTweetId = 1548L;
+
+            var dal = new TwitterUserPostgresDal(_settings);
+
+            await dal.CreateTwitterUserAsync(acct, lastTweetId);
+            var result = await dal.GetTwitterUserAsync(acct);
+
+
+            var updatedLastTweetId = 1600L;
+            var updatedLastSyncId = 1550L;
+            var now = DateTime.Now;
+            var errors = 15;
+            var movedTo = "https://";
+            var movedToAcct = "@account@instance";
+            await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now, movedTo, movedToAcct, false);
+
+            result = await dal.GetTwitterUserAsync(acct);
+
+            Assert.AreEqual(acct, result.Acct);
+            Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId);
+            Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId);
+            Assert.AreEqual(errors, result.FetchingErrorCount);
+            Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100);
+            Assert.AreEqual(movedTo, result.MovedTo);
+            Assert.AreEqual(movedToAcct, result.MovedToAcct);
+        }
+
+        [TestMethod]
+        public async Task CreateUpdateAndGetDeletedUser()
+        {
+            var acct = "myid";
+            var lastTweetId = 1548L;
+
+            var dal = new TwitterUserPostgresDal(_settings);
+
+            await dal.CreateTwitterUserAsync(acct, lastTweetId);
+            var result = await dal.GetTwitterUserAsync(acct);
+
+            var updatedLastTweetId = 1600L;
+            var updatedLastSyncId = 1550L;
+            var now = DateTime.Now;
+            var errors = 15;
+            await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now, null, null, true);
+
+            result = await dal.GetTwitterUserAsync(acct);
+
+            Assert.AreEqual(acct, result.Acct);
+            Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId);
+            Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId);
+            Assert.AreEqual(errors, result.FetchingErrorCount);
+            Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100);
+            Assert.AreEqual(null, result.MovedTo);
+            Assert.AreEqual(null, result.MovedToAcct);
+            Assert.AreEqual(true, result.Deleted);
         }
 
         [TestMethod]
@@ -167,7 +251,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
         public async Task Update_NoId()
         {
             var dal = new TwitterUserPostgresDal(_settings);
-            await dal.UpdateTwitterUserAsync(default, default, default, default, DateTime.UtcNow);
+            await dal.UpdateTwitterUserAsync(default, default, default, default, DateTime.UtcNow, null, null, false);
         }
 
         [TestMethod]
@@ -175,7 +259,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
         public async Task Update_NoLastTweetPostedId()
         {
             var dal = new TwitterUserPostgresDal(_settings);
-            await dal.UpdateTwitterUserAsync(12, default, default, default, DateTime.UtcNow);
+            await dal.UpdateTwitterUserAsync(12, default, default, default, DateTime.UtcNow, null, null, false);
         }
 
         [TestMethod]
@@ -183,7 +267,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
         public async Task Update_NoLastTweetSynchronizedForAllFollowersId()
         {
             var dal = new TwitterUserPostgresDal(_settings);
-            await dal.UpdateTwitterUserAsync(12, 9556, default, default, DateTime.UtcNow);
+            await dal.UpdateTwitterUserAsync(12, 9556, default, default, DateTime.UtcNow, null, null, false);
         }
 
         [TestMethod]
@@ -191,7 +275,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
         public async Task Update_NoLastSync()
         {
             var dal = new TwitterUserPostgresDal(_settings);
-            await dal.UpdateTwitterUserAsync(12, 9556, 65, default, default);
+            await dal.UpdateTwitterUserAsync(12, 9556, 65, default, default, null, null, false);
         }
 
         [TestMethod]
@@ -256,12 +340,79 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
                 await dal.CreateTwitterUserAsync(acct, lastTweetId);
             }
 
-            var result = await dal.GetAllTwitterUsersAsync(1000);
+            for (int i = 0; i < 10; i++)
+            {
+                var acct = $"migrated-myid{i}";
+                var lastTweetId = 1548L;
+
+                await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain");
+            }
+
+            for (int i = 0; i < 10; i++)
+            {
+                var acct = $"deleted-myid{i}";
+                var lastTweetId = 148L;
+
+                await dal.CreateTwitterUserAsync(acct, lastTweetId);
+                var user = await dal.GetTwitterUserAsync(acct);
+                user.Deleted = true;
+                user.LastSync = DateTime.UtcNow;
+                await dal.UpdateTwitterUserAsync(user);
+            }
+
+            var result = await dal.GetAllTwitterUsersAsync(1100, false);
             Assert.AreEqual(1000, result.Length);
             Assert.IsFalse(result[0].Id == default);
             Assert.IsFalse(result[0].Acct == default);
             Assert.IsFalse(result[0].LastTweetPostedId == default);
             Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
+
+            foreach (var user in result)
+            {
+                Assert.IsTrue(string.IsNullOrWhiteSpace(user.MovedTo));
+                Assert.IsTrue(string.IsNullOrWhiteSpace(user.MovedToAcct));
+                Assert.IsFalse(user.Deleted);
+            }
+        }
+
+        [TestMethod]
+        public async Task GetAllTwitterUsers_Top_RetrieveDeleted()
+        {
+            var dal = new TwitterUserPostgresDal(_settings);
+            for (var i = 0; i < 1000; i++)
+            {
+                var acct = $"myid{i}";
+                var lastTweetId = 1548L;
+
+                await dal.CreateTwitterUserAsync(acct, lastTweetId);
+            }
+
+            for (int i = 0; i < 10; i++)
+            {
+                var acct = $"migrated-myid{i}";
+                var lastTweetId = 1548L;
+
+                await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain");
+            }
+
+            for (int i = 0; i < 10; i++)
+            {
+                var acct = $"deleted-myid{i}";
+                var lastTweetId = 148L;
+
+                await dal.CreateTwitterUserAsync(acct, lastTweetId);
+                var user = await dal.GetTwitterUserAsync(acct);
+                user.Deleted = true;
+                user.LastSync = DateTime.UtcNow;
+                await dal.UpdateTwitterUserAsync(user);
+            }
+
+            var result = await dal.GetAllTwitterUsersAsync(1100, true);
+            Assert.AreEqual(1020, result.Length);
+            Assert.IsFalse(result[0].Id == default);
+            Assert.IsFalse(result[0].Acct == default);
+            Assert.IsFalse(result[0].LastTweetPostedId == default);
+            Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
         }
 
         [TestMethod]
@@ -279,7 +430,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
 
             // Update accounts
             var now = DateTime.UtcNow;
-            var allUsers = await dal.GetAllTwitterUsersAsync();
+            var allUsers = await dal.GetAllTwitterUsersAsync(false);
             foreach (var acc in allUsers)
             {
                 var lastSync = now.AddDays(acc.LastTweetPostedId);
@@ -290,7 +441,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             // Create a not init account
             await dal.CreateTwitterUserAsync("not_init", -1);
 
-            var result = await dal.GetAllTwitterUsersAsync(10);
+            var result = await dal.GetAllTwitterUsersAsync(10, false);
 
             Assert.IsTrue(result.Any(x => x.Acct == "myid0"));
             Assert.IsTrue(result.Any(x => x.Acct == "myid8"));
@@ -313,15 +464,15 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
                 await dal.CreateTwitterUserAsync(acct, lastTweetId);
             }
 
-            var allUsers = await dal.GetAllTwitterUsersAsync(100);
+            var allUsers = await dal.GetAllTwitterUsersAsync(100, false);
             for (var i = 0; i < 20; i++)
             {
                 var user = allUsers[i];
                 var date = i % 2 == 0 ? oldest : newest;
-                await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date);
+                await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date, null, null, false);
             }
 
-            var result = await dal.GetAllTwitterUsersAsync(10);
+            var result = await dal.GetAllTwitterUsersAsync(10, false);
             Assert.AreEqual(10, result.Length);
             Assert.IsFalse(result[0].Id == default);
             Assert.IsFalse(result[0].Acct == default);
@@ -344,7 +495,15 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
                 await dal.CreateTwitterUserAsync(acct, lastTweetId);
             }
 
-            var result = await dal.GetAllTwitterUsersAsync();
+            for (int i = 0; i < 10; i++)
+            {
+                var acct = $"migrated-myid{i}";
+                var lastTweetId = 1548L;
+
+                await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain");
+            }
+
+            var result = await dal.GetAllTwitterUsersAsync(false);
             Assert.AreEqual(1000, result.Length);
             Assert.IsFalse(result[0].Id == default);
             Assert.IsFalse(result[0].Acct == default);
@@ -382,7 +541,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
                 if (i == 0 || i == 2 || i == 3)
                 {
                     var t = await dal.GetTwitterUserAsync(acct);
-                    await dal.UpdateTwitterUserAsync(t.Id ,1L,2L, 50+i*2, DateTime.Now);
+                    await dal.UpdateTwitterUserAsync(t.Id ,1L,2L, 50+i*2, DateTime.Now, null, null, false);
                 }
             }
 

src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj

@@ -1,21 +1,21 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
-    <PackageReference Include="Moq" Version="4.14.5" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
-    <PackageReference Include="coverlet.collector" Version="1.2.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.0" />
+    <PackageReference Include="Moq" Version="4.18.4" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.0.3" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.0.3" />
+    <PackageReference Include="coverlet.collector" Version="6.0.0" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
   </ItemGroup>
 
-</Project>
+</Project>

src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj

@@ -1,17 +1,17 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
-    <PackageReference Include="Moq" Version="4.14.5" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
-    <PackageReference Include="coverlet.collector" Version="1.2.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.0" />
+    <PackageReference Include="Moq" Version="4.18.4" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.0.3" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.0.3" />
+    <PackageReference Include="coverlet.collector" Version="6.0.0" />
   </ItemGroup>
 
   <ItemGroup>
@@ -22,4 +22,4 @@
     <Folder Include="Repository\" />
   </ItemGroup>
 
-</Project>
+</Project>

src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessFollowUserTests.cs

@@ -77,7 +77,9 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
             twitterUserDalMock
                 .Setup(x => x.CreateTwitterUserAsync(
                     It.Is<string>(y => y == twitterName),
-                    It.Is<long>(y => y == -1)))
+                    It.Is<long>(y => y == -1),
+                    It.Is<string>(y => y == null),
+                    It.Is<string>(y => y == null)))
                 .Returns(Task.CompletedTask);
             #endregion
 

src/Tests/BirdsiteLive.Domain.Tests/TheFedInfoServiceTests.cs

@@ -0,0 +1,36 @@
+using System;
+using System.Linq;
+using System.Net.Http;
+using System.Threading.Tasks;
+using BirdsiteLive.Common.Regexes;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Moq;
+
+namespace BirdsiteLive.Domain.Tests
+{
+    [TestClass]
+    public class TheFedInfoServiceTests
+    {
+        [TestMethod]
+        public async Task GetBslInstanceListAsyncTest()
+        {
+            var httpClientFactoryMock = new Mock<IHttpClientFactory>();
+            httpClientFactoryMock
+                .Setup(x => x.CreateClient(It.IsAny<string>()))
+                .Returns(new HttpClient());
+
+            var service = new TheFedInfoService(httpClientFactoryMock.Object);
+
+            var bslInstanceList = await service.GetBslInstanceListAsync();
+
+            Assert.IsTrue(bslInstanceList.Count > 0);
+
+            foreach (var instanceInfo in bslInstanceList)
+            {
+                Assert.IsFalse(string.IsNullOrWhiteSpace(instanceInfo.Host));
+                Assert.IsTrue(UrlRegexes.Domain.IsMatch(instanceInfo.Host));
+                Assert.IsTrue(instanceInfo.Version > new Version(0, 1, 0));
+            }
+        }
+    }
+}

src/Tests/BirdsiteLive.Moderation.Tests/BirdsiteLive.Moderation.Tests.csproj

@@ -1,21 +1,21 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
-    <PackageReference Include="Moq" Version="4.14.5" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.1.1" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.1.1" />
-    <PackageReference Include="coverlet.collector" Version="1.3.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.0" />
+    <PackageReference Include="Moq" Version="4.18.4" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.0.3" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.0.3" />
+    <PackageReference Include="coverlet.collector" Version="6.0.0" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\..\BirdsiteLive.Moderation\BirdsiteLive.Moderation.csproj" />
   </ItemGroup>
 
-</Project>
+</Project>

src/Tests/BirdsiteLive.Moderation.Tests/Processors/TwitterAccountModerationProcessorTests.cs

@@ -48,7 +48,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
-                .Setup(x => x.GetAllTwitterUsersAsync())
+                .Setup(x => x.GetAllTwitterUsersAsync(It.Is<bool>(y => y == false)))
                 .ReturnsAsync(allUsers.ToArray());
 
             var moderationRepositoryMock = new Mock<IModerationRepository>(MockBehavior.Strict);
@@ -87,7 +87,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
-                .Setup(x => x.GetAllTwitterUsersAsync())
+                .Setup(x => x.GetAllTwitterUsersAsync(It.Is<bool>(y => y == false)))
                 .ReturnsAsync(allUsers.ToArray());
 
             var moderationRepositoryMock = new Mock<IModerationRepository>(MockBehavior.Strict);
@@ -130,7 +130,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
-                .Setup(x => x.GetAllTwitterUsersAsync())
+                .Setup(x => x.GetAllTwitterUsersAsync(It.Is<bool>(y => y == false)))
                 .ReturnsAsync(allUsers.ToArray());
 
             var moderationRepositoryMock = new Mock<IModerationRepository>(MockBehavior.Strict);
@@ -173,7 +173,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
-                .Setup(x => x.GetAllTwitterUsersAsync())
+                .Setup(x => x.GetAllTwitterUsersAsync(It.Is<bool>(y => y == false)))
                 .ReturnsAsync(allUsers.ToArray());
 
             var moderationRepositoryMock = new Mock<IModerationRepository>(MockBehavior.Strict);

src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj

@@ -1,17 +1,17 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
-    <PackageReference Include="Moq" Version="4.14.5" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
-    <PackageReference Include="coverlet.collector" Version="1.2.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.0" />
+    <PackageReference Include="Moq" Version="4.18.4" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.0.3" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.0.3" />
+    <PackageReference Include="coverlet.collector" Version="6.0.0" />
   </ItemGroup>
 
   <ItemGroup>
@@ -24,4 +24,4 @@
     <ProjectReference Include="..\..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
   </ItemGroup>
 
-</Project>
+</Project>

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs

@@ -64,7 +64,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                     It.Is<long>(y => y == tweets.Last().Id),
                     It.Is<long>(y => y == tweets.Last().Id),
                     It.Is<int>(y => y == 0),
-                    It.IsAny<DateTime>()
+                    It.IsAny<DateTime>(),
+                    It.Is<string>(y => y == null),
+                    It.Is<string>(y => y == null),
+                    It.Is<bool>(y => y == false)
                 ))
                 .Returns(Task.CompletedTask);
 

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs

@@ -40,7 +40,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .Setup(x => x.GetAllTwitterUsersAsync(
-                    It.Is<int>(y => y == maxUsers)))
+                    It.Is<int>(y => y == maxUsers),
+                    It.Is<bool>(y => y == false)))
                 .ReturnsAsync(users);
             
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
@@ -83,7 +84,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .SetupSequence(x => x.GetAllTwitterUsersAsync(
-                    It.Is<int>(y => y == maxUsers)))
+                    It.Is<int>(y => y == maxUsers),
+                    It.Is<bool>(y => y == false)))
                 .ReturnsAsync(users.ToArray())
                 .ReturnsAsync(new SyncTwitterUser[0])
                 .ReturnsAsync(new SyncTwitterUser[0])
@@ -130,7 +132,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .SetupSequence(x => x.GetAllTwitterUsersAsync(
-                    It.Is<int>(y => y == maxUsers)))
+                    It.Is<int>(y => y == maxUsers),
+                    It.Is<bool>(y => y == false)))
                 .ReturnsAsync(users.ToArray())
                 .ReturnsAsync(new SyncTwitterUser[0])
                 .ReturnsAsync(new SyncTwitterUser[0])
@@ -178,7 +181,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .Setup(x => x.GetAllTwitterUsersAsync(
-                    It.Is<int>(y => y == maxUsers)))
+                    It.Is<int>(y => y == maxUsers),
+                    It.Is<bool>(y => y == false)))
                 .ReturnsAsync(new SyncTwitterUser[0]);
 
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
@@ -215,7 +219,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .Setup(x => x.GetAllTwitterUsersAsync(
-                    It.Is<int>(y => y == maxUsers)))
+                    It.Is<int>(y => y == maxUsers),
+                    It.Is<bool>(y => y == false)))
                 .Returns(async () => await DelayFaultedTask<SyncTwitterUser[]>(new Exception()));
 
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs

@@ -1,16 +1,16 @@
-using System;
-using System.Collections.Generic;
-using System.Threading;
-using System.Threading.Tasks;
-using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Moderation.Actions;
 using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Pipeline.Processors;
 using BirdsiteLive.Twitter.Models;
-using Castle.DynamicProxy.Contributors;
 using Microsoft.Extensions.Logging;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Moq;
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
 
 namespace BirdsiteLive.Pipeline.Tests.Processors
 {
@@ -66,17 +66,93 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                     It.Is<long>(y => y == tweet2.Id),
                     It.Is<long>(y => y == tweet2.Id),
                     It.Is<int>(y => y == 0),
-                    It.IsAny<DateTime>()
+                    It.IsAny<DateTime>(),
+                    It.Is<string>(y => y == null),
+                    It.Is<string>(y => y == null),
+                    It.Is<bool>(y => y == false)
                 ))
                 .Returns(Task.CompletedTask);
+
+            var removeTwitterAccountActionMock = new Mock<IRemoveTwitterAccountAction>(MockBehavior.Strict);
             #endregion
 
-            var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object);
+            var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object, removeTwitterAccountActionMock.Object);
             await processor.ProcessAsync(usersWithTweets, CancellationToken.None);
 
             #region Validations
             twitterUserDalMock.VerifyAll();
             loggerMock.VerifyAll();
+            removeTwitterAccountActionMock.VerifyAll();
+            #endregion
+        }
+
+        [TestMethod]
+        [ExpectedException(typeof(ArgumentException))]
+        public async Task ProcessAsync_Exception_Test()
+        {
+            #region Stubs
+            var user = new SyncTwitterUser
+            {
+                Id = 1
+            };
+            var tweet1 = new ExtractedTweet
+            {
+                Id = 36
+            };
+            var tweet2 = new ExtractedTweet
+            {
+                Id = 37
+            };
+            var follower1 = new Follower
+            {
+                FollowingsSyncStatus = new Dictionary<int, long>
+                {
+                    {1, 37}
+                }
+            };
+
+            var usersWithTweets = new UserWithDataToSync
+            {
+                Tweets = new[]
+                {
+                    tweet1,
+                    tweet2
+                },
+                Followers = new[]
+                {
+                    follower1
+                },
+                User = user
+            };
+
+            var loggerMock = new Mock<ILogger<SaveProgressionProcessor>>();
+            #endregion
+
+            #region Mocks
+            var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
+            twitterUserDalMock
+                .Setup(x => x.UpdateTwitterUserAsync(
+                    It.Is<int>(y => y == user.Id),
+                    It.Is<long>(y => y == tweet2.Id),
+                    It.Is<long>(y => y == tweet2.Id),
+                    It.Is<int>(y => y == 0),
+                    It.IsAny<DateTime>(),
+                    It.Is<string>(y => y == null),
+                    It.Is<string>(y => y == null),
+                    It.Is<bool>(y => y == false)
+                ))
+                .Throws(new ArgumentException());
+
+            var removeTwitterAccountActionMock = new Mock<IRemoveTwitterAccountAction>(MockBehavior.Strict);
+            #endregion
+
+            var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object, removeTwitterAccountActionMock.Object);
+            await processor.ProcessAsync(usersWithTweets, CancellationToken.None);
+
+            #region Validations
+            twitterUserDalMock.VerifyAll();
+            loggerMock.VerifyAll();
+            removeTwitterAccountActionMock.VerifyAll();
             #endregion
         }
 
@@ -132,19 +208,25 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                     It.Is<long>(y => y == tweet3.Id),
                     It.Is<long>(y => y == tweet2.Id),
                     It.Is<int>(y => y == 0),
-                    It.IsAny<DateTime>()
+                    It.IsAny<DateTime>(),
+                    It.Is<string>(y => y == null),
+                    It.Is<string>(y => y == null),
+                    It.Is<bool>(y => y == false)
                 ))
                 .Returns(Task.CompletedTask);
 
             var loggerMock = new Mock<ILogger<SaveProgressionProcessor>>();
+
+            var removeTwitterAccountActionMock = new Mock<IRemoveTwitterAccountAction>(MockBehavior.Strict);
             #endregion
 
-            var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object);
+            var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object, removeTwitterAccountActionMock.Object);
             await processor.ProcessAsync(usersWithTweets, CancellationToken.None);
 
             #region Validations
             twitterUserDalMock.VerifyAll();
             loggerMock.VerifyAll();
+            removeTwitterAccountActionMock.VerifyAll();
             #endregion
         }
 
@@ -208,20 +290,130 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                     It.Is<long>(y => y == tweet3.Id),
                     It.Is<long>(y => y == tweet2.Id),
                     It.Is<int>(y => y == 0),
-                    It.IsAny<DateTime>()
+                    It.IsAny<DateTime>(),
+                    It.Is<string>(y => y == null),
+                    It.Is<string>(y => y == null),
+                    It.Is<bool>(y => y == false)
                 ))
                 .Returns(Task.CompletedTask);
 
             var loggerMock = new Mock<ILogger<SaveProgressionProcessor>>();
+
+            var removeTwitterAccountActionMock = new Mock<IRemoveTwitterAccountAction>(MockBehavior.Strict);
             #endregion
 
-            var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object);
+            var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object, removeTwitterAccountActionMock.Object);
             await processor.ProcessAsync(usersWithTweets, CancellationToken.None);
 
             #region Validations
             twitterUserDalMock.VerifyAll();
             loggerMock.VerifyAll();
+            removeTwitterAccountActionMock.VerifyAll();
             #endregion
         }
+
+        [TestMethod]
+        public async Task ProcessAsync_NoTweets_Test()
+        {
+            #region Stubs
+            var user = new SyncTwitterUser
+            {
+                Id = 1,
+                LastTweetPostedId = 42, 
+                LastSync = DateTime.UtcNow.AddDays(-3)
+            };
+            var follower1 = new Follower
+            {
+                FollowingsSyncStatus = new Dictionary<int, long>
+                {
+                    {1, 37}
+                }
+            };
+
+            var usersWithTweets = new UserWithDataToSync
+            {
+                Tweets = Array.Empty<ExtractedTweet>(),
+                Followers = new[]
+                {
+                    follower1
+                },
+                User = user
+            };
+
+            var loggerMock = new Mock<ILogger<SaveProgressionProcessor>>();
+            #endregion
+
+            #region Mocks
+            var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
+            twitterUserDalMock
+                .Setup(x => x.UpdateTwitterUserAsync(
+                    It.Is<SyncTwitterUser>(y => y.LastTweetPostedId == 42 
+                                                && y.LastSync > DateTime.UtcNow.AddDays(-1))
+                ))
+                .Returns(Task.CompletedTask);
+
+            var removeTwitterAccountActionMock = new Mock<IRemoveTwitterAccountAction>(MockBehavior.Strict);
+            #endregion
+
+            var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object, removeTwitterAccountActionMock.Object);
+            await processor.ProcessAsync(usersWithTweets, CancellationToken.None);
+
+            #region Validations
+            twitterUserDalMock.VerifyAll();
+            loggerMock.VerifyAll();
+            removeTwitterAccountActionMock.VerifyAll();
+            #endregion
+        }
+
+        [TestMethod]
+        public async Task ProcessAsync_NoFollower_Test()
+        {
+            #region Stubs
+            var user = new SyncTwitterUser
+            {
+                Id = 1
+            };
+            var tweet1 = new ExtractedTweet
+            {
+                Id = 36
+            };
+            var tweet2 = new ExtractedTweet
+            {
+                Id = 37
+            };
+
+            var usersWithTweets = new UserWithDataToSync
+            {
+                Tweets = new[]
+                {
+                    tweet1,
+                    tweet2
+                },
+                Followers = Array.Empty<Follower>(),
+                User = user
+            };
+
+            var loggerMock = new Mock<ILogger<SaveProgressionProcessor>>();
+            #endregion
+
+            #region Mocks
+            var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
+
+            var removeTwitterAccountActionMock = new Mock<IRemoveTwitterAccountAction>(MockBehavior.Strict);
+            removeTwitterAccountActionMock
+                .Setup(x => x.ProcessAsync(It.Is<SyncTwitterUser>(y => y.Id == user.Id)))
+                .Returns(Task.CompletedTask);
+            #endregion
+
+            var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object, removeTwitterAccountActionMock.Object);
+            await processor.ProcessAsync(usersWithTweets, CancellationToken.None);
+
+            #region Validations
+            twitterUserDalMock.VerifyAll();
+            loggerMock.VerifyAll();
+            removeTwitterAccountActionMock.VerifyAll();
+            #endregion
+        }
+
     }
 }