diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..7847aae --- /dev/null +++ b/.drone.yml @@ -0,0 +1,59 @@ +kind: pipeline +name: testing +type: docker + +steps: + - name: Install Dependencies + image: mcr.microsoft.com/dotnet/sdk:6.0 + commands: + - dotnet restore ./src + + - name: Build + image: mcr.microsoft.com/dotnet/sdk:6.0 + commands: + - dotnet build --configuration Release ./src + + - name: Test + image: mcr.microsoft.com/dotnet/sdk:6.0 + commands: + - sed -i "s/127\.0\.0\.1/database/g" ./src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/Base/PostgresTestingBase.cs + - dotnet test --verbosity minimal ./src + +services: + - name: database + image: postgres:15 + environment: + POSTGRES_USER: birdtest + POSTGRES_PASSWORD: mysecretpassword + POSTGRES_DB: birdsitetest + +--- + +kind: pipeline +name: docker-publish +type: docker + +depends_on: + - testing + +steps: + - name: Build & Publish + privileged: true + image: quay.io/thegeeklab/drone-docker-buildx + settings: + auto_tag: true + repo: git.froth.zone/sam/birdsitelive + registry: git.froth.zone + username: sam + password: + from_secret: password + platforms: + - linux/amd64 + - linux/arm64 + when: + branch: + - master + event: + - push + depends_on: + - "clone" \ No newline at end of file diff --git a/.gitignore b/.gitignore index a6584fc..28aa329 100644 --- a/.gitignore +++ b/.gitignore @@ -352,3 +352,5 @@ MigrationBackup/ # Ionide (cross platform F# VS Code tools) working folder .ionide/ /src/BSLManager/Properties/launchSettings.json + +.dccache \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 11a4422..ddea49f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,15 @@ #See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging. -FROM mcr.microsoft.com/dotnet/aspnet:3.1-buster-slim AS base +FROM mcr.microsoft.com/dotnet/aspnet:6.0 AS base WORKDIR /app EXPOSE 80 EXPOSE 443 -FROM mcr.microsoft.com/dotnet/sdk:3.1-buster AS publish +FROM mcr.microsoft.com/dotnet/sdk:6.0 AS publish +WORKDIR / COPY ./src/ ./src/ -RUN dotnet publish "/src/BirdsiteLive/BirdsiteLive.csproj" -c Release -o /app/publish -RUN dotnet publish "/src/BSLManager/BSLManager.csproj" -r linux-x64 --self-contained true -p:PublishSingleFile=true -p:IncludeAllContentForSelfExtract=true -c Release -o /app/publish +RUN dotnet publish "/src/BirdsiteLive/BirdsiteLive.csproj" -c Release -o /app/publish \ + && dotnet publish "/src/BSLManager/BSLManager.csproj" -r linux-x64 --self-contained true -p:PublishSingleFile=true -p:IncludeAllContentForSelfExtract=true -c Release -o /app/publish FROM base AS final WORKDIR /app diff --git a/INSTALLATION.md b/INSTALLATION.md index 547299d..582401b 100644 --- a/INSTALLATION.md +++ b/INSTALLATION.md @@ -4,6 +4,9 @@ You will need a Twitter API key to make BirdsiteLIVE working. First create an **Standalone App** in the [Twitter developer portal](https://developer.twitter.com/en/portal/projects-and-apps) and retrieve the API Key and API Secret Key. +Please make sure you are using a **Standalone App** API Key and not a **Project App** API Key (that will NOT work with BirdsiteLIVE), if you don't see the **Standalone App** section, you might need to [apply for Elevated Access](https://developer.twitter.com/en/portal/products/elevated) as described in the [API documentation](https://developer.twitter.com/en/support/twitter-api/developer-account). + + ## Server prerequisites Your instance will need [docker](https://docs.docker.com/engine/install/) and [docker-compose](https://docs.docker.com/compose/install/) installed and working. @@ -116,7 +119,7 @@ sudo certbot --nginx -d {your-domain-name.com} Make sure you're redirecting all traffic to https when asked. -Finally check that the auto-revewal will work as espected: +Finally check that the auto-renewal will work as espected: ``` sudo certbot renew --dry-run diff --git a/README.md b/README.md index f35e534..d450653 100644 --- a/README.md +++ b/README.md @@ -24,15 +24,15 @@ BirdsiteLIVE is an ActivityPub bridge from Twitter, it's mostly a pet project/pl ## State of development -The code is pretty messy and far from a good state, since it's a playground for me the aim was to understand some AP concepts, not provide a good state-of-the-art codebase. But I might refactor it to make it cleaner. +The code is pretty messy and far from a good state, since it's a playground for me the aim was to understand some AP concepts, not to provide a good state-of-the-art codebase. But I might refactor it to make it cleaner. ## Official instance -You can find an official (and temporary) instance here: [beta.birdsite.live](https://beta.birdsite.live). This instance can disapear at any time, if you want a long term instance you should install your own or use another one. +There's none! Please read [here why I've stopped it](https://write.as/nicolas-constant/closing-the-official-bsl-instance). ## Installation -I'm providing a [docker build](https://hub.docker.com/r/nicolasconstant/birdsitelive). To install it on your own server, please follow [those instructions](https://github.com/NicolasConstant/BirdsiteLive/blob/master/INSTALLATION.md). More [options](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md) are also available. +I'm providing a [docker build](https://hub.docker.com/r/nicolasconstant/birdsitelive) (linux/amd64 only). To install it on your own server, please follow [those instructions](https://github.com/NicolasConstant/BirdsiteLive/blob/master/INSTALLATION.md). More [options](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md) are also available. Also a [CLI](https://github.com/NicolasConstant/BirdsiteLive/blob/master/BSLManager.md) is available for adminitrative tasks. diff --git a/VARIABLES.md b/VARIABLES.md index ddc1f75..937f66a 100644 --- a/VARIABLES.md +++ b/VARIABLES.md @@ -58,6 +58,8 @@ If both whitelisting and blacklisting are set, only the whitelisting will be act * `Instance:FailingTwitterUserCleanUpThreshold` (default: 700) set the max allowed errors (due to a banned/deleted/private account) from a Twitter Account retrieval before auto-removal. (by default an account is called every 15 mins) * `Instance:MaxStatusFetchAge` (default: 0 - no limit) statuses with a Snowflake older than this age in days will not be fetched by the service and will instead return 410 Gone * `Instance:EnableQuoteRT` (default: false) enable Soapbox-style quote-RTs +* `Instance:FailingFollowerCleanUpThreshold` (default: 30000) set the max allowed errors from a Follower (Fediverse) Account before auto-removal. (often due to account suppression, instance issues, etc) +* `Instance:UserCacheCapacity` (default: 10000) set the caching limit of the Twitter User retrieval. Must be higher than the number of synchronized accounts on the instance. # Docker Compose full example diff --git a/docker-compose.yml b/docker-compose.yml index 351e192..54ac98c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ networks: services: server: - image: pasture/birdsitelive:latest + image: git.froth.zone/birdsitelive:latest restart: always container_name: birdsitelive environment: @@ -27,7 +27,7 @@ services: - db db: - image: postgres:13 + image: postgres:15 restart: always environment: - POSTGRES_USER=birdsitelive diff --git a/src/BSLManager/BSLManager.csproj b/src/BSLManager/BSLManager.csproj index 52e5cde..be41c6d 100644 --- a/src/BSLManager/BSLManager.csproj +++ b/src/BSLManager/BSLManager.csproj @@ -2,7 +2,7 @@ Exe - netcoreapp3.1 + net6.0 @@ -10,7 +10,7 @@ - + diff --git a/src/BirdsiteLive.ActivityPub/ApDeserializer.cs b/src/BirdsiteLive.ActivityPub/ApDeserializer.cs index 17dadbe..169bbe6 100644 --- a/src/BirdsiteLive.ActivityPub/ApDeserializer.cs +++ b/src/BirdsiteLive.ActivityPub/ApDeserializer.cs @@ -1,4 +1,5 @@ using System; +using BirdsiteLive.ActivityPub.Models; using Newtonsoft.Json; namespace BirdsiteLive.ActivityPub @@ -19,6 +20,8 @@ namespace BirdsiteLive.ActivityPub if(a.apObject.type == "Follow") return JsonConvert.DeserializeObject(json); break; + case "Delete": + return JsonConvert.DeserializeObject(json); case "Accept": var accept = JsonConvert.DeserializeObject(json); //var acceptType = JsonConvert.DeserializeObject(accept.apObject); diff --git a/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj b/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj index a690b63..d283d39 100644 --- a/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj +++ b/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj @@ -1,7 +1,7 @@ - netstandard2.0 + net6.0 diff --git a/src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs b/src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs new file mode 100644 index 0000000..c628a61 --- /dev/null +++ b/src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs @@ -0,0 +1,11 @@ +using Newtonsoft.Json; + +namespace BirdsiteLive.ActivityPub.Models +{ + public class ActivityDelete : Activity + { + public string[] to { get; set; } + [JsonProperty("object")] + public object apObject { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.ActivityPub/Models/Actor.cs b/src/BirdsiteLive.ActivityPub/Models/Actor.cs index 12905be..3461670 100644 --- a/src/BirdsiteLive.ActivityPub/Models/Actor.cs +++ b/src/BirdsiteLive.ActivityPub/Models/Actor.cs @@ -20,6 +20,7 @@ namespace BirdsiteLive.ActivityPub public string name { get; set; } public string summary { get; set; } public string url { get; set; } + public string movedTo { get; set; } public bool manuallyApprovesFollowers { get; set; } public string inbox { get; set; } public bool? discoverable { get; set; } = true; diff --git a/src/BirdsiteLive.Common/BirdsiteLive.Common.csproj b/src/BirdsiteLive.Common/BirdsiteLive.Common.csproj index 9f5c4f4..dbc1517 100644 --- a/src/BirdsiteLive.Common/BirdsiteLive.Common.csproj +++ b/src/BirdsiteLive.Common/BirdsiteLive.Common.csproj @@ -1,7 +1,7 @@ - netstandard2.0 + net6.0 diff --git a/src/BirdsiteLive.Common/Regexes/UrlRegexes.cs b/src/BirdsiteLive.Common/Regexes/UrlRegexes.cs index 1f2b279..cb93838 100644 --- a/src/BirdsiteLive.Common/Regexes/UrlRegexes.cs +++ b/src/BirdsiteLive.Common/Regexes/UrlRegexes.cs @@ -5,5 +5,7 @@ namespace BirdsiteLive.Common.Regexes public class UrlRegexes { public static readonly Regex Url = new Regex(@"(.?)(((http|ftp|https):\/\/)[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])?)"); + + public static readonly Regex Domain = new Regex(@"^[a-zA-Z0-9\-_]+(\.[a-zA-Z0-9\-_]+)+$"); } } \ No newline at end of file diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs index 73cc7ee..b1a9210 100644 --- a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs +++ b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs @@ -30,5 +30,8 @@ public int MaxStatusFetchAge { get; set; } public bool EnableQuoteRT { get; set; } + public int FailingFollowerCleanUpThreshold { get; set; } = -1; + + public int UserCacheCapacity { get; set; } } } diff --git a/src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj b/src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj index f0d9f4f..263ca43 100644 --- a/src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj +++ b/src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj @@ -1,7 +1,7 @@ - netstandard2.0 + net6.0 diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs index 71a1457..2e0129d 100644 --- a/src/BirdsiteLive.Domain/ActivityPubService.cs +++ b/src/BirdsiteLive.Domain/ActivityPubService.cs @@ -11,17 +11,24 @@ using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.Common.Settings; using Microsoft.Extensions.Logging; using Newtonsoft.Json; -using Org.BouncyCastle.Bcpg; namespace BirdsiteLive.Domain { public interface IActivityPubService { + Task GetUserIdAsync(string acct); Task GetUser(string objectId); Task PostDataAsync(T data, string targetHost, string actorUrl, string inbox = null); Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox); - Task WebFinger(string account); + Task DeleteUserAsync(string username, string targetHost, string targetInbox); + Task WebFinger(string account); + } + + public class WebFinger + { + public string subject { get; set; } + public string[] aliases { get; set; } } public class ActivityPubService : IActivityPubService @@ -41,11 +48,35 @@ namespace BirdsiteLive.Domain } #endregion + public async Task GetUserIdAsync(string acct) + { + var splittedAcct = acct.Trim('@').Split('@'); + + var url = $"https://{splittedAcct[1]}/.well-known/webfinger?resource=acct:{splittedAcct[0]}@{splittedAcct[1]}"; + + var httpClient = _httpClientFactory.CreateClient(); + httpClient.DefaultRequestHeaders.Add("Accept", "application/json"); + var result = await httpClient.GetAsync(url); + + result.EnsureSuccessStatusCode(); + + var content = await result.Content.ReadAsStringAsync(); + + var actor = JsonConvert.DeserializeObject(content); + return actor.aliases.FirstOrDefault(); + } + public async Task GetUser(string objectId) { var httpClient = _httpClientFactory.CreateClient(); httpClient.DefaultRequestHeaders.Add("Accept", "application/activity+json"); var result = await httpClient.GetAsync(objectId); + + if (result.StatusCode == HttpStatusCode.Gone) + throw new FollowerIsGoneException(); + + result.EnsureSuccessStatusCode(); + var content = await result.Content.ReadAsStringAsync(); var actor = JsonConvert.DeserializeObject(content); @@ -53,6 +84,31 @@ namespace BirdsiteLive.Domain return actor; } + public async Task DeleteUserAsync(string username, string targetHost, string targetInbox) + { + try + { + var actor = UrlFactory.GetActorUrl(_instanceSettings.Domain, username); + + var deleteUser = new ActivityDelete + { + context = "https://www.w3.org/ns/activitystreams", + id = $"{actor}#delete", + type = "Delete", + actor = actor, + to = new [] { "https://www.w3.org/ns/activitystreams#Public" }, + apObject = actor + }; + + await PostDataAsync(deleteUser, targetHost, actor, targetInbox); + } + catch (Exception e) + { + _logger.LogError(e, "Error deleting {Username} to {Host}{Inbox}", username, targetHost, targetInbox); + throw; + } + } + public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox) { try diff --git a/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj b/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj index 8c601b4..be5c3f6 100644 --- a/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj +++ b/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj @@ -1,7 +1,7 @@ - netstandard2.0 + net6.0 @@ -15,4 +15,8 @@ + + + + diff --git a/src/BirdsiteLive.Domain/BusinessUseCases/ProcessDeleteUser.cs b/src/BirdsiteLive.Domain/BusinessUseCases/ProcessDeleteUser.cs new file mode 100644 index 0000000..a35b6c8 --- /dev/null +++ b/src/BirdsiteLive.Domain/BusinessUseCases/ProcessDeleteUser.cs @@ -0,0 +1,51 @@ +using System.Linq; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; + +namespace BirdsiteLive.Domain.BusinessUseCases +{ + public interface IProcessDeleteUser + { + Task ExecuteAsync(Follower follower); + Task ExecuteAsync(string followerUsername, string followerDomain); + } + + public class ProcessDeleteUser : IProcessDeleteUser + { + private readonly IFollowersDal _followersDal; + private readonly ITwitterUserDal _twitterUserDal; + + #region Ctor + public ProcessDeleteUser(IFollowersDal followersDal, ITwitterUserDal twitterUserDal) + { + _followersDal = followersDal; + _twitterUserDal = twitterUserDal; + } + #endregion + + public async Task ExecuteAsync(string followerUsername, string followerDomain) + { + // Get Follower and Twitter Users + var follower = await _followersDal.GetFollowerAsync(followerUsername, followerDomain); + if (follower == null) return; + + await ExecuteAsync(follower); + } + + public async Task ExecuteAsync(Follower follower) + { + // Remove twitter users if no more followers + var followings = follower.Followings; + foreach (var following in followings) + { + var followers = await _followersDal.GetFollowersAsync(following); + if (followers.Length == 1 && followers.First().Id == follower.Id) + await _twitterUserDal.DeleteTwitterUserAsync(following); + } + + // Remove follower from DB + await _followersDal.DeleteFollowerAsync(follower.Id); + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/Enum/MigrationTypeEnum.cs b/src/BirdsiteLive.Domain/Enum/MigrationTypeEnum.cs new file mode 100644 index 0000000..92b1444 --- /dev/null +++ b/src/BirdsiteLive.Domain/Enum/MigrationTypeEnum.cs @@ -0,0 +1,9 @@ +namespace BirdsiteLive.Domain.Enum +{ + public enum MigrationTypeEnum + { + Unknown = 0, + Migration = 1, + Deletion = 2 + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/Exceptions/FollowerIsGoneException.cs b/src/BirdsiteLive.Domain/Exceptions/FollowerIsGoneException.cs new file mode 100644 index 0000000..3063cab --- /dev/null +++ b/src/BirdsiteLive.Domain/Exceptions/FollowerIsGoneException.cs @@ -0,0 +1,8 @@ +using System; + +namespace BirdsiteLive.Domain +{ + public class FollowerIsGoneException : Exception + { + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/MigrationService.cs b/src/BirdsiteLive.Domain/MigrationService.cs new file mode 100644 index 0000000..e0dfdb5 --- /dev/null +++ b/src/BirdsiteLive.Domain/MigrationService.cs @@ -0,0 +1,352 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using BirdsiteLive.Twitter; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; +using BirdsiteLive.ActivityPub; +using BirdsiteLive.ActivityPub.Models; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.ActivityPub.Converters; +using BirdsiteLive.Common.Settings; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Domain.Enum; +using System.Net.Http; +using BirdsiteLive.Common.Regexes; +using Microsoft.Extensions.Logging; + +namespace BirdsiteLive.Domain +{ + public class MigrationService + { + private readonly InstanceSettings _instanceSettings; + private readonly ITheFedInfoService _theFedInfoService; + private readonly ITwitterTweetsService _twitterTweetsService; + private readonly IActivityPubService _activityPubService; + private readonly ITwitterUserDal _twitterUserDal; + private readonly IFollowersDal _followersDal; + private readonly IHttpClientFactory _httpClientFactory; + private readonly ILogger _logger; + + #region Ctor + public MigrationService(ITwitterTweetsService twitterTweetsService, IActivityPubService activityPubService, ITwitterUserDal twitterUserDal, IFollowersDal followersDal, InstanceSettings instanceSettings, ITheFedInfoService theFedInfoService, IHttpClientFactory httpClientFactory, ILogger logger) + { + _twitterTweetsService = twitterTweetsService; + _activityPubService = activityPubService; + _twitterUserDal = twitterUserDal; + _followersDal = followersDal; + _instanceSettings = instanceSettings; + _theFedInfoService = theFedInfoService; + _httpClientFactory = httpClientFactory; + _logger = logger; + } + #endregion + + public string GetMigrationCode(string acct) + { + var hash = GetHashString(acct); + return $"[[BirdsiteLIVE-MigrationCode|{hash.Substring(0, 10)}]]"; + } + + public string GetDeletionCode(string acct) + { + var hash = GetHashString(acct); + return $"[[BirdsiteLIVE-DeletionCode|{hash.Substring(0, 10)}]]"; + } + + public bool ValidateTweet(string acct, string tweetId, MigrationTypeEnum type) + { + string code; + if (type == MigrationTypeEnum.Migration) + code = GetMigrationCode(acct); + else if (type == MigrationTypeEnum.Deletion) + code = GetDeletionCode(acct); + else + throw new NotImplementedException(); + + var castedTweetId = ExtractedTweetId(tweetId); + var tweet = _twitterTweetsService.GetTweet(castedTweetId); + + if (tweet == null) + throw new Exception("Tweet not found"); + + if (tweet.CreatorName.Trim().ToLowerInvariant() != acct.Trim().ToLowerInvariant()) + throw new Exception($"Tweet not published by @{acct}"); + + if (!tweet.MessageContent.Contains(code)) + { + var message = "Tweet don't have migration code"; + if (type == MigrationTypeEnum.Deletion) + message = "Tweet don't have deletion code"; + + throw new Exception(message); + } + + return true; + } + + private long ExtractedTweetId(string tweetId) + { + if (string.IsNullOrWhiteSpace(tweetId)) + throw new ArgumentException("No provided Tweet ID"); + + long castedId; + if (long.TryParse(tweetId, out castedId)) + return castedId; + + var urlPart = tweetId.Split('/').LastOrDefault(); + if (long.TryParse(urlPart, out castedId)) + return castedId; + + throw new ArgumentException("Unvalid Tweet ID"); + } + + public async Task ValidateFediverseAcctAsync(string fediverseAcct) + { + if (string.IsNullOrWhiteSpace(fediverseAcct)) + throw new ArgumentException("Please provide Fediverse account"); + + if (!fediverseAcct.Contains('@') || !fediverseAcct.StartsWith("@") || fediverseAcct.Trim('@').Split('@').Length != 2) + throw new ArgumentException("Please provide valid Fediverse handle"); + + var objectId = await _activityPubService.GetUserIdAsync(fediverseAcct); + var user = await _activityPubService.GetUser(objectId); + + var result = new ValidatedFediverseUser + { + FediverseAcct = fediverseAcct, + ObjectId = objectId, + User = user, + IsValid = user != null + }; + + return result; + } + + public async Task MigrateAccountAsync(ValidatedFediverseUser validatedUser, string acct) + { + // Apply moved to + var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct); + if (twitterAccount == null) + { + await _twitterUserDal.CreateTwitterUserAsync(acct, -1, validatedUser.ObjectId, validatedUser.FediverseAcct); + twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct); + } + + twitterAccount.MovedTo = validatedUser.User.id; + twitterAccount.MovedToAcct = validatedUser.FediverseAcct; + twitterAccount.LastSync = DateTime.UtcNow; + await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); + + // Notify Followers + var message = $@"

[BSL MIRROR SERVICE NOTIFICATION]
This bot has been disabled by its original owner.
It has been redirected to {validatedUser.FediverseAcct}.

"; + NotifyFollowers(acct, twitterAccount, message); + } + + private void NotifyFollowers(string acct, SyncTwitterUser twitterAccount, string message) + { + var t = Task.Run(async () => + { + var followers = await _followersDal.GetFollowersAsync(twitterAccount.Id); + foreach (var follower in followers) + { + try + { + var noteId = Guid.NewGuid().ToString(); + var actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, acct); + var noteUrl = UrlFactory.GetNoteUrl(_instanceSettings.Domain, acct, noteId); + + //var to = validatedUser.ObjectId; + var to = follower.ActorId; + var cc = new string[0]; + + var note = new Note + { + id = noteUrl, + + published = DateTime.UtcNow.ToString("s") + "Z", + url = noteUrl, + attributedTo = actorUrl, + + to = new[] { to }, + cc = cc, + + content = message, + tag = new Tag[]{ + new Tag() + { + type = "Mention", + href = follower.ActorId, + name = $"@{follower.Acct}@{follower.Host}" + } + }, + }; + + if (!string.IsNullOrWhiteSpace(follower.SharedInboxRoute)) + await _activityPubService.PostNewNoteActivity(note, acct, noteId, follower.Host, follower.SharedInboxRoute); + else + await _activityPubService.PostNewNoteActivity(note, acct, noteId, follower.Host, follower.InboxRoute); + } + catch (Exception e) + { + _logger.LogError(e, e.Message); + } + } + }); + } + + public async Task DeleteAccountAsync(string acct) + { + // Apply deleted state + var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct); + if (twitterAccount == null) + { + await _twitterUserDal.CreateTwitterUserAsync(acct, -1); + twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct); + } + + twitterAccount.Deleted = true; + twitterAccount.LastSync = DateTime.UtcNow; + await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); + + // Notify Followers + var message = $@"

[BSL MIRROR SERVICE NOTIFICATION]
This bot has been deleted by its original owner.

"; + NotifyFollowers(acct, twitterAccount, message); + + // Delete remote accounts + DeleteRemoteAccounts(acct); + } + + private void DeleteRemoteAccounts(string acct) + { + var t = Task.Run(async () => + { + var allUsers = await _followersDal.GetAllFollowersAsync(); + + var followersWtSharedInbox = allUsers + .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute)) + .GroupBy(x => x.Host) + .ToList(); + foreach (var followerGroup in followersWtSharedInbox) + { + var host = followerGroup.First().Host; + var sharedInbox = followerGroup.First().SharedInboxRoute; + + var t1 = Task.Run(async () => + { + try + { + await _activityPubService.DeleteUserAsync(acct, host, sharedInbox); + } + catch (Exception e) + { + _logger.LogError(e, e.Message); + } + }); + } + + var followerWtInbox = allUsers + .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute)) + .ToList(); + foreach (var followerGroup in followerWtInbox) + { + var host = followerGroup.Host; + var sharedInbox = followerGroup.InboxRoute; + + var t1 = Task.Run(async () => + { + try + { + await _activityPubService.DeleteUserAsync(acct, host, sharedInbox); + } + catch (Exception e) + { + _logger.LogError(e, e.Message); + } + }); + } + }); + } + + public async Task TriggerRemoteMigrationAsync(string id, string tweetIdStg, string handle) + { + var url = $"https://{{0}}/migration/move/{{1}}/{{2}}/{handle}"; + await ProcessRemoteMigrationAsync(id, tweetIdStg, url); + + } + + public async Task TriggerRemoteDeleteAsync(string id, string tweetIdStg) + { + var url = $"https://{{0}}/migration/delete/{{1}}/{{2}}"; + await ProcessRemoteMigrationAsync(id, tweetIdStg, url); + } + + private async Task ProcessRemoteMigrationAsync(string id, string tweetIdStg, string urlPattern) + { + try + { + var instances = await RetrieveCompatibleBslInstancesAsync(); + var tweetId = ExtractedTweetId(tweetIdStg); + + foreach (var instance in instances) + { + try + { + var host = instance.Host; + if(!UrlRegexes.Domain.IsMatch(host)) continue; + + var url = string.Format(urlPattern, host, id, tweetId); + + var client = _httpClientFactory.CreateClient(); + var result = await client.PostAsync(url, null); + result.EnsureSuccessStatusCode(); + } + catch (Exception e) + { + _logger.LogError(e, e.Message); + } + } + } + catch (Exception e) + { + _logger.LogError(e, e.Message); + } + } + + private async Task> RetrieveCompatibleBslInstancesAsync() + { + var instances = await _theFedInfoService.GetBslInstanceListAsync(); + var filteredInstances = instances + .Where(x => x.Version >= new Version(0, 21, 0)) + .Where(x => string.Compare(x.Host, + _instanceSettings.Domain, + StringComparison.InvariantCultureIgnoreCase) != 0) + .ToList(); + return filteredInstances; + } + + private byte[] GetHash(string inputString) + { + using (HashAlgorithm algorithm = SHA256.Create()) + return algorithm.ComputeHash(Encoding.UTF8.GetBytes(inputString)); + } + + private string GetHashString(string inputString) + { + StringBuilder sb = new StringBuilder(); + foreach (byte b in GetHash(inputString)) + sb.Append(b.ToString("X2")); + + return sb.ToString(); + } + } + + public class ValidatedFediverseUser + { + public string FediverseAcct { get; set; } + public string ObjectId { get; set; } + public Actor User { get; set; } + public bool IsValid { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/TheFedInfoService.cs b/src/BirdsiteLive.Domain/TheFedInfoService.cs new file mode 100644 index 0000000..e85d7fa --- /dev/null +++ b/src/BirdsiteLive.Domain/TheFedInfoService.cs @@ -0,0 +1,162 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json; + +namespace BirdsiteLive.Domain +{ + public interface ITheFedInfoService + { + Task> GetBslInstanceListAsync(); + } + + public class TheFedInfoService : ITheFedInfoService + { + private readonly IHttpClientFactory _httpClientFactory; + + #region Ctor + public TheFedInfoService(IHttpClientFactory httpClientFactory) + { + _httpClientFactory = httpClientFactory; + } + #endregion + + public async Task> GetBslInstanceListAsync() + { + var cancellationToken = CancellationToken.None; + + var result = await CallGraphQlAsync( + new Uri("https://the-federation.info/graphql"), + HttpMethod.Get, + "query ($platform: String!) { nodes(platform: $platform) { host, version } }", + new + { + platform = "birdsitelive", + }, + cancellationToken); + + var convertedResults = ConvertResults(result); + return convertedResults; + } + + private List ConvertResults(GraphQLResponse qlData) + { + var results = new List(); + + foreach (var instanceInfo in qlData.Data.Nodes) + { + try + { + var rawVersion = instanceInfo.Version.Split('+').First(); + if (string.IsNullOrWhiteSpace(rawVersion)) continue; + var version = Version.Parse(rawVersion); + if(version <= new Version(0,1,0)) continue; + + var instance = new BslInstanceInfo + { + Host = instanceInfo.Host, + Version = version + }; + results.Add(instance); + } + catch (Exception e) + { + Console.WriteLine(e); + } + } + + return results; + } + + private async Task> CallGraphQlAsync(Uri endpoint, HttpMethod method, string query, object variables, CancellationToken cancellationToken) + { + var content = new StringContent(SerializeGraphQlCall(query, variables), Encoding.UTF8, "application/json"); + var httpRequestMessage = new HttpRequestMessage + { + Method = method, + Content = content, + RequestUri = endpoint, + }; + //add authorization headers if necessary here + httpRequestMessage.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + var httpClient = _httpClientFactory.CreateClient(); + using (var response = await httpClient.SendAsync(httpRequestMessage, cancellationToken)) + { + //if (response.IsSuccessStatusCode) + if (response?.Content.Headers.ContentType?.MediaType == "application/json") + { + var responseString = await response.Content.ReadAsStringAsync(); //cancellationToken supported for .NET 5/6 + return DeserializeGraphQlCall(responseString); + } + else + { + throw new ApplicationException($"Unable to contact '{endpoint}': {response.StatusCode} - {response.ReasonPhrase}"); + } + } + } + + private string SerializeGraphQlCall(string query, object variables) + { + var sb = new StringBuilder(); + var textWriter = new StringWriter(sb); + var serializer = new JsonSerializer(); + serializer.Serialize(textWriter, new + { + query = query, + variables = variables, + }); + return sb.ToString(); + } + + private GraphQLResponse DeserializeGraphQlCall(string response) + { + var serializer = new JsonSerializer(); + var stringReader = new StringReader(response); + var jsonReader = new JsonTextReader(stringReader); + var result = serializer.Deserialize>(jsonReader); + return result; + } + + private class GraphQLResponse + { + public List Errors { get; set; } + public TResponse Data { get; set; } + } + + private class GraphQLError + { + public string Message { get; set; } + public List Locations { get; set; } + public List Path { get; set; } //either int or string + } + + private class GraphQLErrorLocation + { + public int Line { get; set; } + public int Column { get; set; } + } + + private class MyResponseData + { + public Node[] Nodes { get; set; } + } + + private class Node + { + public string Host { get; set; } + public string Version { get; set; } + } + } + + public class BslInstanceInfo + { + public string Host { get; set; } + public Version Version { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/Tools/SigValidationResultExtractor.cs b/src/BirdsiteLive.Domain/Tools/SigValidationResultExtractor.cs new file mode 100644 index 0000000..1d6890a --- /dev/null +++ b/src/BirdsiteLive.Domain/Tools/SigValidationResultExtractor.cs @@ -0,0 +1,22 @@ +using System.Linq; + +namespace BirdsiteLive.Domain.Tools +{ + public class SigValidationResultExtractor + { + public static string GetUserName(SignatureValidationResult result) + { + return result.User.preferredUsername.ToLowerInvariant().Trim(); + } + + public static string GetHost(SignatureValidationResult result) + { + return result.User.url.Replace("https://", string.Empty).Split('/').First(); + } + + public static string GetSharedInbox(SignatureValidationResult result) + { + return result.User?.endpoints?.sharedInbox; + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index c3817ed..3457b99 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -12,6 +12,7 @@ using BirdsiteLive.Common.Regexes; using BirdsiteLive.Common.Settings; using BirdsiteLive.Cryptography; using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; using BirdsiteLive.Domain.BusinessUseCases; using BirdsiteLive.Domain.Repository; using BirdsiteLive.Domain.Statistics; @@ -25,15 +26,17 @@ namespace BirdsiteLive.Domain { public interface IUserService { - Actor GetUser(TwitterUser twitterUser); + Actor GetUser(TwitterUser twitterUser, SyncTwitterUser dbTwitterUser); Task FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityFollow activity, string body); Task UndoFollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityUndoFollow activity, string body); Task SendRejectFollowAsync(ActivityFollow activity, string followerHost); + Task DeleteRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityDelete activity, string body); } public class UserService : IUserService { + private readonly IProcessDeleteUser _processDeleteUser; private readonly IProcessFollowUser _processFollowUser; private readonly IProcessUndoFollowUser _processUndoFollowUser; @@ -50,7 +53,7 @@ namespace BirdsiteLive.Domain private readonly IFollowersDal _followerDal; #region Ctor - public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler, ITwitterUserService twitterUserService, IModerationRepository moderationRepository, IFollowersDal followerDal) + public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler, ITwitterUserService twitterUserService, IModerationRepository moderationRepository, IFollowersDal followerDal, IProcessDeleteUser processDeleteUser) { _instanceSettings = instanceSettings; _cryptoService = cryptoService; @@ -62,10 +65,11 @@ namespace BirdsiteLive.Domain _twitterUserService = twitterUserService; _moderationRepository = moderationRepository; _followerDal = followerDal; + _processDeleteUser = processDeleteUser; } #endregion - public Actor GetUser(TwitterUser twitterUser) + public Actor GetUser(TwitterUser twitterUser, SyncTwitterUser dbTwitterUser) { var actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, twitterUser.Acct); var acct = twitterUser.Acct.ToLowerInvariant(); @@ -116,9 +120,10 @@ namespace BirdsiteLive.Domain preferredUsername = acct, name = twitterUser.Name, inbox = $"{actorUrl}/inbox", - summary = description, + summary = "[UNOFFICIAL MIRROR: This is a view of Twitter using ActivityPub]

" + description, url = actorUrl, manuallyApprovesFollowers = twitterUser.Protected, + discoverable = false, publicKey = new PublicKey() { id = $"{actorUrl}#main-key", @@ -135,11 +140,32 @@ namespace BirdsiteLive.Domain mediaType = "image/jpeg", url = twitterUser.ProfileBannerURL }, - attachment = attachments.ToArray(), + attachment = new [] + { + new UserAttachment + { + type = "PropertyValue", + name = "Official Account", + value = $"https://twitter.com/{acct}" + }, + new UserAttachment + { + type = "PropertyValue", + name = "Disclaimer", + value = "This is an automatically created and managed mirror profile from Twitter. While it reflects exactly the content of the original account, it doesn't provide support for interactions and replies. It is an equivalent view from other 3rd party Twitter client apps and uses the same technical means to provide it." + }, + new UserAttachment + { + type = "PropertyValue", + name = "Take control of this account", + value = $"MANAGE" + } + }, endpoints = new EndPoints { sharedInbox = $"https://{_instanceSettings.Domain}/inbox" - } + }, + movedTo = dbTwitterUser?.MovedTo }; if (twitterUser.Verified) @@ -172,10 +198,10 @@ namespace BirdsiteLive.Domain if (!sigValidation.SignatureIsValidated) return false; // Prepare data - var followerUserName = sigValidation.User.preferredUsername.ToLowerInvariant().Trim(); - var followerHost = sigValidation.User.url.Replace("https://", string.Empty).Split('/').First(); + var followerUserName = SigValidationResultExtractor.GetUserName(sigValidation); + var followerHost = SigValidationResultExtractor.GetHost(sigValidation); var followerInbox = sigValidation.User.inbox; - var followerSharedInbox = sigValidation.User?.endpoints?.sharedInbox; + var followerSharedInbox = SigValidationResultExtractor.GetSharedInbox(sigValidation); var twitterUser = activity.apObject.Split('/').Last().Replace("@", string.Empty).ToLowerInvariant().Trim(); // Make sure to only keep routes @@ -269,7 +295,7 @@ namespace BirdsiteLive.Domain return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; //TODO: revamp this for better error handling } - + private string OnlyKeepRoute(string inbox, string host) { if (string.IsNullOrWhiteSpace(inbox)) @@ -314,6 +340,22 @@ namespace BirdsiteLive.Domain return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; //TODO: revamp this for better error handling } + public async Task DeleteRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, + ActivityDelete activity, string body) + { + // Validate + var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body); + if (!sigValidation.SignatureIsValidated) return false; + + // Remove user and followings + var followerUserName = SigValidationResultExtractor.GetUserName(sigValidation); + var followerHost = SigValidationResultExtractor.GetHost(sigValidation); + + await _processDeleteUser.ExecuteAsync(followerUserName, followerHost); + + return true; + } + private async Task ValidateSignature(string actor, string rawSig, string method, string path, string queryString, Dictionary requestHeaders, string body) { //Check Date Validity diff --git a/src/BirdsiteLive.Moderation/Actions/RemoveFollowerAction.cs b/src/BirdsiteLive.Moderation/Actions/RemoveFollowerAction.cs index 8ab3132..4721154 100644 --- a/src/BirdsiteLive.Moderation/Actions/RemoveFollowerAction.cs +++ b/src/BirdsiteLive.Moderation/Actions/RemoveFollowerAction.cs @@ -1,12 +1,6 @@ -using System; -using System.Linq; -using System.Threading.Tasks; -using BirdsiteLive.ActivityPub; -using BirdsiteLive.ActivityPub.Converters; -using BirdsiteLive.Common.Settings; -using BirdsiteLive.DAL.Contracts; +using System.Threading.Tasks; using BirdsiteLive.DAL.Models; -using BirdsiteLive.Domain; +using BirdsiteLive.Domain.BusinessUseCases; namespace BirdsiteLive.Moderation.Actions { @@ -17,16 +11,14 @@ namespace BirdsiteLive.Moderation.Actions public class RemoveFollowerAction : IRemoveFollowerAction { - private readonly IFollowersDal _followersDal; - private readonly ITwitterUserDal _twitterUserDal; private readonly IRejectAllFollowingsAction _rejectAllFollowingsAction; + private readonly IProcessDeleteUser _processDeleteUser; #region Ctor - public RemoveFollowerAction(IFollowersDal followersDal, ITwitterUserDal twitterUserDal, IRejectAllFollowingsAction rejectAllFollowingsAction) + public RemoveFollowerAction(IRejectAllFollowingsAction rejectAllFollowingsAction, IProcessDeleteUser processDeleteUser) { - _followersDal = followersDal; - _twitterUserDal = twitterUserDal; _rejectAllFollowingsAction = rejectAllFollowingsAction; + _processDeleteUser = processDeleteUser; } #endregion @@ -36,16 +28,7 @@ namespace BirdsiteLive.Moderation.Actions await _rejectAllFollowingsAction.ProcessAsync(follower); // Remove twitter users if no more followers - var followings = follower.Followings; - foreach (var following in followings) - { - var followers = await _followersDal.GetFollowersAsync(following); - if (followers.Length == 1 && followers.First().Id == follower.Id) - await _twitterUserDal.DeleteTwitterUserAsync(following); - } - - // Remove follower from DB - await _followersDal.DeleteFollowerAsync(follower.Id); + await _processDeleteUser.ExecuteAsync(follower); } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Moderation/BirdsiteLive.Moderation.csproj b/src/BirdsiteLive.Moderation/BirdsiteLive.Moderation.csproj index b7bbeea..0dfa367 100644 --- a/src/BirdsiteLive.Moderation/BirdsiteLive.Moderation.csproj +++ b/src/BirdsiteLive.Moderation/BirdsiteLive.Moderation.csproj @@ -1,7 +1,7 @@ - netstandard2.0 + net6.0 diff --git a/src/BirdsiteLive.Moderation/Processors/TwitterAccountModerationProcessor.cs b/src/BirdsiteLive.Moderation/Processors/TwitterAccountModerationProcessor.cs index 91e3931..2f4d50e 100644 --- a/src/BirdsiteLive.Moderation/Processors/TwitterAccountModerationProcessor.cs +++ b/src/BirdsiteLive.Moderation/Processors/TwitterAccountModerationProcessor.cs @@ -29,7 +29,7 @@ namespace BirdsiteLive.Moderation.Processors { if (type == ModerationTypeEnum.None) return; - var twitterUsers = await _twitterUserDal.GetAllTwitterUsersAsync(); + var twitterUsers = await _twitterUserDal.GetAllTwitterUsersAsync(false); foreach (var user in twitterUsers) { diff --git a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj index 8601b19..add907f 100644 --- a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj +++ b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj @@ -1,7 +1,7 @@  - netstandard2.0 + net6.0 latest diff --git a/src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs index 3a36be3..739d50b 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs @@ -9,6 +9,7 @@ using BirdsiteLive.Moderation.Actions; using BirdsiteLive.Pipeline.Contracts; using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Twitter; +using BirdsiteLive.Twitter.Models; namespace BirdsiteLive.Pipeline.Processors { @@ -35,26 +36,61 @@ namespace BirdsiteLive.Pipeline.Processors foreach (var user in syncTwitterUsers) { - var userView = _twitterUserService.GetUser(user.Acct); - if (userView == null) - { - await AnalyseFailingUserAsync(user); - } - else if (!userView.Protected) - { - user.FetchingErrorCount = 0; - var userWtData = new UserWithDataToSync - { - User = user - }; - usersWtData.Add(userWtData); - } - } + TwitterUser userView = null; + try + { + userView = _twitterUserService.GetUser(user.Acct); + } + catch (UserNotFoundException) + { + await ProcessNotFoundUserAsync(user); + continue; + } + catch (UserHasBeenSuspendedException) + { + await ProcessNotFoundUserAsync(user); + continue; + } + catch (RateLimitExceededException) + { + await ProcessRateLimitExceededAsync(user); + continue; + } + catch (Exception) + { + // ignored + } + + if (userView == null || userView.Protected) + { + await ProcessFailingUserAsync(user); + continue; + } + + user.FetchingErrorCount = 0; + var userWtData = new UserWithDataToSync + { + User = user + }; + usersWtData.Add(userWtData); + } return usersWtData.ToArray(); } - private async Task AnalyseFailingUserAsync(SyncTwitterUser user) + private async Task ProcessRateLimitExceededAsync(SyncTwitterUser user) + { + var dbUser = await _twitterUserDal.GetTwitterUserAsync(user.Acct); + dbUser.LastSync = DateTime.UtcNow; + await _twitterUserDal.UpdateTwitterUserAsync(dbUser); + } + + private async Task ProcessNotFoundUserAsync(SyncTwitterUser user) + { + await _removeTwitterAccountAction.ProcessAsync(user); + } + + private async Task ProcessFailingUserAsync(SyncTwitterUser user) { var dbUser = await _twitterUserDal.GetTwitterUserAsync(user.Acct); dbUser.FetchingErrorCount++; @@ -68,9 +104,6 @@ namespace BirdsiteLive.Pipeline.Processors { await _twitterUserDal.UpdateTwitterUserAsync(dbUser); } - - // Purge - _twitterUserService.PurgeUser(user.Acct); } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs index 58d35d0..321fbf0 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs @@ -49,12 +49,12 @@ namespace BirdsiteLive.Pipeline.Processors { var tweetId = tweets.Last().Id; var now = DateTime.UtcNow; - await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now); + await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now, user.MovedTo, user.MovedToAcct, user.Deleted); } else { var now = DateTime.UtcNow; - await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now); + await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now, user.MovedTo, user.MovedToAcct, user.Deleted); } } diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs index 973b672..d9d0ffb 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs @@ -39,7 +39,7 @@ namespace BirdsiteLive.Pipeline.Processors try { var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync(); - var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber); + var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber, false); var userCount = users.Any() ? users.Length : 1; var splitNumber = (int) Math.Ceiling(userCount / 15d); diff --git a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs index 1437255..1f94871 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs @@ -3,6 +3,8 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Moderation.Actions; using BirdsiteLive.Pipeline.Contracts; using BirdsiteLive.Pipeline.Models; using Microsoft.Extensions.Logging; @@ -13,12 +15,14 @@ namespace BirdsiteLive.Pipeline.Processors { private readonly ITwitterUserDal _twitterUserDal; private readonly ILogger _logger; + private readonly IRemoveTwitterAccountAction _removeTwitterAccountAction; #region Ctor - public SaveProgressionProcessor(ITwitterUserDal twitterUserDal, ILogger logger) + public SaveProgressionProcessor(ITwitterUserDal twitterUserDal, ILogger logger, IRemoveTwitterAccountAction removeTwitterAccountAction) { _twitterUserDal = twitterUserDal; _logger = logger; + _removeTwitterAccountAction = removeTwitterAccountAction; } #endregion @@ -28,28 +32,23 @@ namespace BirdsiteLive.Pipeline.Processors { if (userWithTweetsToSync.Tweets.Length == 0) { - _logger.LogWarning("No tweets synchronized"); + _logger.LogInformation("No tweets synchronized"); + await UpdateUserSyncDateAsync(userWithTweetsToSync.User); return; } if(userWithTweetsToSync.Followers.Length == 0) { - _logger.LogWarning("No Followers found for {User}", userWithTweetsToSync.User.Acct); + _logger.LogInformation("No Followers found for {User}", userWithTweetsToSync.User.Acct); + await _removeTwitterAccountAction.ProcessAsync(userWithTweetsToSync.User); return; } var userId = userWithTweetsToSync.User.Id; var followingSyncStatuses = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).ToList(); - - if (followingSyncStatuses.Count == 0) - { - _logger.LogWarning("No Followers sync found for {User}, Id: {UserId}", userWithTweetsToSync.User.Acct, userId); - return; - } - var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max(); var minimumSync = followingSyncStatuses.Min(); var now = DateTime.UtcNow; - await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now); + await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now, userWithTweetsToSync.User.MovedTo, userWithTweetsToSync.User.MovedToAcct, userWithTweetsToSync.User.Deleted); } catch (Exception e) { @@ -57,5 +56,11 @@ namespace BirdsiteLive.Pipeline.Processors throw; } } + + private async Task UpdateUserSyncDateAsync(SyncTwitterUser user) + { + user.LastSync = DateTime.UtcNow; + await _twitterUserDal.UpdateTwitterUserAsync(user); + } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs index 65f9610..e210f39 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs @@ -5,9 +5,11 @@ using System.Net; using System.Threading; using System.Threading.Tasks; using System.Xml; +using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Domain; +using BirdsiteLive.Moderation.Actions; using BirdsiteLive.Pipeline.Contracts; using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Pipeline.Processors.SubTasks; @@ -23,14 +25,18 @@ namespace BirdsiteLive.Pipeline.Processors private readonly ISendTweetsToInboxTask _sendTweetsToInboxTask; private readonly ISendTweetsToSharedInboxTask _sendTweetsToSharedInbox; private readonly IFollowersDal _followersDal; + private readonly InstanceSettings _instanceSettings; private readonly ILogger _logger; + private readonly IRemoveFollowerAction _removeFollowerAction; #region Ctor - public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox, IFollowersDal followersDal, ILogger logger) + public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox, IFollowersDal followersDal, ILogger logger, InstanceSettings instanceSettings, IRemoveFollowerAction removeFollowerAction) { _sendTweetsToInboxTask = sendTweetsToInboxTask; _sendTweetsToSharedInbox = sendTweetsToSharedInbox; _logger = logger; + _instanceSettings = instanceSettings; + _removeFollowerAction = removeFollowerAction; _followersDal = followersDal; } #endregion @@ -107,7 +113,17 @@ namespace BirdsiteLive.Pipeline.Processors private async Task ProcessFailingUserAsync(Follower follower) { follower.PostingErrorCount++; - await _followersDal.UpdateFollowerAsync(follower); + + if (follower.PostingErrorCount > _instanceSettings.FailingFollowerCleanUpThreshold + && _instanceSettings.FailingFollowerCleanUpThreshold > 0 + || follower.PostingErrorCount > 2147483600) + { + await _removeFollowerAction.ProcessAsync(follower); + } + else + { + await _followersDal.UpdateFollowerAsync(follower); + } } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj b/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj index 438b4e1..45c4ce9 100644 --- a/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj +++ b/src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj @@ -1,7 +1,7 @@ - netstandard2.0 + net6.0 diff --git a/src/BirdsiteLive.Twitter/CachedTwitterUserService.cs b/src/BirdsiteLive.Twitter/CachedTwitterUserService.cs index ac20a62..c49104a 100644 --- a/src/BirdsiteLive.Twitter/CachedTwitterUserService.cs +++ b/src/BirdsiteLive.Twitter/CachedTwitterUserService.cs @@ -1,4 +1,5 @@ using System; +using BirdsiteLive.Common.Settings; using BirdsiteLive.Twitter.Models; using Microsoft.Extensions.Caching.Memory; @@ -13,11 +14,8 @@ namespace BirdsiteLive.Twitter { private readonly ITwitterUserService _twitterService; - private MemoryCache _userCache = new MemoryCache(new MemoryCacheOptions() - { - SizeLimit = 5000 - }); - private MemoryCacheEntryOptions _cacheEntryOptions = new MemoryCacheEntryOptions() + private readonly MemoryCache _userCache; + private readonly MemoryCacheEntryOptions _cacheEntryOptions = new MemoryCacheEntryOptions() .SetSize(1)//Size amount //Priority on removing when reaching size limit (memory pressure) .SetPriority(CacheItemPriority.High) @@ -27,9 +25,14 @@ namespace BirdsiteLive.Twitter .SetAbsoluteExpiration(TimeSpan.FromDays(7)); #region Ctor - public CachedTwitterUserService(ITwitterUserService twitterService) + public CachedTwitterUserService(ITwitterUserService twitterService, InstanceSettings settings) { _twitterService = twitterService; + + _userCache = new MemoryCache(new MemoryCacheOptions() + { + SizeLimit = settings.UserCacheCapacity + }); } #endregion @@ -44,6 +47,11 @@ namespace BirdsiteLive.Twitter return user; } + public bool IsUserApiRateLimited() + { + return _twitterService.IsUserApiRateLimited(); + } + public void PurgeUser(string username) { _userCache.Remove(username); diff --git a/src/BirdsiteLive.Twitter/Exceptions/RateLimitExceededException.cs b/src/BirdsiteLive.Twitter/Exceptions/RateLimitExceededException.cs new file mode 100644 index 0000000..93a093a --- /dev/null +++ b/src/BirdsiteLive.Twitter/Exceptions/RateLimitExceededException.cs @@ -0,0 +1,9 @@ +using System; + +namespace BirdsiteLive.Twitter +{ + public class RateLimitExceededException : Exception + { + + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/Exceptions/UserHasBeenSuspendedException.cs b/src/BirdsiteLive.Twitter/Exceptions/UserHasBeenSuspendedException.cs new file mode 100644 index 0000000..03bd835 --- /dev/null +++ b/src/BirdsiteLive.Twitter/Exceptions/UserHasBeenSuspendedException.cs @@ -0,0 +1,9 @@ +using System; + +namespace BirdsiteLive.Twitter +{ + public class UserHasBeenSuspendedException : Exception + { + + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/Exceptions/UserNotFoundException.cs b/src/BirdsiteLive.Twitter/Exceptions/UserNotFoundException.cs new file mode 100644 index 0000000..1dffc72 --- /dev/null +++ b/src/BirdsiteLive.Twitter/Exceptions/UserNotFoundException.cs @@ -0,0 +1,9 @@ +using System; + +namespace BirdsiteLive.Twitter +{ + public class UserNotFoundException : Exception + { + + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs index a44f6bb..5767a3f 100644 --- a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs +++ b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs @@ -40,7 +40,8 @@ namespace BirdsiteLive.Twitter.Extractors IsRetweet = tweet.IsRetweet || tweet.QuotedStatusId != null, RetweetUrl = ExtractRetweetUrl(tweet), IsSensitive = tweet.PossiblySensitive, - QuoteTweetUrl = tweet.QuotedStatusId != null ? "https://" + _instanceSettings.Domain + "/users/" + tweet.QuotedTweet.CreatedBy.ScreenName + "/statuses/" + tweet.QuotedStatusId : null + QuoteTweetUrl = tweet.QuotedStatusId != null ? "https://" + _instanceSettings.Domain + "/users/" + tweet.QuotedTweet.CreatedBy.ScreenName + "/statuses/" + tweet.QuotedStatusId : null, + CreatorName = tweet.CreatedBy.UserIdentifier.ScreenName }; return extractedTweet; diff --git a/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs b/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs index c9fa9b9..afd3088 100644 --- a/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs +++ b/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs @@ -17,5 +17,6 @@ namespace BirdsiteLive.Twitter.Models public string RetweetUrl { get; set; } public bool IsSensitive { get; set; } public string QuoteTweetUrl { get; set; } + public string CreatorName { get; set; } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs b/src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs index 668be76..8063985 100644 --- a/src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs +++ b/src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs @@ -13,6 +13,8 @@ namespace BirdsiteLive.Statistics.Domain void CalledTweetApi(); void CalledTimelineApi(); ApiStatistics GetStatistics(); + + int GetCurrentUserCalls(); } //Rate limits: https://developer.twitter.com/en/docs/twitter-api/v1/rate-limits @@ -60,7 +62,12 @@ namespace BirdsiteLive.Statistics.Domain foreach (var old in oldSnapshots) _snapshots.TryRemove(old, out var data); } - public void CalledUserApi() //GET users/show - 900/15mins + public int GetCurrentUserCalls() + { + return _userCalls; + } + + public void CalledUserApi() //GET users/show - 300/15mins { Interlocked.Increment(ref _userCalls); } diff --git a/src/BirdsiteLive.Twitter/TwitterUserService.cs b/src/BirdsiteLive.Twitter/TwitterUserService.cs index 166e466..d9e8add 100644 --- a/src/BirdsiteLive.Twitter/TwitterUserService.cs +++ b/src/BirdsiteLive.Twitter/TwitterUserService.cs @@ -6,6 +6,7 @@ using BirdsiteLive.Twitter.Models; using BirdsiteLive.Twitter.Tools; using Microsoft.Extensions.Logging; using Tweetinvi; +using Tweetinvi.Exceptions; using Tweetinvi.Models; namespace BirdsiteLive.Twitter @@ -13,6 +14,7 @@ namespace BirdsiteLive.Twitter public interface ITwitterUserService { TwitterUser GetUser(string username); + bool IsUserApiRateLimited(); } public class TwitterUserService : ITwitterUserService @@ -32,27 +34,46 @@ namespace BirdsiteLive.Twitter public TwitterUser GetUser(string username) { + //Check if API is saturated + if (IsUserApiRateLimited()) throw new RateLimitExceededException(); + + //Proceed to account retrieval _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized(); ExceptionHandler.SwallowWebExceptions = false; + RateLimit.RateLimitTrackerMode = RateLimitTrackerMode.TrackOnly; IUser user; try { user = User.GetUserFromScreenName(username); - _statisticsHandler.CalledUserApi(); - if (user == null) + } + catch (TwitterException e) + { + if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("User has been suspended".ToLowerInvariant()))) { - _logger.LogWarning("User {username} not found", username); - return null; + throw new UserHasBeenSuspendedException(); + } + else if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("User not found".ToLowerInvariant()))) + { + throw new UserNotFoundException(); + } + else if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("Rate limit exceeded".ToLowerInvariant()))) + { + throw new RateLimitExceededException(); + } + else + { + throw; } } catch (Exception e) { _logger.LogError(e, "Error retrieving user {Username}", username); - - // TODO keep track of error, see where to remove user if too much errors - - return null; + throw; + } + finally + { + _statisticsHandler.CalledUserApi(); } // Expand URLs @@ -74,5 +95,32 @@ namespace BirdsiteLive.Twitter Verified = user.Verified }; } + + public bool IsUserApiRateLimited() + { + // Retrieve limit from tooling + _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized(); + ExceptionHandler.SwallowWebExceptions = false; + RateLimit.RateLimitTrackerMode = RateLimitTrackerMode.TrackOnly; + + try + { + var queryRateLimits = RateLimit.GetQueryRateLimit("https://api.twitter.com/1.1/users/show.json?screen_name=mastodon"); + + if (queryRateLimits != null) + { + return queryRateLimits.Remaining <= 0; + } + } + catch (Exception e) + { + _logger.LogError(e, "Error retrieving rate limits"); + } + + // Fallback + var currentCalls = _statisticsHandler.GetCurrentUserCalls(); + var maxCalls = _statisticsHandler.GetStatistics().UserCallsMax; + return currentCalls >= maxCalls; + } } } \ No newline at end of file diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index 5935a02..a29d317 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -1,10 +1,10 @@  - netcoreapp3.1 + net6.0 d21486de-a812-47eb-a419-05682bb68856 Linux - 0.19.1 + 0.22.0 diff --git a/src/BirdsiteLive/Controllers/DebugingController.cs b/src/BirdsiteLive/Controllers/DebugingController.cs index 252486e..8f37f22 100644 --- a/src/BirdsiteLive/Controllers/DebugingController.cs +++ b/src/BirdsiteLive/Controllers/DebugingController.cs @@ -14,6 +14,7 @@ using Newtonsoft.Json; namespace BirdsiteLive.Controllers { + #if DEBUG public class DebugingController : Controller { private readonly InstanceSettings _instanceSettings; @@ -58,17 +59,22 @@ namespace BirdsiteLive.Controllers [HttpPost] public async Task PostNote() { - var username = "gra"; + var username = "twitter"; var actor = $"https://{_instanceSettings.Domain}/users/{username}"; - var targetHost = "mastodon.technology"; - var target = $"{targetHost}/users/testtest"; - var inbox = $"/users/testtest/inbox"; + var targetHost = "ioc.exchange"; + var target = $"https://{targetHost}/users/test"; + //var inbox = $"/users/testtest/inbox"; + var inbox = $"/inbox"; var noteGuid = Guid.NewGuid(); var noteId = $"https://{_instanceSettings.Domain}/users/{username}/statuses/{noteGuid}"; var noteUrl = $"https://{_instanceSettings.Domain}/@{username}/{noteGuid}"; - + var to = $"{actor}/followers"; + to = target; + + var cc = new[] { "https://www.w3.org/ns/activitystreams#Public" }; + cc = new string[0]; var now = DateTime.UtcNow; var nowString = now.ToString("s") + "Z"; @@ -80,12 +86,12 @@ namespace BirdsiteLive.Controllers type = "Create", actor = actor, published = nowString, - to = new []{ to }, - //cc = new [] { "https://www.w3.org/ns/activitystreams#Public" }, + to = new[] { to }, + cc = cc, apObject = new Note() { id = noteId, - summary = null, + summary = null, inReplyTo = null, published = nowString, url = noteUrl, @@ -93,7 +99,8 @@ namespace BirdsiteLive.Controllers // Unlisted to = new[] { to }, - cc = new [] { "https://www.w3.org/ns/activitystreams#Public" }, + cc = cc, + //cc = new[] { "https://www.w3.org/ns/activitystreams#Public" }, //// Public //to = new[] { "https://www.w3.org/ns/activitystreams#Public" }, @@ -101,8 +108,16 @@ namespace BirdsiteLive.Controllers sensitive = false, content = "

TEST PUBLIC

", + //content = "

@test test

", attachment = new Attachment[0], - tag = new Tag[0] + tag = new Tag[]{ + new Tag() + { + type = "Mention", + href = target, + name = "@test@ioc.exchange" + } + }, } }; @@ -124,7 +139,19 @@ namespace BirdsiteLive.Controllers await _userService.SendRejectFollowAsync(activityFollow, "mastodon.technology"); return View("Index"); } + + [HttpPost] + public async Task PostDeleteUser() + { + var userName = "twitter"; + var host = "ioc.exchange"; + var inbox = "/inbox"; + + await _activityPubService.DeleteUserAsync(userName, host, inbox); + return View("Index"); + } } + #endif public static class HtmlHelperExtensions { diff --git a/src/BirdsiteLive/Controllers/InboxController.cs b/src/BirdsiteLive/Controllers/InboxController.cs index db055cf..f92a0a6 100644 --- a/src/BirdsiteLive/Controllers/InboxController.cs +++ b/src/BirdsiteLive/Controllers/InboxController.cs @@ -3,6 +3,10 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading.Tasks; +using BirdsiteLive.ActivityPub; +using BirdsiteLive.ActivityPub.Models; +using BirdsiteLive.Domain; +using BirdsiteLive.Tools; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; @@ -13,11 +17,13 @@ namespace BirdsiteLive.Controllers public class InboxController : ControllerBase { private readonly ILogger _logger; + private readonly IUserService _userService; #region Ctor - public InboxController(ILogger logger) + public InboxController(ILogger logger, IUserService userService) { _logger = logger; + _userService = userService; } #endregion @@ -25,15 +31,32 @@ namespace BirdsiteLive.Controllers [HttpPost] public async Task Inbox() { - var r = Request; - using (var reader = new StreamReader(Request.Body)) + try { - var body = await reader.ReadToEndAsync(); + var r = Request; + using (var reader = new StreamReader(Request.Body)) + { + var body = await reader.ReadToEndAsync(); - _logger.LogTrace("Inbox: {Body}", body); - //System.IO.File.WriteAllText($@"C:\apdebug\inbox\{Guid.NewGuid()}.json", body); + _logger.LogTrace("Inbox: {Body}", body); + //System.IO.File.WriteAllText($@"C:\apdebug\inbox\{Guid.NewGuid()}.json", body); + var activity = ApDeserializer.ProcessActivity(body); + var signature = r.Headers["Signature"].First(); + + switch (activity?.type) + { + case "Delete": + { + var succeeded = await _userService.DeleteRequestedAsync(signature, r.Method, r.Path, + r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), activity as ActivityDelete, body); + if (succeeded) return Accepted(); + else return Unauthorized(); + } + } + } } + catch (FollowerIsGoneException) { } //TODO: check if user in DB return Accepted(); } diff --git a/src/BirdsiteLive/Controllers/MigrationController.cs b/src/BirdsiteLive/Controllers/MigrationController.cs new file mode 100644 index 0000000..f2cde09 --- /dev/null +++ b/src/BirdsiteLive/Controllers/MigrationController.cs @@ -0,0 +1,227 @@ +using System; +using Microsoft.AspNetCore.Mvc; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; +using Npgsql.TypeHandlers; +using BirdsiteLive.Domain; +using BirdsiteLive.Domain.Enum; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.Models; +using System.Reflection.Metadata; + +namespace BirdsiteLive.Controllers +{ + public class MigrationController : Controller + { + private readonly MigrationService _migrationService; + private readonly ITwitterUserDal _twitterUserDal; + + #region Ctor + public MigrationController(MigrationService migrationService, ITwitterUserDal twitterUserDal) + { + _migrationService = migrationService; + _twitterUserDal = twitterUserDal; + } + #endregion + + [HttpGet] + [Route("/migration/move/{id}")] + public IActionResult IndexMove(string id) + { + var migrationCode = _migrationService.GetMigrationCode(id); + var data = new MigrationData() + { + Acct = id, + MigrationCode = migrationCode + }; + + return View("Index", data); + } + + [HttpGet] + [Route("/migration/delete/{id}")] + public IActionResult IndexDelete(string id) + { + var migrationCode = _migrationService.GetDeletionCode(id); + var data = new MigrationData() + { + Acct = id, + MigrationCode = migrationCode + }; + + return View("Delete", data); + } + + [HttpPost] + [Route("/migration/move/{id}")] + public async Task MigrateMove(string id, string tweetid, string handle) + { + var migrationCode = _migrationService.GetMigrationCode(id); + var data = new MigrationData() + { + Acct = id, + MigrationCode = migrationCode, + + IsAcctProvided = !string.IsNullOrWhiteSpace(handle), + IsTweetProvided = !string.IsNullOrWhiteSpace(tweetid), + + TweetId = tweetid, + FediverseAccount = handle + }; + ValidatedFediverseUser fediverseUserValidation = null; + + //Verify can be migrated + var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id); + if (twitterAccount != null && twitterAccount.Deleted) + { + data.ErrorMessage = "This account has been deleted, it can't be migrated"; + return View("Index", data); + } + if (twitterAccount != null && + (!string.IsNullOrWhiteSpace(twitterAccount.MovedTo) + || !string.IsNullOrWhiteSpace(twitterAccount.MovedToAcct))) + { + data.ErrorMessage = "This account has been moved already, it can't be migrated again"; + return View("Index", data); + } + + // Start migration + try + { + fediverseUserValidation = await _migrationService.ValidateFediverseAcctAsync(handle); + var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Migration); + + data.IsAcctValid = fediverseUserValidation.IsValid; + data.IsTweetValid = isTweetValid; + } + catch (Exception e) + { + data.ErrorMessage = e.Message; + } + + if (data.IsAcctValid && data.IsTweetValid && fediverseUserValidation != null) + { + try + { + await _migrationService.MigrateAccountAsync(fediverseUserValidation, id); + _migrationService.TriggerRemoteMigrationAsync(id, tweetid, handle); + data.MigrationSuccess = true; + } + catch (Exception e) + { + Console.WriteLine(e); + data.ErrorMessage = e.Message; + } + } + + return View("Index", data); + } + + [HttpPost] + [Route("/migration/delete/{id}")] + public async Task MigrateDelete(string id, string tweetid) + { + var deletionCode = _migrationService.GetDeletionCode(id); + + var data = new MigrationData() + { + Acct = id, + MigrationCode = deletionCode, + + IsTweetProvided = !string.IsNullOrWhiteSpace(tweetid), + + TweetId = tweetid + }; + + //Verify can be deleted + var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id); + if (twitterAccount != null && twitterAccount.Deleted) + { + data.ErrorMessage = "This account has been deleted, it can't be deleted again"; + return View("Delete", data); + } + + // Start deletion + try + { + var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Deletion); + data.IsTweetValid = isTweetValid; + } + catch (Exception e) + { + data.ErrorMessage = e.Message; + } + + if (data.IsTweetValid) + { + try + { + await _migrationService.DeleteAccountAsync(id); + _migrationService.TriggerRemoteDeleteAsync(id, tweetid); + data.MigrationSuccess = true; + } + catch (Exception e) + { + Console.WriteLine(e); + data.ErrorMessage = e.Message; + } + } + + return View("Delete", data); + } + + [HttpPost] + [Route("/migration/move/{id}/{tweetid}/{handle}")] + public async Task RemoteMigrateMove(string id, string tweetid, string handle) + { + //Check inputs + if (string.IsNullOrWhiteSpace(id) || string.IsNullOrWhiteSpace(tweetid) || + string.IsNullOrWhiteSpace(handle)) + return StatusCode(422); + + //Verify can be migrated + var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id); + if (twitterAccount != null && (twitterAccount.Deleted + || !string.IsNullOrWhiteSpace(twitterAccount.MovedTo) + || !string.IsNullOrWhiteSpace(twitterAccount.MovedToAcct))) + return Ok(); + + // Start migration + var fediverseUserValidation = await _migrationService.ValidateFediverseAcctAsync(handle); + var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Migration); + + if (fediverseUserValidation.IsValid && isTweetValid) + { + await _migrationService.MigrateAccountAsync(fediverseUserValidation, id); + return Ok(); + } + + return StatusCode(400); + } + + [HttpPost] + [Route("/migration/delete/{id}/{tweetid}")] + public async Task RemoteMigrateDelete(string id, string tweetid) + { + //Check inputs + if (string.IsNullOrWhiteSpace(id) || string.IsNullOrWhiteSpace(tweetid)) + return StatusCode(422); + + //Verify can be deleted + var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id); + if (twitterAccount != null && twitterAccount.Deleted) return Ok(); + + // Start deletion + var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Deletion); + + if (isTweetValid) + { + await _migrationService.DeleteAccountAsync(id); + return Ok(); + } + + return StatusCode(400); + } + } +} diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index 1a42eae..edc3478 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -3,7 +3,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Mime; -using System.Runtime.InteropServices.WindowsRuntime; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; @@ -11,8 +10,11 @@ using BirdsiteLive.ActivityPub; using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.Common.Regexes; using BirdsiteLive.Common.Settings; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; using BirdsiteLive.Domain; using BirdsiteLive.Models; +using BirdsiteLive.Tools; using BirdsiteLive.Twitter; using BirdsiteLive.Twitter.Models; using Microsoft.AspNetCore.Http; @@ -27,6 +29,7 @@ namespace BirdsiteLive.Controllers { private readonly ITwitterUserService _twitterUserService; private readonly ITwitterTweetsService _twitterTweetService; + private readonly ITwitterUserDal _twitterUserDal; private readonly IUserService _userService; private readonly IStatusService _statusService; private readonly InstanceSettings _instanceSettings; @@ -34,7 +37,7 @@ namespace BirdsiteLive.Controllers private readonly ILogger _logger; #region Ctor - public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ITwitterTweetsService twitterTweetService, IActivityPubService activityPubService, ILogger logger) + public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ITwitterTweetsService twitterTweetService, IActivityPubService activityPubService, ILogger logger, ITwitterUserDal twitterUserDal) { _twitterUserService = twitterUserService; _userService = userService; @@ -43,6 +46,7 @@ namespace BirdsiteLive.Controllers _twitterTweetService = twitterTweetService; _activityPubService = activityPubService; _logger = logger; + _twitterUserDal = twitterUserDal; } #endregion @@ -57,21 +61,52 @@ namespace BirdsiteLive.Controllers } return View("UserNotFound"); } - + [Route("/@{id}")] [Route("/users/{id}")] - [Route("/users/{id}/remote_follow")] - public IActionResult Index(string id) + public async Task Index(string id) { _logger.LogTrace("User Index: {Id}", id); id = id.Trim(new[] { ' ', '@' }).ToLowerInvariant(); + TwitterUser user = null; + var isSaturated = false; + var notFound = false; + // Ensure valid username // https://help.twitter.com/en/managing-your-account/twitter-username-rules - TwitterUser user = null; if (!string.IsNullOrWhiteSpace(id) && UserRegexes.TwitterAccount.IsMatch(id) && id.Length <= 15) - user = _twitterUserService.GetUser(id); + { + try + { + user = _twitterUserService.GetUser(id); + } + catch (UserNotFoundException) + { + notFound = true; + } + catch (UserHasBeenSuspendedException) + { + notFound = true; + } + catch (RateLimitExceededException) + { + isSaturated = true; + } + catch (Exception e) + { + _logger.LogError(e, "Exception getting {Id}", id); + throw; + } + } + else + { + notFound = true; + } + + //var isSaturated = _twitterUserService.IsUserApiRateLimited(); + var dbUser = await _twitterUserDal.GetTwitterUserAsync(id); var acceptHeaders = Request.Headers["Accept"]; if (acceptHeaders.Any()) @@ -79,8 +114,10 @@ namespace BirdsiteLive.Controllers var r = acceptHeaders.First(); if (r.Contains("application/activity+json")) { - if (user == null) return NotFound(); - var apUser = _userService.GetUser(user); + if (isSaturated) return new ObjectResult("Too Many Requests") { StatusCode = 429 }; + if (notFound) return NotFound(); + if (dbUser != null && dbUser.Deleted) return new ObjectResult("Gone") { StatusCode = 410 }; + var apUser = _userService.GetUser(user, dbUser); var jsonApUser = JsonConvert.SerializeObject(apUser, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore @@ -89,7 +126,8 @@ namespace BirdsiteLive.Controllers } } - if (user == null) return View("UserNotFound"); + if (isSaturated) return View("ApiSaturated"); + if (notFound) return View("UserNotFound"); var displayableUser = new DisplayTwitterUser { @@ -99,11 +137,21 @@ namespace BirdsiteLive.Controllers Url = user.Url, ProfileImageUrl = user.ProfileImageUrl, Protected = user.Protected, + + InstanceHandle = $"@{user.Acct.ToLowerInvariant()}@{_instanceSettings.Domain}", - InstanceHandle = $"@{user.Acct.ToLowerInvariant()}@{_instanceSettings.Domain}" + MovedTo = dbUser?.MovedTo, + MovedToAcct = dbUser?.MovedToAcct, + Deleted = dbUser?.Deleted ?? false, }; return View(displayableUser); } + + [Route("/users/{id}/remote_follow")] + public async Task IndexRemoteFollow(string id) + { + return Redirect($"/users/{id}"); + } [Route("/@{id}/{statusId}")] [Route("/users/{id}/statuses/{statusId}")] @@ -150,41 +198,69 @@ namespace BirdsiteLive.Controllers [HttpPost] public async Task Inbox() { - var r = Request; - using (var reader = new StreamReader(Request.Body)) + try { - var body = await reader.ReadToEndAsync(); - - _logger.LogTrace("User Inbox: {Body}", body); - //System.IO.File.WriteAllText($@"C:\apdebug\{Guid.NewGuid()}.json", body); - - var activity = ApDeserializer.ProcessActivity(body); - // Do something - var signature = r.Headers["Signature"].First(); - - switch (activity?.type) + var r = Request; + using (var reader = new StreamReader(Request.Body)) { - case "Follow": + var body = await reader.ReadToEndAsync(); + + _logger.LogTrace("User Inbox: {Body}", body); + //System.IO.File.WriteAllText($@"C:\apdebug\{Guid.NewGuid()}.json", body); + + var activity = ApDeserializer.ProcessActivity(body); + var signature = r.Headers["Signature"].First(); + + switch (activity?.type) + { + case "Follow": { var succeeded = await _userService.FollowRequestedAsync(signature, r.Method, r.Path, - r.QueryString.ToString(), RequestHeaders(r.Headers), activity as ActivityFollow, body); + r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), + activity as ActivityFollow, body); + if (succeeded) return Accepted(); + else return Unauthorized(); + } + case "Undo": + if (activity is ActivityUndoFollow) + { + var succeeded = await _userService.UndoFollowRequestedAsync(signature, r.Method, r.Path, + r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), + activity as ActivityUndoFollow, body); + if (succeeded) return Accepted(); + else return Unauthorized(); + } - if (succeeded) return Accepted(); - else return Unauthorized(); - } - case "Undo": - if (activity is ActivityUndoFollow) + return Accepted(); + case "Delete": { - var succeeded = await _userService.UndoFollowRequestedAsync(signature, r.Method, r.Path, - r.QueryString.ToString(), RequestHeaders(r.Headers), activity as ActivityUndoFollow, body); + var succeeded = await _userService.DeleteRequestedAsync(signature, r.Method, r.Path, + r.QueryString.ToString(), HeaderHandler.RequestHeaders(r.Headers), + activity as ActivityDelete, body); if (succeeded) return Accepted(); else return Unauthorized(); } - return Accepted(); - default: - return Accepted(); + default: + return Accepted(); + } } } + catch (FollowerIsGoneException) //TODO: check if user in DB + { + return Accepted(); + } + catch (UserNotFoundException) + { + return NotFound(); + } + catch (UserHasBeenSuspendedException) + { + return NotFound(); + } + catch (RateLimitExceededException) + { + return new ObjectResult("Too Many Requests") { StatusCode = 429 }; + } } [Route("/users/{id}/followers")] @@ -202,11 +278,6 @@ namespace BirdsiteLive.Controllers return Content(jsonApUser, "application/activity+json; charset=utf-8"); } - private Dictionary RequestHeaders(IHeaderDictionary header) - { - return header.ToDictionary, string, string>(h => h.Key.ToLowerInvariant(), h => h.Value); - } - [Route("/users/{actor}/remote_follow")] [HttpPost] public async Task RemoteFollow(string actor) diff --git a/src/BirdsiteLive/Controllers/WellKnownController.cs b/src/BirdsiteLive/Controllers/WellKnownController.cs index 5b80abb..f11939f 100644 --- a/src/BirdsiteLive/Controllers/WellKnownController.cs +++ b/src/BirdsiteLive/Controllers/WellKnownController.cs @@ -12,6 +12,7 @@ using BirdsiteLive.Models; using BirdsiteLive.Models.WellKnownModels; using BirdsiteLive.Twitter; using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace BirdsiteLive.Controllers @@ -23,13 +24,15 @@ namespace BirdsiteLive.Controllers private readonly ITwitterUserService _twitterUserService; private readonly ITwitterUserDal _twitterUserDal; private readonly InstanceSettings _settings; - + private readonly ILogger _logger; + #region Ctor - public WellKnownController(InstanceSettings settings, ITwitterUserService twitterUserService, ITwitterUserDal twitterUserDal, IModerationRepository moderationRepository) + public WellKnownController(InstanceSettings settings, ITwitterUserService twitterUserService, ITwitterUserDal twitterUserDal, IModerationRepository moderationRepository, ILogger logger) { _twitterUserService = twitterUserService; _twitterUserDal = twitterUserDal; _moderationRepository = moderationRepository; + _logger = logger; _settings = settings; } #endregion @@ -140,30 +143,54 @@ namespace BirdsiteLive.Controllers [Route("/.well-known/webfinger")] public IActionResult Webfinger(string resource = null) { - var acct = resource.Split("acct:")[1].Trim(); + if (string.IsNullOrWhiteSpace(resource)) + return BadRequest(); string name = null; string domain = null; - var splitAcct = acct.Split('@', StringSplitOptions.RemoveEmptyEntries); + if (resource.StartsWith("acct:")) + { + var acct = resource.Split("acct:")[1].Trim(); + var splitAcct = acct.Split('@', StringSplitOptions.RemoveEmptyEntries); - var atCount = acct.Count(x => x == '@'); - if (atCount == 1 && acct.StartsWith('@')) - { - name = splitAcct[1]; + var atCount = acct.Count(x => x == '@'); + if (atCount == 1 && acct.StartsWith('@')) + { + name = splitAcct[1]; + } + else if (atCount == 1 || atCount == 2) + { + name = splitAcct[0]; + domain = splitAcct[1]; + } + else + { + return BadRequest(); + } } - else if (atCount == 1 || atCount == 2) + else if (resource.StartsWith("https://")) { - name = splitAcct[0]; - domain = splitAcct[1]; + try + { + name = resource.Split('/').Last().Trim(); + domain = resource.Split("https://", StringSplitOptions.RemoveEmptyEntries)[0].Split('/')[0].Trim(); + } + catch (Exception e) + { + _logger.LogError(e, "Error parsing {Resource}", resource); + throw new NotImplementedException(); + } } else { - return BadRequest(); + _logger.LogError("Error parsing {Resource}", resource); + throw new NotImplementedException(); } // Ensure lowercase name = name.ToLowerInvariant(); + domain = domain?.ToLowerInvariant(); // Ensure valid username // https://help.twitter.com/en/managing-your-account/twitter-username-rules @@ -173,9 +200,27 @@ namespace BirdsiteLive.Controllers if (!string.IsNullOrWhiteSpace(domain) && domain != _settings.Domain) return NotFound(); - var user = _twitterUserService.GetUser(name); - if (user == null) + try + { + _twitterUserService.GetUser(name); + } + catch (UserNotFoundException) + { return NotFound(); + } + catch (UserHasBeenSuspendedException) + { + return NotFound(); + } + catch (RateLimitExceededException) + { + return new ObjectResult("Too Many Requests") { StatusCode = 429 }; + } + catch (Exception e) + { + _logger.LogError(e, "Exception getting {Name}", name); + throw; + } var actorUrl = UrlFactory.GetActorUrl(_settings.Domain, name); diff --git a/src/BirdsiteLive/Models/DisplayTwitterUser.cs b/src/BirdsiteLive/Models/DisplayTwitterUser.cs index 3a93875..0b17174 100644 --- a/src/BirdsiteLive/Models/DisplayTwitterUser.cs +++ b/src/BirdsiteLive/Models/DisplayTwitterUser.cs @@ -10,5 +10,9 @@ public bool Protected { get; set; } public string InstanceHandle { get; set; } + + public string MovedTo { get; set; } + public string MovedToAcct { get; set; } + public bool Deleted { get; set; } } } \ No newline at end of file diff --git a/src/BirdsiteLive/Models/MigrationData.cs b/src/BirdsiteLive/Models/MigrationData.cs new file mode 100644 index 0000000..77185f9 --- /dev/null +++ b/src/BirdsiteLive/Models/MigrationData.cs @@ -0,0 +1,21 @@ +namespace BirdsiteLive.Models +{ + public class MigrationData + { + public string Acct { get; set; } + + public string FediverseAccount { get; set; } + public string TweetId { get; set; } + + public string MigrationCode { get; set; } + + public bool IsTweetProvided { get; set; } + public bool IsAcctProvided { get; set; } + + public bool IsTweetValid { get; set; } + public bool IsAcctValid { get; set; } + + public string ErrorMessage { get; set; } + public bool MigrationSuccess { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Tools/HeaderHandler.cs b/src/BirdsiteLive/Tools/HeaderHandler.cs new file mode 100644 index 0000000..74ecf29 --- /dev/null +++ b/src/BirdsiteLive/Tools/HeaderHandler.cs @@ -0,0 +1,15 @@ +using System.Collections.Generic; +using System.Linq; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Primitives; + +namespace BirdsiteLive.Tools +{ + public class HeaderHandler + { + public static Dictionary RequestHeaders(IHeaderDictionary header) + { + return header.ToDictionary, string, string>(h => h.Key.ToLowerInvariant(), h => h.Value); + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Views/Debuging/Index.cshtml b/src/BirdsiteLive/Views/Debuging/Index.cshtml index 5bcde75..e343cf2 100644 --- a/src/BirdsiteLive/Views/Debuging/Index.cshtml +++ b/src/BirdsiteLive/Views/Debuging/Index.cshtml @@ -23,4 +23,10 @@ + + +
+ + +
\ No newline at end of file diff --git a/src/BirdsiteLive/Views/Migration/Delete.cshtml b/src/BirdsiteLive/Views/Migration/Delete.cshtml new file mode 100644 index 0000000..568b08a --- /dev/null +++ b/src/BirdsiteLive/Views/Migration/Delete.cshtml @@ -0,0 +1,51 @@ +@model MigrationData +@{ + ViewData["Title"] = "Migration"; +} + +
+ @if (!string.IsNullOrWhiteSpace(ViewData.Model.ErrorMessage)) + { + + } + + @if (ViewData.Model.MigrationSuccess) + { + + } + +

Delete @@@ViewData.Model.Acct mirror

+ + @if (!ViewData.Model.IsTweetProvided) + { +

What is needed?

+ +

You'll need access to the Twitter account to provide proof of ownership.

+ +

What will deletion do?

+ +

+ Deletion will remove all followers, delete the account and will be blacklisted so that it can't be recreated.
+

+ } + +

Start the deletion!

+ +

Please copy and post this string in a public Tweet (the string must be untampered, but you can write anything you want before or after it):

+ + +
+ +

Provide deletion information:

+
+
+ + +
+ +
+
\ No newline at end of file diff --git a/src/BirdsiteLive/Views/Migration/Index.cshtml b/src/BirdsiteLive/Views/Migration/Index.cshtml new file mode 100644 index 0000000..2d1db53 --- /dev/null +++ b/src/BirdsiteLive/Views/Migration/Index.cshtml @@ -0,0 +1,66 @@ +@model MigrationData +@{ + ViewData["Title"] = "Migration"; +} + +
+ @if (!string.IsNullOrWhiteSpace(ViewData.Model.ErrorMessage)) + { + + } + + @if (ViewData.Model.MigrationSuccess) + { + + } + +

Migrate @@@ViewData.Model.Acct mirror to my Fediverse account

+ + @if (!ViewData.Model.IsAcctProvided && !ViewData.Model.IsTweetProvided) + { +

What is needed?

+ +

You'll need a Fediverse account and access to the Twitter account to provide proof of ownership.

+ +

What will migration do?

+ +

+ Migration will notify followers of the migration of the mirror account to your fediverse account and will be disabled after that.
+

+ } + +

Start the migration!

+ +

Please copy and post this string in a public Tweet (the string must be untampered, but you can write anything you want before or after it):

+ + +
+ +

Provide migration information:

+
+ @*
+ + + We'll never share your email with anyone else. +
*@ +
+ + +
+
+ + +
+ +
+
+
+
+
+ I don't have a fediverse account and I'd like to delete this mirror. +
+
\ No newline at end of file diff --git a/src/BirdsiteLive/Views/Users/ApiSaturated.cshtml b/src/BirdsiteLive/Views/Users/ApiSaturated.cshtml new file mode 100644 index 0000000..1f807aa --- /dev/null +++ b/src/BirdsiteLive/Views/Users/ApiSaturated.cshtml @@ -0,0 +1,13 @@ +@using BirdsiteLive.Controllers; +@{ + ViewData["Title"] = "Api Saturated"; +} + +
+

429 Too Many Requests

+

+
+ The API is saturated.
+ Please consider using another instance. +

+
\ No newline at end of file diff --git a/src/BirdsiteLive/Views/Users/Index.cshtml b/src/BirdsiteLive/Views/Users/Index.cshtml index f3812cc..d984892 100644 --- a/src/BirdsiteLive/Views/Users/Index.cshtml +++ b/src/BirdsiteLive/Views/Users/Index.cshtml @@ -31,7 +31,20 @@

- @if (ViewData.Model.Protected) + @if (ViewData.Model.Deleted) + { + + } + else if (!string.IsNullOrEmpty(ViewData.Model.MovedTo)) + { + + } + else if (ViewData.Model.Protected) { } + +
+ I'm the owner of this account and I would like to take control of this mirror. +
\ No newline at end of file diff --git a/src/BirdsiteLive/appsettings.json b/src/BirdsiteLive/appsettings.json index 416493c..d734200 100644 --- a/src/BirdsiteLive/appsettings.json +++ b/src/BirdsiteLive/appsettings.json @@ -30,7 +30,9 @@ "DiscloseInstanceRestrictions": false, "SensitiveTwitterAccounts": null, "FailingTwitterUserCleanUpThreshold": 700, - "MaxStatusFetchAge": 0 + "MaxStatusFetchAge": 0, + "FailingFollowerCleanUpThreshold": 30000, + "UserCacheCapacity": 10000 }, "Db": { "Type": "postgres", diff --git a/src/BirdsiteLive/wwwroot/css/birdsite.css b/src/BirdsiteLive/wwwroot/css/birdsite.css index 5b6023c..159a50a 100644 --- a/src/BirdsiteLive/wwwroot/css/birdsite.css +++ b/src/BirdsiteLive/wwwroot/css/birdsite.css @@ -71,3 +71,18 @@ margin-left: 60px; /*font-weight: bold;*/ } + +.user-owner { + font-size: .8em; + padding-top: 20px; +} + +/** Migration **/ + +.migration__title { + font-size: 1.8em; +} + +.migration__subtitle { + font-size: 1.4em; +} \ No newline at end of file diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/BirdsiteLive.DAL.Postgres.csproj b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/BirdsiteLive.DAL.Postgres.csproj index 690c878..0baeb19 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/BirdsiteLive.DAL.Postgres.csproj +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/BirdsiteLive.DAL.Postgres.csproj @@ -1,7 +1,7 @@  - netstandard2.0 + net6.0 diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs index 2e3acea..55b38f6 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs @@ -23,7 +23,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public class DbInitializerPostgresDal : PostgresBase, IDbInitializerDal { private readonly PostgresTools _tools; - private readonly Version _currentVersion = new Version(2, 3); + private readonly Version _currentVersion = new Version(2, 5); private const string DbVersionType = "db-version"; #region Ctor @@ -134,7 +134,9 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers new Tuple(new Version(1,0), new Version(2,0)), new Tuple(new Version(2,0), new Version(2,1)), new Tuple(new Version(2,1), new Version(2,2)), - new Tuple(new Version(2,2), new Version(2,3)) + new Tuple(new Version(2,2), new Version(2,3)), + new Tuple(new Version(2,3), new Version(2,4)), + new Tuple(new Version(2,4), new Version(2,5)) }; } @@ -163,6 +165,25 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers var addPostingError = $@"ALTER TABLE {_settings.FollowersTableName} ADD postingErrorCount SMALLINT"; await _tools.ExecuteRequestAsync(addPostingError); } + else if (from == new Version(2, 3) && to == new Version(2, 4)) + { + var alterLastSync = $@"ALTER TABLE {_settings.TwitterUserTableName} ALTER COLUMN fetchingErrorCount TYPE INTEGER"; + await _tools.ExecuteRequestAsync(alterLastSync); + + var alterPostingError = $@"ALTER TABLE {_settings.FollowersTableName} ALTER COLUMN postingErrorCount TYPE INTEGER"; + await _tools.ExecuteRequestAsync(alterPostingError); + } + else if (from == new Version(2, 4) && to == new Version(2, 5)) + { + var addMovedTo = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD movedTo VARCHAR(2048)"; + await _tools.ExecuteRequestAsync(addMovedTo); + + var addMovedToAcct = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD movedToAcct VARCHAR(305)"; + await _tools.ExecuteRequestAsync(addMovedToAcct); + + var addDeletedToAcct = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD deleted BOOLEAN"; + await _tools.ExecuteRequestAsync(addDeletedToAcct); + } else { throw new NotImplementedException(); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs index 11214d4..d542a76 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs @@ -18,7 +18,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } #endregion - public async Task CreateTwitterUserAsync(string acct, long lastTweetPostedId) + public async Task CreateTwitterUserAsync(string acct, long lastTweetPostedId, string movedTo = null, string movedToAcct = null) { acct = acct.ToLowerInvariant(); @@ -27,8 +27,15 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers dbConnection.Open(); await dbConnection.ExecuteAsync( - $"INSERT INTO {_settings.TwitterUserTableName} (acct,lastTweetPostedId,lastTweetSynchronizedForAllFollowersId) VALUES(@acct,@lastTweetPostedId,@lastTweetSynchronizedForAllFollowersId)", - new { acct, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = lastTweetPostedId }); + $"INSERT INTO {_settings.TwitterUserTableName} (acct,lastTweetPostedId,lastTweetSynchronizedForAllFollowersId, movedTo, movedToAcct) VALUES(@acct,@lastTweetPostedId,@lastTweetSynchronizedForAllFollowersId,@movedTo,@movedToAcct)", + new + { + acct, + lastTweetPostedId, + lastTweetSynchronizedForAllFollowersId = lastTweetPostedId, + movedTo, + movedToAcct + }); } } @@ -62,7 +69,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public async Task GetTwitterUsersCountAsync() { - var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName}"; + var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE"; using (var dbConnection = Connection) { @@ -75,7 +82,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public async Task GetFailingTwitterUsersCountAsync() { - var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE fetchingErrorCount > 0"; + var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE fetchingErrorCount > 0 AND (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE"; using (var dbConnection = Connection) { @@ -86,9 +93,10 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } } - public async Task GetAllTwitterUsersAsync(int maxNumber) + public async Task GetAllTwitterUsersAsync(int maxNumber, bool retrieveDisabledUser) { - var query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber"; + var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber"; + if (retrieveDisabledUser) query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber"; using (var dbConnection = Connection) { @@ -99,9 +107,10 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } } - public async Task GetAllTwitterUsersAsync() + public async Task GetAllTwitterUsersAsync(bool retrieveDisabledUser) { - var query = $"SELECT * FROM {_settings.TwitterUserTableName}"; + var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE"; + if(retrieveDisabledUser) query = $"SELECT * FROM {_settings.TwitterUserTableName}"; using (var dbConnection = Connection) { @@ -112,26 +121,36 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } } - public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync) + public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync, string movedTo, string movedToAcct, bool deleted) { if(id == default) throw new ArgumentException("id"); if(lastTweetPostedId == default) throw new ArgumentException("lastTweetPostedId"); if(lastTweetSynchronizedForAllFollowersId == default) throw new ArgumentException("lastTweetSynchronizedForAllFollowersId"); if(lastSync == default) throw new ArgumentException("lastSync"); - var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId, fetchingErrorCount = @fetchingErrorCount, lastSync = @lastSync WHERE id = @id"; + var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId, fetchingErrorCount = @fetchingErrorCount, lastSync = @lastSync, movedTo = @movedTo, movedToAcct = @movedToAcct, deleted = @deleted WHERE id = @id"; using (var dbConnection = Connection) { dbConnection.Open(); - await dbConnection.QueryAsync(query, new { id, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId, fetchingErrorCount, lastSync = lastSync.ToUniversalTime() }); + await dbConnection.QueryAsync(query, new + { + id, + lastTweetPostedId, + lastTweetSynchronizedForAllFollowersId, + fetchingErrorCount, + lastSync = lastSync.ToUniversalTime(), + movedTo, + movedToAcct, + deleted + }); } } public async Task UpdateTwitterUserAsync(SyncTwitterUser user) { - await UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, user.LastSync); + await UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, user.LastSync, user.MovedTo, user.MovedToAcct, user.Deleted); } public async Task DeleteTwitterUserAsync(string acct) diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/BirdsiteLive.DAL.csproj b/src/DataAccessLayers/BirdsiteLive.DAL/BirdsiteLive.DAL.csproj index 84e0bf0..847e55f 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/BirdsiteLive.DAL.csproj +++ b/src/DataAccessLayers/BirdsiteLive.DAL/BirdsiteLive.DAL.csproj @@ -1,7 +1,7 @@ - netstandard2.0 + net6.0 diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs index ef2cc36..0c58881 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs @@ -6,12 +6,13 @@ namespace BirdsiteLive.DAL.Contracts { public interface ITwitterUserDal { - Task CreateTwitterUserAsync(string acct, long lastTweetPostedId); + Task CreateTwitterUserAsync(string acct, long lastTweetPostedId, string movedTo = null, + string movedToAcct = null); Task GetTwitterUserAsync(string acct); Task GetTwitterUserAsync(int id); - Task GetAllTwitterUsersAsync(int maxNumber); - Task GetAllTwitterUsersAsync(); - Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync); + Task GetAllTwitterUsersAsync(int maxNumber, bool retrieveDisabledUser); + Task GetAllTwitterUsersAsync(bool retrieveDisabledUser); + Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync, string movedTo, string movedToAcct, bool deleted); Task UpdateTwitterUserAsync(SyncTwitterUser user); Task DeleteTwitterUserAsync(string acct); Task DeleteTwitterUserAsync(int id); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs index 8b18ba1..5089afc 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs @@ -12,6 +12,11 @@ namespace BirdsiteLive.DAL.Models public DateTime LastSync { get; set; } - public int FetchingErrorCount { get; set; } //TODO: update DAL + public int FetchingErrorCount { get; set; } + + public string MovedTo { get; set; } + public string MovedToAcct { get; set; } + + public bool Deleted { get; set; } } } \ No newline at end of file diff --git a/src/Tests/BSLManager.Tests/BSLManager.Tests.csproj b/src/Tests/BSLManager.Tests/BSLManager.Tests.csproj index 033cfe1..8d350bd 100644 --- a/src/Tests/BSLManager.Tests/BSLManager.Tests.csproj +++ b/src/Tests/BSLManager.Tests/BSLManager.Tests.csproj @@ -1,20 +1,21 @@ - netcoreapp3.1 + net6.0 false - - - - + + + + + - + \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.ActivityPub.Tests/ApDeserializerTests.cs b/src/Tests/BirdsiteLive.ActivityPub.Tests/ApDeserializerTests.cs index 3c85113..3d64e90 100644 --- a/src/Tests/BirdsiteLive.ActivityPub.Tests/ApDeserializerTests.cs +++ b/src/Tests/BirdsiteLive.ActivityPub.Tests/ApDeserializerTests.cs @@ -1,4 +1,5 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; +using BirdsiteLive.ActivityPub.Models; +using Microsoft.VisualStudio.TestTools.UnitTesting; using Newtonsoft.Json; namespace BirdsiteLive.ActivityPub.Tests @@ -48,6 +49,20 @@ namespace BirdsiteLive.ActivityPub.Tests Assert.AreEqual("https://mamot.fr/users/testtest", data.apObject.apObject); } + [TestMethod] + public void DeleteDeserializationTest() + { + var json = + "{\"@context\": \"https://www.w3.org/ns/activitystreams\", \"id\": \"https://mastodon.technology/users/deleteduser#delete\", \"type\": \"Delete\", \"actor\": \"https://mastodon.technology/users/deleteduser\", \"to\": [\"https://www.w3.org/ns/activitystreams#Public\"],\"object\": \"https://mastodon.technology/users/deleteduser\",\"signature\": {\"type\": \"RsaSignature2017\",\"creator\": \"https://mastodon.technology/users/deleteduser#main-key\",\"created\": \"2020-11-19T22:43:01Z\",\"signatureValue\": \"peksQao4v5N+sMZgHXZ6xZnGaZrd0s+LqZimu63cnp7O5NBJM6gY9AAu/vKUgrh4C50r66f9OQdHg5yChQhc4ViE+yLR/3/e59YQimelmXJPpcC99Nt0YLU/iTRLsBehY3cDdC6+ogJKgpkToQvB6tG2KrPdrkreYh4Il4eXLKMfiQhgdKluOvenLnl2erPWfE02hIu/jpuljyxSuvJunMdU4yQVSZHTtk/I8q3jjzIzhgyb7ICWU5Hkx0H/47Q24ztsvOgiTWNgO+v6l9vA7qIhztENiRPhzGP5RCCzUKRAe6bcSu1Wfa3NKWqB9BeJ7s+2y2bD7ubPbiEE1MQV7Q==\"}}"; + + var data = ApDeserializer.ProcessActivity(json) as ActivityDelete; + + Assert.AreEqual("https://mastodon.technology/users/deleteduser#delete", data.id); + Assert.AreEqual("Delete", data.type); + Assert.AreEqual("https://mastodon.technology/users/deleteduser", data.actor); + Assert.AreEqual("https://mastodon.technology/users/deleteduser", data.apObject); + } + //[TestMethod] //public void NoteDeserializationTest() //{ diff --git a/src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj b/src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj index 611d29e..3619c7d 100644 --- a/src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj +++ b/src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj @@ -1,20 +1,20 @@ - netcoreapp3.1 + net6.0 false - - - - + + + + - + \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj b/src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj index 0a52603..8308c44 100644 --- a/src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj +++ b/src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj @@ -1,20 +1,20 @@ - netcoreapp3.1 + net6.0 false - - - - + + + + - + \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Common.Tests/Regexes/UrlRegexesTests.cs b/src/Tests/BirdsiteLive.Common.Tests/Regexes/UrlRegexesTests.cs new file mode 100644 index 0000000..78a06be --- /dev/null +++ b/src/Tests/BirdsiteLive.Common.Tests/Regexes/UrlRegexesTests.cs @@ -0,0 +1,72 @@ +using BirdsiteLive.Common.Regexes; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace BirdsiteLive.Common.Tests +{ + [TestClass] + public class UrlRegexesTests + { + [TestMethod] + public void Url_Test() + { + var input = "https://misskey.tdl/users/8hwf6zy2k1#main-key"; + Assert.IsTrue(UrlRegexes.Url.IsMatch(input)); + } + + [TestMethod] + public void Url_Not_Test() + { + var input = "misskey.tdl/users/8hwf6zy2k1#main-key"; + Assert.IsFalse(UrlRegexes.Url.IsMatch(input)); + } + + [TestMethod] + public void Domain_Test() + { + var input = "misskey-data_sq.tdl"; + Assert.IsTrue(UrlRegexes.Domain.IsMatch(input)); + } + + [TestMethod] + public void Domain_Numbers_Test() + { + var input = "miss45654QAzedqskey-data_sq.tdl"; + Assert.IsTrue(UrlRegexes.Domain.IsMatch(input)); + } + + [TestMethod] + public void Domain_Subdomain_Test() + { + var input = "s.sub.dqdq-_Dz9sd.tdl"; + Assert.IsTrue(UrlRegexes.Domain.IsMatch(input)); + } + + [TestMethod] + public void Domain_Not_Test() + { + var input = "mis$s45654QAzedqskey-data_sq.tdl"; + Assert.IsFalse(UrlRegexes.Domain.IsMatch(input)); + } + + [TestMethod] + public void Domain_Slash_Test() + { + var input = "miss45654QAz/edqskey-data_sq.tdl"; + Assert.IsFalse(UrlRegexes.Domain.IsMatch(input)); + } + + [TestMethod] + public void Domain_NotSub_Test() + { + var input = ".mis$s45654QAzedqskey-data_sq.tdl"; + Assert.IsFalse(UrlRegexes.Domain.IsMatch(input)); + } + + [TestMethod] + public void Domain_NotExt_Test() + { + var input = ".mis$s45654QAzedqskey-data_sq.tdl"; + Assert.IsFalse(UrlRegexes.Domain.IsMatch(input)); + } + } +} \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Cryptography.Tests/BirdsiteLive.Cryptography.Tests.csproj b/src/Tests/BirdsiteLive.Cryptography.Tests/BirdsiteLive.Cryptography.Tests.csproj index 0c9c0a6..d4ad825 100644 --- a/src/Tests/BirdsiteLive.Cryptography.Tests/BirdsiteLive.Cryptography.Tests.csproj +++ b/src/Tests/BirdsiteLive.Cryptography.Tests/BirdsiteLive.Cryptography.Tests.csproj @@ -1,20 +1,20 @@ - netcoreapp3.1 + net6.0 false - - - - + + + + - + \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/BirdsiteLive.DAL.Postgres.Tests.csproj b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/BirdsiteLive.DAL.Postgres.Tests.csproj index da05ef2..d4b3561 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/BirdsiteLive.DAL.Postgres.Tests.csproj +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/BirdsiteLive.DAL.Postgres.Tests.csproj @@ -1,16 +1,16 @@ - netcoreapp3.1 + net6.0 false - - - - + + + + diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/Base/PostgresTestingBase.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/Base/PostgresTestingBase.cs index 72bf352..d68a07e 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/Base/PostgresTestingBase.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/Base/PostgresTestingBase.cs @@ -14,7 +14,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers.Base { _settings = new PostgresSettings { - ConnString = "Host=127.0.0.1;Username=postgres;Password=mysecretpassword;Database=mytestdb", + ConnString = "Host=127.0.0.1;Username=birdtest;Password=mysecretpassword;Database=birdsitetest", DbVersionTableName = "DbVersionTableName" + RandomGenerator.GetString(4), CachedTweetsTableName = "CachedTweetsTableName" + RandomGenerator.GetString(4), FollowersTableName = "FollowersTableName" + RandomGenerator.GetString(4), diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs index a22df0f..3927f13 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs @@ -340,6 +340,48 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers Assert.AreEqual(10, result.PostingErrorCount); } + [TestMethod] + public async Task CreateUpdateAndGetFollower_Integer() + { + var acct = "myhandle"; + var host = "domain.ext"; + var following = new[] { 12, 19, 23 }; + var followingSync = new Dictionary() + { + {12, 165L}, + {19, 166L}, + {23, 167L} + }; + var inboxRoute = "/myhandle/inbox"; + var sharedInboxRoute = "/inbox"; + var actorId = $"https://{host}/{acct}"; + + var dal = new FollowersPostgresDal(_settings); + await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync); + var result = await dal.GetFollowerAsync(acct, host); + + var updatedFollowing = new List { 12, 19, 23, 24 }; + var updatedFollowingSync = new Dictionary(){ + {12, 170L}, + {19, 171L}, + {23, 172L}, + {24, 173L} + }; + result.Followings = updatedFollowing.ToList(); + result.FollowingsSyncStatus = updatedFollowingSync; + result.PostingErrorCount = 32768; + + await dal.UpdateFollowerAsync(result); + result = await dal.GetFollowerAsync(acct, host); + + Assert.AreEqual(updatedFollowing.Count, result.Followings.Count); + Assert.AreEqual(updatedFollowing[0], result.Followings[0]); + Assert.AreEqual(updatedFollowingSync.Count, result.FollowingsSyncStatus.Count); + Assert.AreEqual(updatedFollowingSync.First().Key, result.FollowingsSyncStatus.First().Key); + Assert.AreEqual(updatedFollowingSync.First().Value, result.FollowingsSyncStatus.First().Value); + Assert.AreEqual(32768, result.PostingErrorCount); + } + [TestMethod] public async Task CreateUpdateAndGetFollower_Remove() { diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs index 0b007b6..936bb73 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs @@ -71,6 +71,28 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers Assert.AreEqual(result.Id, resultById.Id); } + [TestMethod] + public async Task CreateAndGetMigratedUser_byId() + { + var acct = "myid"; + var lastTweetId = 1548L; + var movedTo = "https://"; + var movedToAcct = "@account@instance"; + + var dal = new TwitterUserPostgresDal(_settings); + + await dal.CreateTwitterUserAsync(acct, lastTweetId, movedTo, movedToAcct); + var result = await dal.GetTwitterUserAsync(acct); + var resultById = await dal.GetTwitterUserAsync(result.Id); + + Assert.AreEqual(acct, resultById.Acct); + Assert.AreEqual(lastTweetId, resultById.LastTweetPostedId); + Assert.AreEqual(lastTweetId, resultById.LastTweetSynchronizedForAllFollowersId); + Assert.AreEqual(result.Id, resultById.Id); + Assert.AreEqual(movedTo, result.MovedTo); + Assert.AreEqual(movedToAcct, result.MovedToAcct); + } + [TestMethod] public async Task CreateUpdateAndGetUser() { @@ -87,7 +109,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers var updatedLastSyncId = 1550L; var now = DateTime.Now; var errors = 15; - await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now); + await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now, null, null, false); result = await dal.GetTwitterUserAsync(acct); @@ -96,6 +118,68 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId); Assert.AreEqual(errors, result.FetchingErrorCount); Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100); + Assert.AreEqual(null, result.MovedTo); + Assert.AreEqual(null, result.MovedToAcct); + } + + [TestMethod] + public async Task CreateUpdateAndGetMigratedUser() + { + var acct = "myid"; + var lastTweetId = 1548L; + + var dal = new TwitterUserPostgresDal(_settings); + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + var result = await dal.GetTwitterUserAsync(acct); + + + var updatedLastTweetId = 1600L; + var updatedLastSyncId = 1550L; + var now = DateTime.Now; + var errors = 15; + var movedTo = "https://"; + var movedToAcct = "@account@instance"; + await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now, movedTo, movedToAcct, false); + + result = await dal.GetTwitterUserAsync(acct); + + Assert.AreEqual(acct, result.Acct); + Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId); + Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId); + Assert.AreEqual(errors, result.FetchingErrorCount); + Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100); + Assert.AreEqual(movedTo, result.MovedTo); + Assert.AreEqual(movedToAcct, result.MovedToAcct); + } + + [TestMethod] + public async Task CreateUpdateAndGetDeletedUser() + { + var acct = "myid"; + var lastTweetId = 1548L; + + var dal = new TwitterUserPostgresDal(_settings); + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + var result = await dal.GetTwitterUserAsync(acct); + + var updatedLastTweetId = 1600L; + var updatedLastSyncId = 1550L; + var now = DateTime.Now; + var errors = 15; + await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now, null, null, true); + + result = await dal.GetTwitterUserAsync(acct); + + Assert.AreEqual(acct, result.Acct); + Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId); + Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId); + Assert.AreEqual(errors, result.FetchingErrorCount); + Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100); + Assert.AreEqual(null, result.MovedTo); + Assert.AreEqual(null, result.MovedToAcct); + Assert.AreEqual(true, result.Deleted); } [TestMethod] @@ -130,12 +214,44 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100); } + [TestMethod] + public async Task CreateUpdate3AndGetUser() + { + var acct = "myid"; + var lastTweetId = 1548L; + + var dal = new TwitterUserPostgresDal(_settings); + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + var result = await dal.GetTwitterUserAsync(acct); + + + var updatedLastTweetId = 1600L; + var updatedLastSyncId = 1550L; + var now = DateTime.Now; + var errors = 32768; + + result.LastTweetPostedId = updatedLastTweetId; + result.LastTweetSynchronizedForAllFollowersId = updatedLastSyncId; + result.FetchingErrorCount = errors; + result.LastSync = now; + await dal.UpdateTwitterUserAsync(result); + + result = await dal.GetTwitterUserAsync(acct); + + Assert.AreEqual(acct, result.Acct); + Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId); + Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId); + Assert.AreEqual(errors, result.FetchingErrorCount); + Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100); + } + [TestMethod] [ExpectedException(typeof(ArgumentException))] public async Task Update_NoId() { var dal = new TwitterUserPostgresDal(_settings); - await dal.UpdateTwitterUserAsync(default, default, default, default, DateTime.UtcNow); + await dal.UpdateTwitterUserAsync(default, default, default, default, DateTime.UtcNow, null, null, false); } [TestMethod] @@ -143,7 +259,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task Update_NoLastTweetPostedId() { var dal = new TwitterUserPostgresDal(_settings); - await dal.UpdateTwitterUserAsync(12, default, default, default, DateTime.UtcNow); + await dal.UpdateTwitterUserAsync(12, default, default, default, DateTime.UtcNow, null, null, false); } [TestMethod] @@ -151,7 +267,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task Update_NoLastTweetSynchronizedForAllFollowersId() { var dal = new TwitterUserPostgresDal(_settings); - await dal.UpdateTwitterUserAsync(12, 9556, default, default, DateTime.UtcNow); + await dal.UpdateTwitterUserAsync(12, 9556, default, default, DateTime.UtcNow, null, null, false); } [TestMethod] @@ -159,7 +275,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers public async Task Update_NoLastSync() { var dal = new TwitterUserPostgresDal(_settings); - await dal.UpdateTwitterUserAsync(12, 9556, 65, default, default); + await dal.UpdateTwitterUserAsync(12, 9556, 65, default, default, null, null, false); } [TestMethod] @@ -224,12 +340,79 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers await dal.CreateTwitterUserAsync(acct, lastTweetId); } - var result = await dal.GetAllTwitterUsersAsync(1000); + for (int i = 0; i < 10; i++) + { + var acct = $"migrated-myid{i}"; + var lastTweetId = 1548L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain"); + } + + for (int i = 0; i < 10; i++) + { + var acct = $"deleted-myid{i}"; + var lastTweetId = 148L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + var user = await dal.GetTwitterUserAsync(acct); + user.Deleted = true; + user.LastSync = DateTime.UtcNow; + await dal.UpdateTwitterUserAsync(user); + } + + var result = await dal.GetAllTwitterUsersAsync(1100, false); Assert.AreEqual(1000, result.Length); Assert.IsFalse(result[0].Id == default); Assert.IsFalse(result[0].Acct == default); Assert.IsFalse(result[0].LastTweetPostedId == default); Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default); + + foreach (var user in result) + { + Assert.IsTrue(string.IsNullOrWhiteSpace(user.MovedTo)); + Assert.IsTrue(string.IsNullOrWhiteSpace(user.MovedToAcct)); + Assert.IsFalse(user.Deleted); + } + } + + [TestMethod] + public async Task GetAllTwitterUsers_Top_RetrieveDeleted() + { + var dal = new TwitterUserPostgresDal(_settings); + for (var i = 0; i < 1000; i++) + { + var acct = $"myid{i}"; + var lastTweetId = 1548L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + } + + for (int i = 0; i < 10; i++) + { + var acct = $"migrated-myid{i}"; + var lastTweetId = 1548L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain"); + } + + for (int i = 0; i < 10; i++) + { + var acct = $"deleted-myid{i}"; + var lastTweetId = 148L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + var user = await dal.GetTwitterUserAsync(acct); + user.Deleted = true; + user.LastSync = DateTime.UtcNow; + await dal.UpdateTwitterUserAsync(user); + } + + var result = await dal.GetAllTwitterUsersAsync(1100, true); + Assert.AreEqual(1020, result.Length); + Assert.IsFalse(result[0].Id == default); + Assert.IsFalse(result[0].Acct == default); + Assert.IsFalse(result[0].LastTweetPostedId == default); + Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default); } [TestMethod] @@ -247,7 +430,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers // Update accounts var now = DateTime.UtcNow; - var allUsers = await dal.GetAllTwitterUsersAsync(); + var allUsers = await dal.GetAllTwitterUsersAsync(false); foreach (var acc in allUsers) { var lastSync = now.AddDays(acc.LastTweetPostedId); @@ -258,7 +441,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers // Create a not init account await dal.CreateTwitterUserAsync("not_init", -1); - var result = await dal.GetAllTwitterUsersAsync(10); + var result = await dal.GetAllTwitterUsersAsync(10, false); Assert.IsTrue(result.Any(x => x.Acct == "myid0")); Assert.IsTrue(result.Any(x => x.Acct == "myid8")); @@ -281,15 +464,15 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers await dal.CreateTwitterUserAsync(acct, lastTweetId); } - var allUsers = await dal.GetAllTwitterUsersAsync(100); + var allUsers = await dal.GetAllTwitterUsersAsync(100, false); for (var i = 0; i < 20; i++) { var user = allUsers[i]; var date = i % 2 == 0 ? oldest : newest; - await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date); + await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date, null, null, false); } - var result = await dal.GetAllTwitterUsersAsync(10); + var result = await dal.GetAllTwitterUsersAsync(10, false); Assert.AreEqual(10, result.Length); Assert.IsFalse(result[0].Id == default); Assert.IsFalse(result[0].Acct == default); @@ -312,7 +495,15 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers await dal.CreateTwitterUserAsync(acct, lastTweetId); } - var result = await dal.GetAllTwitterUsersAsync(); + for (int i = 0; i < 10; i++) + { + var acct = $"migrated-myid{i}"; + var lastTweetId = 1548L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain"); + } + + var result = await dal.GetAllTwitterUsersAsync(false); Assert.AreEqual(1000, result.Length); Assert.IsFalse(result[0].Id == default); Assert.IsFalse(result[0].Acct == default); @@ -350,7 +541,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers if (i == 0 || i == 2 || i == 3) { var t = await dal.GetTwitterUserAsync(acct); - await dal.UpdateTwitterUserAsync(t.Id ,1L,2L, 50+i*2, DateTime.Now); + await dal.UpdateTwitterUserAsync(t.Id ,1L,2L, 50+i*2, DateTime.Now, null, null, false); } } diff --git a/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj b/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj index 0992b02..d12987d 100644 --- a/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj +++ b/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj @@ -1,21 +1,21 @@ - netcoreapp3.1 + net6.0 false - - - - - + + + + + - + \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj b/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj index 626c941..aaebf2a 100644 --- a/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj +++ b/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj @@ -1,17 +1,17 @@ - netcoreapp3.1 + net6.0 false - - - - - + + + + + @@ -22,4 +22,4 @@ - + \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessDeleteUserTests.cs b/src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessDeleteUserTests.cs new file mode 100644 index 0000000..85900da --- /dev/null +++ b/src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessDeleteUserTests.cs @@ -0,0 +1,97 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Domain.BusinessUseCases; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace BirdsiteLive.Domain.Tests.BusinessUseCases +{ + [TestClass] + public class ProcessDeleteUserTests + { + [TestMethod] + public async Task ExecuteAsync_NoMoreFollowings() + { + #region Stubs + var follower = new Follower + { + Id = 12, + Followings = new List { 1 } + }; + #endregion + + #region Mocks + var followersDalMock = new Mock(MockBehavior.Strict); + followersDalMock + .Setup(x => x.GetFollowersAsync( + It.Is(y => y == 1))) + .ReturnsAsync(new[] { follower }); + + followersDalMock + .Setup(x => x.DeleteFollowerAsync( + It.Is(y => y == 12))) + .Returns(Task.CompletedTask); + + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.DeleteTwitterUserAsync( + It.Is(y => y == 1))) + .Returns(Task.CompletedTask); + #endregion + + var action = new ProcessDeleteUser(followersDalMock.Object, twitterUserDalMock.Object); + await action.ExecuteAsync(follower); + + #region Validations + followersDalMock.VerifyAll(); + twitterUserDalMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ExecuteAsync_HaveFollowings() + { + #region Stubs + var follower = new Follower + { + Id = 12, + Followings = new List { 1 } + }; + + var followers = new List + { + follower, + new Follower + { + Id = 11 + } + }; + #endregion + + #region Mocks + var followersDalMock = new Mock(MockBehavior.Strict); + followersDalMock + .Setup(x => x.GetFollowersAsync( + It.Is(y => y == 1))) + .ReturnsAsync(followers.ToArray()); + + followersDalMock + .Setup(x => x.DeleteFollowerAsync( + It.Is(y => y == 12))) + .Returns(Task.CompletedTask); + + var twitterUserDalMock = new Mock(MockBehavior.Strict); + #endregion + + var action = new ProcessDeleteUser(followersDalMock.Object, twitterUserDalMock.Object); + await action.ExecuteAsync(follower); + + #region Validations + followersDalMock.VerifyAll(); + twitterUserDalMock.VerifyAll(); + #endregion + } + } +} \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessFollowUserTests.cs b/src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessFollowUserTests.cs index 0fb03ae..8f2f393 100644 --- a/src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessFollowUserTests.cs +++ b/src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessFollowUserTests.cs @@ -77,7 +77,9 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases twitterUserDalMock .Setup(x => x.CreateTwitterUserAsync( It.Is(y => y == twitterName), - It.Is(y => y == -1))) + It.Is(y => y == -1), + It.Is(y => y == null), + It.Is(y => y == null))) .Returns(Task.CompletedTask); #endregion diff --git a/src/Tests/BirdsiteLive.Domain.Tests/TheFedInfoServiceTests.cs b/src/Tests/BirdsiteLive.Domain.Tests/TheFedInfoServiceTests.cs new file mode 100644 index 0000000..6ebc242 --- /dev/null +++ b/src/Tests/BirdsiteLive.Domain.Tests/TheFedInfoServiceTests.cs @@ -0,0 +1,36 @@ +using System; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; +using BirdsiteLive.Common.Regexes; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace BirdsiteLive.Domain.Tests +{ + [TestClass] + public class TheFedInfoServiceTests + { + [TestMethod] + public async Task GetBslInstanceListAsyncTest() + { + var httpClientFactoryMock = new Mock(); + httpClientFactoryMock + .Setup(x => x.CreateClient(It.IsAny())) + .Returns(new HttpClient()); + + var service = new TheFedInfoService(httpClientFactoryMock.Object); + + var bslInstanceList = await service.GetBslInstanceListAsync(); + + Assert.IsTrue(bslInstanceList.Count > 0); + + foreach (var instanceInfo in bslInstanceList) + { + Assert.IsFalse(string.IsNullOrWhiteSpace(instanceInfo.Host)); + Assert.IsTrue(UrlRegexes.Domain.IsMatch(instanceInfo.Host)); + Assert.IsTrue(instanceInfo.Version > new Version(0, 1, 0)); + } + } + } +} \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Moderation.Tests/Actions/RemoveFollowerActionTests.cs b/src/Tests/BirdsiteLive.Moderation.Tests/Actions/RemoveFollowerActionTests.cs index 3b83739..34f40b2 100644 --- a/src/Tests/BirdsiteLive.Moderation.Tests/Actions/RemoveFollowerActionTests.cs +++ b/src/Tests/BirdsiteLive.Moderation.Tests/Actions/RemoveFollowerActionTests.cs @@ -2,6 +2,7 @@ using System.Threading.Tasks; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; +using BirdsiteLive.Domain.BusinessUseCases; using BirdsiteLive.Moderation.Actions; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -29,31 +30,19 @@ namespace BirdsiteLive.Moderation.Tests.Actions It.Is(y => y.Id == follower.Id))) .Returns(Task.CompletedTask); - var followersDalMock = new Mock(MockBehavior.Strict); - followersDalMock - .Setup(x => x.GetFollowersAsync( - It.Is(y => y == 1))) - .ReturnsAsync(new[] {follower}); - - followersDalMock - .Setup(x => x.DeleteFollowerAsync( - It.Is(y => y == 12))) - .Returns(Task.CompletedTask); - - var twitterUserDalMock = new Mock(MockBehavior.Strict); - twitterUserDalMock - .Setup(x => x.DeleteTwitterUserAsync( - It.Is(y => y == 1))) + var processDeleteUserMock = new Mock(MockBehavior.Strict); + processDeleteUserMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Id == follower.Id))) .Returns(Task.CompletedTask); #endregion - var action = new RemoveFollowerAction(followersDalMock.Object, twitterUserDalMock.Object, rejectAllFollowingsActionMock.Object); + var action = new RemoveFollowerAction(rejectAllFollowingsActionMock.Object, processDeleteUserMock.Object); await action.ProcessAsync(follower); #region Validations - followersDalMock.VerifyAll(); - twitterUserDalMock.VerifyAll(); rejectAllFollowingsActionMock.VerifyAll(); + processDeleteUserMock.VerifyAll(); #endregion } @@ -66,15 +55,6 @@ namespace BirdsiteLive.Moderation.Tests.Actions Id = 12, Followings = new List { 1 } }; - - var followers = new List - { - follower, - new Follower - { - Id = 11 - } - }; #endregion #region Mocks @@ -84,27 +64,19 @@ namespace BirdsiteLive.Moderation.Tests.Actions It.Is(y => y.Id == follower.Id))) .Returns(Task.CompletedTask); - var followersDalMock = new Mock(MockBehavior.Strict); - followersDalMock - .Setup(x => x.GetFollowersAsync( - It.Is(y => y == 1))) - .ReturnsAsync(followers.ToArray()); - - followersDalMock - .Setup(x => x.DeleteFollowerAsync( - It.Is(y => y == 12))) + var processDeleteUserMock = new Mock(MockBehavior.Strict); + processDeleteUserMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Id == follower.Id))) .Returns(Task.CompletedTask); - - var twitterUserDalMock = new Mock(MockBehavior.Strict); #endregion - var action = new RemoveFollowerAction(followersDalMock.Object, twitterUserDalMock.Object, rejectAllFollowingsActionMock.Object); + var action = new RemoveFollowerAction(rejectAllFollowingsActionMock.Object, processDeleteUserMock.Object); await action.ProcessAsync(follower); #region Validations - followersDalMock.VerifyAll(); - twitterUserDalMock.VerifyAll(); rejectAllFollowingsActionMock.VerifyAll(); + processDeleteUserMock.VerifyAll(); #endregion } } diff --git a/src/Tests/BirdsiteLive.Moderation.Tests/BirdsiteLive.Moderation.Tests.csproj b/src/Tests/BirdsiteLive.Moderation.Tests/BirdsiteLive.Moderation.Tests.csproj index e85b592..1e66e53 100644 --- a/src/Tests/BirdsiteLive.Moderation.Tests/BirdsiteLive.Moderation.Tests.csproj +++ b/src/Tests/BirdsiteLive.Moderation.Tests/BirdsiteLive.Moderation.Tests.csproj @@ -1,21 +1,21 @@ - netcoreapp3.1 + net6.0 false - - - - - + + + + + - + \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Moderation.Tests/Processors/TwitterAccountModerationProcessorTests.cs b/src/Tests/BirdsiteLive.Moderation.Tests/Processors/TwitterAccountModerationProcessorTests.cs index 21d1288..8473424 100644 --- a/src/Tests/BirdsiteLive.Moderation.Tests/Processors/TwitterAccountModerationProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Moderation.Tests/Processors/TwitterAccountModerationProcessorTests.cs @@ -48,7 +48,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync(It.Is(y => y == false))) .ReturnsAsync(allUsers.ToArray()); var moderationRepositoryMock = new Mock(MockBehavior.Strict); @@ -87,7 +87,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync(It.Is(y => y == false))) .ReturnsAsync(allUsers.ToArray()); var moderationRepositoryMock = new Mock(MockBehavior.Strict); @@ -130,7 +130,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync(It.Is(y => y == false))) .ReturnsAsync(allUsers.ToArray()); var moderationRepositoryMock = new Mock(MockBehavior.Strict); @@ -173,7 +173,7 @@ namespace BirdsiteLive.Moderation.Tests.Processors #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync(It.Is(y => y == false))) .ReturnsAsync(allUsers.ToArray()); var moderationRepositoryMock = new Mock(MockBehavior.Strict); diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj b/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj index d1cfd06..38d3a60 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj @@ -1,17 +1,17 @@ - netcoreapp3.1 + net6.0 false - - - - - + + + + + @@ -24,4 +24,4 @@ - + \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs index d5fbeef..cd2d116 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RefreshTwitterUserStatusProcessorTests.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -159,11 +160,136 @@ namespace BirdsiteLive.Pipeline.Tests.Processors twitterUserServiceMock .Setup(x => x.GetUser(It.Is(y => y == acct2))) - .Returns((TwitterUser) null); + .Throws(new UserNotFoundException()); + + var twitterUserDalMock = new Mock(MockBehavior.Strict); + + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); + removeTwitterAccountActionMock + .Setup(x => x.ProcessAsync(It.Is(y => y.Acct == acct2))) + .Returns(Task.CompletedTask); + #endregion + + var processor = new RefreshTwitterUserStatusProcessor(twitterUserServiceMock.Object, twitterUserDalMock.Object, removeTwitterAccountActionMock.Object, settings); + var result = await processor.ProcessAsync(users.ToArray(), CancellationToken.None); + + #region Validations + Assert.AreEqual(1, result.Length); + Assert.IsTrue(result.Any(x => x.User.Id == userId1)); + + twitterUserServiceMock.VerifyAll(); + twitterUserDalMock.VerifyAll(); + removeTwitterAccountActionMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_Suspended_Test() + { + #region Stubs + var userId1 = 1; + var acct1 = "user1"; + + var userId2 = 2; + var acct2 = "user2"; + + var users = new List + { + new SyncTwitterUser + { + Id = userId1, + Acct = acct1 + }, + new SyncTwitterUser + { + Id = userId2, + Acct = acct2 + } + }; + + var settings = new InstanceSettings + { + FailingTwitterUserCleanUpThreshold = 300 + }; + #endregion + + #region Mocks + var twitterUserServiceMock = new Mock(MockBehavior.Strict); + twitterUserServiceMock + .Setup(x => x.GetUser(It.Is(y => y == acct1))) + .Returns(new TwitterUser + { + Protected = false + }); twitterUserServiceMock - .Setup(x => x.PurgeUser(It.Is(y => y == acct2))); + .Setup(x => x.GetUser(It.Is(y => y == acct2))) + .Throws(new UserHasBeenSuspendedException()); + var twitterUserDalMock = new Mock(MockBehavior.Strict); + + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); + removeTwitterAccountActionMock + .Setup(x => x.ProcessAsync(It.Is(y => y.Acct == acct2))) + .Returns(Task.CompletedTask); + #endregion + + var processor = new RefreshTwitterUserStatusProcessor(twitterUserServiceMock.Object, twitterUserDalMock.Object, removeTwitterAccountActionMock.Object, settings); + var result = await processor.ProcessAsync(users.ToArray(), CancellationToken.None); + + #region Validations + Assert.AreEqual(1, result.Length); + Assert.IsTrue(result.Any(x => x.User.Id == userId1)); + + twitterUserServiceMock.VerifyAll(); + twitterUserDalMock.VerifyAll(); + removeTwitterAccountActionMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_Exception_Test() + { + #region Stubs + var userId1 = 1; + var acct1 = "user1"; + + var userId2 = 2; + var acct2 = "user2"; + + var users = new List + { + new SyncTwitterUser + { + Id = userId1, + Acct = acct1 + }, + new SyncTwitterUser + { + Id = userId2, + Acct = acct2 + } + }; + + var settings = new InstanceSettings + { + FailingTwitterUserCleanUpThreshold = 300 + }; + #endregion + + #region Mocks + var twitterUserServiceMock = new Mock(MockBehavior.Strict); + twitterUserServiceMock + .Setup(x => x.GetUser(It.Is(y => y == acct1))) + .Returns(new TwitterUser + { + Protected = false + }); + + twitterUserServiceMock + .Setup(x => x.GetUser(It.Is(y => y == acct2))) + .Throws(new Exception()); + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) @@ -194,7 +320,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors } [TestMethod] - public async Task ProcessAsync_Unfound_OverThreshold_Test() + public async Task ProcessAsync_Error_Test() { #region Stubs var userId1 = 1; @@ -235,10 +361,79 @@ namespace BirdsiteLive.Pipeline.Tests.Processors twitterUserServiceMock .Setup(x => x.GetUser(It.Is(y => y == acct2))) .Returns((TwitterUser)null); + + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) + .ReturnsAsync(new SyncTwitterUser + { + Id = userId2, + FetchingErrorCount = 0 + }); + + twitterUserDalMock + .Setup(x => x.UpdateTwitterUserAsync(It.Is(y => y.Id == userId2 && y.FetchingErrorCount == 1))) + .Returns(Task.CompletedTask); + + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); + #endregion + + var processor = new RefreshTwitterUserStatusProcessor(twitterUserServiceMock.Object, twitterUserDalMock.Object, removeTwitterAccountActionMock.Object, settings); + var result = await processor.ProcessAsync(users.ToArray(), CancellationToken.None); + + #region Validations + Assert.AreEqual(1, result.Length); + Assert.IsTrue(result.Any(x => x.User.Id == userId1)); + + twitterUserServiceMock.VerifyAll(); + twitterUserDalMock.VerifyAll(); + removeTwitterAccountActionMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_Error_OverThreshold_Test() + { + #region Stubs + var userId1 = 1; + var acct1 = "user1"; + + var userId2 = 2; + var acct2 = "user2"; + + var users = new List + { + new SyncTwitterUser + { + Id = userId1, + Acct = acct1 + }, + new SyncTwitterUser + { + Id = userId2, + Acct = acct2 + } + }; + + var settings = new InstanceSettings + { + FailingTwitterUserCleanUpThreshold = 300 + }; + #endregion + + #region Mocks + var twitterUserServiceMock = new Mock(MockBehavior.Strict); + twitterUserServiceMock + .Setup(x => x.GetUser(It.Is(y => y == acct1))) + .Returns(new TwitterUser + { + Protected = false + }); twitterUserServiceMock - .Setup(x => x.PurgeUser(It.Is(y => y == acct2))); - + .Setup(x => x.GetUser(It.Is(y => y == acct2))) + .Returns((TwitterUser)null); + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) @@ -312,8 +507,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors { Protected = true }); - + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) + .ReturnsAsync(new SyncTwitterUser + { + Id = userId2, + FetchingErrorCount = 0 + }); + + twitterUserDalMock + .Setup(x => x.UpdateTwitterUserAsync(It.Is(y => y.Id == userId2 && y.FetchingErrorCount == 1))) + .Returns(Task.CompletedTask); + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); #endregion @@ -331,7 +538,81 @@ namespace BirdsiteLive.Pipeline.Tests.Processors } [TestMethod] - public async Task ProcessAsync_Unfound_NotInit_Test() + public async Task ProcessAsync_Protected_OverThreshold_Test() + { + #region Stubs + var userId1 = 1; + var acct1 = "user1"; + + var userId2 = 2; + var acct2 = "user2"; + + var users = new List + { + new SyncTwitterUser + { + Id = userId1, + Acct = acct1 + }, + new SyncTwitterUser + { + Id = userId2, + Acct = acct2 + } + }; + + var settings = new InstanceSettings + { + FailingTwitterUserCleanUpThreshold = 300 + }; + #endregion + + #region Mocks + var twitterUserServiceMock = new Mock(MockBehavior.Strict); + twitterUserServiceMock + .Setup(x => x.GetUser(It.Is(y => y == acct1))) + .Returns(new TwitterUser + { + Protected = false + }); + + twitterUserServiceMock + .Setup(x => x.GetUser(It.Is(y => y == acct2))) + .Returns(new TwitterUser + { + Protected = true + }); + + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) + .ReturnsAsync(new SyncTwitterUser + { + Id = userId2, + FetchingErrorCount = 500 + }); + + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); + removeTwitterAccountActionMock + .Setup(x => x.ProcessAsync(It.Is(y => y.Id == userId2))) + .Returns(Task.CompletedTask); + #endregion + + var processor = new RefreshTwitterUserStatusProcessor(twitterUserServiceMock.Object, twitterUserDalMock.Object, removeTwitterAccountActionMock.Object, settings); + var result = await processor.ProcessAsync(users.ToArray(), CancellationToken.None); + + #region Validations + Assert.AreEqual(1, result.Length); + Assert.IsTrue(result.Any(x => x.User.Id == userId1)); + + twitterUserServiceMock.VerifyAll(); + twitterUserDalMock.VerifyAll(); + removeTwitterAccountActionMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_Error_NotInit_Test() { #region Stubs var userId1 = 1; @@ -361,9 +642,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Setup(x => x.GetUser(It.Is(y => y == acct1))) .Returns((TwitterUser)null); - twitterUserServiceMock - .Setup(x => x.PurgeUser(It.Is(y => y == acct1))); - var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct1))) @@ -388,5 +666,77 @@ namespace BirdsiteLive.Pipeline.Tests.Processors removeTwitterAccountActionMock.VerifyAll(); #endregion } + + [TestMethod] + public async Task ProcessAsync_RateLimited_Test() + { + #region Stubs + var userId1 = 1; + var acct1 = "user1"; + + var userId2 = 2; + var acct2 = "user2"; + + var users = new List + { + new SyncTwitterUser + { + Id = userId1, + Acct = acct1 + }, + new SyncTwitterUser + { + Id = userId2, + Acct = acct2 + } + }; + + var settings = new InstanceSettings + { + FailingTwitterUserCleanUpThreshold = 300 + }; + #endregion + + #region Mocks + var twitterUserServiceMock = new Mock(MockBehavior.Strict); + twitterUserServiceMock + .Setup(x => x.GetUser(It.Is(y => y == acct1))) + .Returns(new TwitterUser + { + Protected = false, + }); + + twitterUserServiceMock + .Setup(x => x.GetUser(It.Is(y => y == acct2))) + .Throws(new RateLimitExceededException()); + + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.GetTwitterUserAsync(It.Is(y => y == acct2))) + .ReturnsAsync(new SyncTwitterUser + { + Id = userId2, + FetchingErrorCount = 20 + }); + + twitterUserDalMock + .Setup(x => x.UpdateTwitterUserAsync(It.Is(y => y.Id == userId2 && y.FetchingErrorCount == 20))) + .Returns(Task.CompletedTask); + + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); + #endregion + + var processor = new RefreshTwitterUserStatusProcessor(twitterUserServiceMock.Object, twitterUserDalMock.Object, removeTwitterAccountActionMock.Object, settings); + var result = await processor.ProcessAsync(users.ToArray(), CancellationToken.None); + + #region Validations + Assert.AreEqual(1, result.Length); + Assert.IsTrue(result.Any(x => x.User.Id == userId1)); + + twitterUserServiceMock.VerifyAll(); + twitterUserDalMock.VerifyAll(); + removeTwitterAccountActionMock.VerifyAll(); + #endregion + } } } \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs index 17a3aa2..f95ad82 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs @@ -64,7 +64,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == tweets.Last().Id), It.Is(y => y == tweets.Last().Id), It.Is(y => y == 0), - It.IsAny() + It.IsAny(), + It.Is(y => y == null), + It.Is(y => y == null), + It.Is(y => y == false) )) .Returns(Task.CompletedTask); diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs index 4d0e465..daf0bfa 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs @@ -40,7 +40,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == maxUsers))) + It.Is(y => y == maxUsers), + It.Is(y => y == false))) .ReturnsAsync(users); var loggerMock = new Mock>(); @@ -83,7 +84,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .SetupSequence(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == maxUsers))) + It.Is(y => y == maxUsers), + It.Is(y => y == false))) .ReturnsAsync(users.ToArray()) .ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0]) @@ -130,7 +132,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .SetupSequence(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == maxUsers))) + It.Is(y => y == maxUsers), + It.Is(y => y == false))) .ReturnsAsync(users.ToArray()) .ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0]) @@ -178,7 +181,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == maxUsers))) + It.Is(y => y == maxUsers), + It.Is(y => y == false))) .ReturnsAsync(new SyncTwitterUser[0]); var loggerMock = new Mock>(); @@ -215,7 +219,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == maxUsers))) + It.Is(y => y == maxUsers), + It.Is(y => y == false))) .Returns(async () => await DelayFaultedTask(new Exception())); var loggerMock = new Mock>(); diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs index 4587071..d245713 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs @@ -1,16 +1,16 @@ -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; +using BirdsiteLive.Moderation.Actions; using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Pipeline.Processors; using BirdsiteLive.Twitter.Models; -using Castle.DynamicProxy.Contributors; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; namespace BirdsiteLive.Pipeline.Tests.Processors { @@ -66,17 +66,93 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == tweet2.Id), It.Is(y => y == tweet2.Id), It.Is(y => y == 0), - It.IsAny() + It.IsAny(), + It.Is(y => y == null), + It.Is(y => y == null), + It.Is(y => y == false) )) .Returns(Task.CompletedTask); + + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object); + var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object, removeTwitterAccountActionMock.Object); await processor.ProcessAsync(usersWithTweets, CancellationToken.None); #region Validations twitterUserDalMock.VerifyAll(); loggerMock.VerifyAll(); + removeTwitterAccountActionMock.VerifyAll(); + #endregion + } + + [TestMethod] + [ExpectedException(typeof(ArgumentException))] + public async Task ProcessAsync_Exception_Test() + { + #region Stubs + var user = new SyncTwitterUser + { + Id = 1 + }; + var tweet1 = new ExtractedTweet + { + Id = 36 + }; + var tweet2 = new ExtractedTweet + { + Id = 37 + }; + var follower1 = new Follower + { + FollowingsSyncStatus = new Dictionary + { + {1, 37} + } + }; + + var usersWithTweets = new UserWithDataToSync + { + Tweets = new[] + { + tweet1, + tweet2 + }, + Followers = new[] + { + follower1 + }, + User = user + }; + + var loggerMock = new Mock>(); + #endregion + + #region Mocks + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.UpdateTwitterUserAsync( + It.Is(y => y == user.Id), + It.Is(y => y == tweet2.Id), + It.Is(y => y == tweet2.Id), + It.Is(y => y == 0), + It.IsAny(), + It.Is(y => y == null), + It.Is(y => y == null), + It.Is(y => y == false) + )) + .Throws(new ArgumentException()); + + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); + #endregion + + var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object, removeTwitterAccountActionMock.Object); + await processor.ProcessAsync(usersWithTweets, CancellationToken.None); + + #region Validations + twitterUserDalMock.VerifyAll(); + loggerMock.VerifyAll(); + removeTwitterAccountActionMock.VerifyAll(); #endregion } @@ -132,19 +208,25 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == tweet3.Id), It.Is(y => y == tweet2.Id), It.Is(y => y == 0), - It.IsAny() + It.IsAny(), + It.Is(y => y == null), + It.Is(y => y == null), + It.Is(y => y == false) )) .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object); + var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object, removeTwitterAccountActionMock.Object); await processor.ProcessAsync(usersWithTweets, CancellationToken.None); #region Validations twitterUserDalMock.VerifyAll(); loggerMock.VerifyAll(); + removeTwitterAccountActionMock.VerifyAll(); #endregion } @@ -208,20 +290,130 @@ namespace BirdsiteLive.Pipeline.Tests.Processors It.Is(y => y == tweet3.Id), It.Is(y => y == tweet2.Id), It.Is(y => y == 0), - It.IsAny() + It.IsAny(), + It.Is(y => y == null), + It.Is(y => y == null), + It.Is(y => y == false) )) .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object); + var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object, removeTwitterAccountActionMock.Object); await processor.ProcessAsync(usersWithTweets, CancellationToken.None); #region Validations twitterUserDalMock.VerifyAll(); loggerMock.VerifyAll(); + removeTwitterAccountActionMock.VerifyAll(); #endregion } + + [TestMethod] + public async Task ProcessAsync_NoTweets_Test() + { + #region Stubs + var user = new SyncTwitterUser + { + Id = 1, + LastTweetPostedId = 42, + LastSync = DateTime.UtcNow.AddDays(-3) + }; + var follower1 = new Follower + { + FollowingsSyncStatus = new Dictionary + { + {1, 37} + } + }; + + var usersWithTweets = new UserWithDataToSync + { + Tweets = Array.Empty(), + Followers = new[] + { + follower1 + }, + User = user + }; + + var loggerMock = new Mock>(); + #endregion + + #region Mocks + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.UpdateTwitterUserAsync( + It.Is(y => y.LastTweetPostedId == 42 + && y.LastSync > DateTime.UtcNow.AddDays(-1)) + )) + .Returns(Task.CompletedTask); + + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); + #endregion + + var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object, removeTwitterAccountActionMock.Object); + await processor.ProcessAsync(usersWithTweets, CancellationToken.None); + + #region Validations + twitterUserDalMock.VerifyAll(); + loggerMock.VerifyAll(); + removeTwitterAccountActionMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_NoFollower_Test() + { + #region Stubs + var user = new SyncTwitterUser + { + Id = 1 + }; + var tweet1 = new ExtractedTweet + { + Id = 36 + }; + var tweet2 = new ExtractedTweet + { + Id = 37 + }; + + var usersWithTweets = new UserWithDataToSync + { + Tweets = new[] + { + tweet1, + tweet2 + }, + Followers = Array.Empty(), + User = user + }; + + var loggerMock = new Mock>(); + #endregion + + #region Mocks + var twitterUserDalMock = new Mock(MockBehavior.Strict); + + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); + removeTwitterAccountActionMock + .Setup(x => x.ProcessAsync(It.Is(y => y.Id == user.Id))) + .Returns(Task.CompletedTask); + #endregion + + var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object, removeTwitterAccountActionMock.Object); + await processor.ProcessAsync(usersWithTweets, CancellationToken.None); + + #region Validations + twitterUserDalMock.VerifyAll(); + loggerMock.VerifyAll(); + removeTwitterAccountActionMock.VerifyAll(); + #endregion + } + } } \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs index 53aa12a..8a78038 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs @@ -2,8 +2,10 @@ using System.Threading; using System.Threading.Tasks; using System.Xml; +using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; +using BirdsiteLive.Moderation.Actions; using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Pipeline.Processors; using BirdsiteLive.Pipeline.Processors.SubTasks; @@ -72,17 +74,22 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); var followersDalMock = new Mock(MockBehavior.Strict); - + var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -147,15 +154,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var followersDalMock = new Mock(MockBehavior.Strict); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -229,15 +241,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -312,15 +329,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -400,15 +422,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -471,15 +498,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var followersDalMock = new Mock(MockBehavior.Strict); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -543,15 +575,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors var followersDalMock = new Mock(MockBehavior.Strict); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -623,15 +660,196 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_MultiInstances_Inbox_OneTweet_Error_SettingsThreshold_Test() + { + #region Stubs + var tweetId = 1; + var host1 = "domain1.ext"; + var host2 = "domain2.ext"; + var inbox = "/user/inbox"; + var userId1 = 2; + var userId2 = 3; + var userAcct = "user"; + + var userWithTweets = new UserWithDataToSync() + { + Tweets = new[] + { + new ExtractedTweet + { + Id = tweetId + } + }, + User = new SyncTwitterUser + { + Acct = userAcct + }, + Followers = new[] + { + new Follower + { + Id = userId1, + Host = host1, + InboxRoute = inbox + }, + new Follower + { + Id = userId2, + Host = host2, + InboxRoute = inbox, + PostingErrorCount = 42 + }, + } + }; + #endregion + + #region Mocks + var sendTweetsToInboxTaskMock = new Mock(MockBehavior.Strict); + sendTweetsToInboxTaskMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Length == 1), + It.Is(y => y.Id == userId1), + It.Is(y => y.Acct == userAcct))) + .Returns(Task.CompletedTask); + + sendTweetsToInboxTaskMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Length == 1), + It.Is(y => y.Id == userId2), + It.Is(y => y.Acct == userAcct))) + .Throws(new Exception()); + + var sendTweetsToSharedInboxTaskMock = new Mock(MockBehavior.Strict); + + var followersDalMock = new Mock(MockBehavior.Strict); + + var loggerMock = new Mock>(); + + var settings = new InstanceSettings + { + FailingFollowerCleanUpThreshold = 10 + }; + + var removeFollowerMock = new Mock(MockBehavior.Strict); + removeFollowerMock + .Setup(x => x.ProcessAsync(It.Is(y => y.Id == userId2))) + .Returns(Task.CompletedTask); + #endregion + + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); + var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); + + #region Validations + sendTweetsToInboxTaskMock.VerifyAll(); + sendTweetsToSharedInboxTaskMock.VerifyAll(); + followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_MultiInstances_Inbox_OneTweet_Error_MaxThreshold_Test() + { + #region Stubs + var tweetId = 1; + var host1 = "domain1.ext"; + var host2 = "domain2.ext"; + var inbox = "/user/inbox"; + var userId1 = 2; + var userId2 = 3; + var userAcct = "user"; + + var userWithTweets = new UserWithDataToSync() + { + Tweets = new[] + { + new ExtractedTweet + { + Id = tweetId + } + }, + User = new SyncTwitterUser + { + Acct = userAcct + }, + Followers = new[] + { + new Follower + { + Id = userId1, + Host = host1, + InboxRoute = inbox + }, + new Follower + { + Id = userId2, + Host = host2, + InboxRoute = inbox, + PostingErrorCount = 2147483600 + }, + } + }; + #endregion + + #region Mocks + var sendTweetsToInboxTaskMock = new Mock(MockBehavior.Strict); + sendTweetsToInboxTaskMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Length == 1), + It.Is(y => y.Id == userId1), + It.Is(y => y.Acct == userAcct))) + .Returns(Task.CompletedTask); + + sendTweetsToInboxTaskMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Length == 1), + It.Is(y => y.Id == userId2), + It.Is(y => y.Acct == userAcct))) + .Throws(new Exception()); + + var sendTweetsToSharedInboxTaskMock = new Mock(MockBehavior.Strict); + + var followersDalMock = new Mock(MockBehavior.Strict); + + var loggerMock = new Mock>(); + + var settings = new InstanceSettings + { + FailingFollowerCleanUpThreshold = 0 + }; + + var removeFollowerMock = new Mock(MockBehavior.Strict); + removeFollowerMock + .Setup(x => x.ProcessAsync(It.Is(y => y.Id == userId2))) + .Returns(Task.CompletedTask); + #endregion + + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); + var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); + + #region Validations + sendTweetsToInboxTaskMock.VerifyAll(); + sendTweetsToSharedInboxTaskMock.VerifyAll(); + followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -704,15 +922,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } @@ -790,15 +1013,20 @@ namespace BirdsiteLive.Pipeline.Tests.Processors .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var settings = new InstanceSettings(); + + var removeFollowerMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object); + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, followersDalMock.Object, loggerMock.Object, settings, removeFollowerMock.Object); var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); #region Validations sendTweetsToInboxTaskMock.VerifyAll(); sendTweetsToSharedInboxTaskMock.VerifyAll(); followersDalMock.VerifyAll(); + removeFollowerMock.VerifyAll(); #endregion } }