feat(db): Purge user when no one follows them

Signed-off-by: Sam Therapy <sam@samtherapy.net>

VARIABLES.md

@@ -53,9 +53,10 @@ If both whitelisting and blacklisting are set, only the whitelisting will be act
 * `Instance:ShowAboutInstanceOnProfiles` (default: true) show "About [instance name]" on profiles with a link to /About
 * `Instance:MaxFollowsPerUser` (default: 0 - no limit) limit the number of follows per user - any follow count above this number will be Rejected
 * `Instance:DiscloseInstanceRestrictions` (default: false) disclose your instance's restrictions on its About page
-* `Instance:UnlistedTwitterAccounts` (default: null) to enable unlisted publication for selected twitter accounts, separated by `;` (please limit this to brands and other public profiles). 
-* `Instance:SensitiveTwitterAccounts` (default: null) mark all media from given accounts as sensitive by default, separated by `;`. 
+* `Instance:UnlistedTwitterAccounts` (default: null) to enable unlisted publication for selected twitter accounts, separated by `;` (please limit this to brands and other public profiles).
+* `Instance:SensitiveTwitterAccounts` (default: null) mark all media from given accounts as sensitive by default, separated by `;`.
 * `Instance:FailingTwitterUserCleanUpThreshold` (default: 700) set the max allowed errors (due to a banned/deleted/private account) from a Twitter Account retrieval before auto-removal. (by default an account is called every 15 mins)
+* `Instance:FailingFollowerCleanUpThreshold` (default: 30000) set the max allowed errors from a Follower (Fediverse) Account before auto-removal. (often due to account suppression, instance issues, etc)
 * `Instance:MaxStatusFetchAge` (default: 0 - no limit) statuses with a Snowflake older than this age in days will not be fetched by the service and will instead return 410 Gone
 * `Instance:EnableQuoteRT` (default: false) enable Soapbox-style quote-RTs
 

docker-compose.yml

@@ -6,7 +6,7 @@ networks:
 
 services:
     server:
-        image: pasture/birdsitelive:latest
+        image: git.froth.zone/birdsitelive:latest
         restart: always
         container_name: birdsitelive
         environment:
@@ -27,7 +27,7 @@ services:
             - db
 
     db:
-        image: postgres:13
+        image: postgres:15
         restart: always
         environment:
             - POSTGRES_USER=birdsitelive

src/BSLManager/BSLManager.csproj

@@ -10,7 +10,7 @@
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="5.0.0" />
-    <PackageReference Include="Terminal.Gui" Version="1.0.0-beta.11" />
+    <PackageReference Include="Terminal.Gui" Version="1.8.0" />
   </ItemGroup>
 
   <ItemGroup>

src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs

@@ -4,81 +4,84 @@ using System.Threading;
 using System.Threading.Tasks;
 
 using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.Moderation.Actions;
 using BirdsiteLive.Pipeline.Contracts;
 using BirdsiteLive.Pipeline.Models;
 
 using Microsoft.Extensions.Logging;
 
-namespace BirdsiteLive.Pipeline.Processors
+namespace BirdsiteLive.Pipeline.Processors;
+
+public class SaveProgressionProcessor : ISaveProgressionProcessor
 {
-    public class SaveProgressionProcessor : ISaveProgressionProcessor
+    private readonly ITwitterUserDal _twitterUserDal;
+    private readonly ILogger<SaveProgressionProcessor> _logger;
+    private readonly IRemoveTwitterAccountAction _removeTwitterAccountAction;
+
+    #region Ctor
+    public SaveProgressionProcessor(
+        ITwitterUserDal twitterUserDal,
+        ILogger<SaveProgressionProcessor> logger,
+        IRemoveTwitterAccountAction removeTwitterAccountAction
+    )
     {
-        private readonly ITwitterUserDal _twitterUserDal;
-        private readonly ILogger<SaveProgressionProcessor> _logger;
+        _twitterUserDal = twitterUserDal;
+        _logger = logger;
+        _removeTwitterAccountAction = removeTwitterAccountAction;
+    }
+    #endregion
 
-        #region Ctor
-        public SaveProgressionProcessor(
-            ITwitterUserDal twitterUserDal,
-            ILogger<SaveProgressionProcessor> logger
-        )
+    public async Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
+    {
+        try
         {
-            _twitterUserDal = twitterUserDal;
-            _logger = logger;
-        }
-        #endregion
-
-        public async Task ProcessAsync(
-            UserWithDataToSync userWithTweetsToSync,
-            CancellationToken ct
-        )
-        {
-            try
+            if (userWithTweetsToSync.Tweets.Length == 0)
             {
-                if (userWithTweetsToSync.Tweets.Length == 0)
-                {
-                    _logger.LogWarning("No tweets synchronized");
-                    return;
-                }
-                if (userWithTweetsToSync.Followers.Length == 0)
-                {
-                    _logger.LogWarning(
-                        "No Followers found for {User}",
-                        userWithTweetsToSync.User.Acct
-                    );
-                    return;
-                }
-
-                var userId = userWithTweetsToSync.User.Id;
-                var followingSyncStatuses = userWithTweetsToSync.Followers
-                    .Select(x => x.FollowingsSyncStatus[userId])
-                    .ToList();
-
-                if (followingSyncStatuses.Count == 0)
-                {
-                    _logger.LogWarning(
-                        "No Followers sync found for {User}, Id: {UserId}",
-                        userWithTweetsToSync.User.Acct,
-                        userId
-                    );
-                    return;
-                }
-
-                var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();
-                var minimumSync = followingSyncStatuses.Min();
-                var now = DateTime.UtcNow;
-                await _twitterUserDal.UpdateTwitterUserAsync(
-                    userId,
-                    lastPostedTweet,
-                    minimumSync,
-                    userWithTweetsToSync.User.FetchingErrorCount,
-                    now
+                _logger.LogWarning("No tweets synchronized");
+                return;
+            }
+            if (userWithTweetsToSync.Followers.Length == 0)
+            {
+                _logger.LogWarning(
+                    "No Followers found for {User}, purging them",
+                    userWithTweetsToSync.User.Acct
                 );
+                await _removeTwitterAccountAction.ProcessAsync(userWithTweetsToSync.User);
+                _logger.LogInformation("Account {User}, purged", userWithTweetsToSync.User.Acct);
+
+                return;
             }
-            catch (Exception e)
+
+            var userId = userWithTweetsToSync.User.Id;
+            var followingSyncStatuses = userWithTweetsToSync.Followers
+                .Select(x => x.FollowingsSyncStatus[userId])
+                .ToList();
+
+            if (followingSyncStatuses.Count == 0)
             {
-                _logger.LogError(e, "SaveProgressionProcessor.ProcessAsync() Exception");
-                throw;
+                _logger.LogWarning(
+                    "No Followers sync found for {User}, Id: {UserId}",
+                    userWithTweetsToSync.User.Acct,
+                    userId
+                );
+                return;
             }
+
+            var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();
+            var minimumSync = followingSyncStatuses.Min();
+            var now = DateTime.UtcNow;
+            await _twitterUserDal.UpdateTwitterUserAsync(
+                userId,
+                lastPostedTweet,
+                minimumSync,
+                userWithTweetsToSync.User.FetchingErrorCount,
+                now
+            );
+        }
+        catch (Exception e)
+        {
+            _logger.LogError(e, "SaveProgressionProcessor.ProcessAsync() Exception");
+            throw;
         }
     }
 }

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs

@@ -5,12 +5,11 @@ using System.Threading.Tasks;
 
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Moderation.Actions;
 using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Pipeline.Processors;
 using BirdsiteLive.Twitter.Models;
 
-using Castle.DynamicProxy.Contributors;
-
 using Microsoft.Extensions.Logging;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 
@@ -39,8 +38,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 Followers = new[] { follower1 },
                 User = user
             };
-
-            var loggerMock = new Mock<ILogger<SaveProgressionProcessor>>();
             #endregion
 
             #region Mocks
@@ -57,11 +54,16 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                         )
                 )
                 .Returns(Task.CompletedTask);
+            var loggerMock = new Mock<ILogger<SaveProgressionProcessor>>();
+            var removeTwitterAccountActionMock = new Mock<IRemoveTwitterAccountAction>(
+                MockBehavior.Strict
+            );
             #endregion
 
             var processor = new SaveProgressionProcessor(
                 twitterUserDalMock.Object,
-                loggerMock.Object
+                loggerMock.Object,
+                removeTwitterAccountActionMock.Object
             );
             await processor.ProcessAsync(usersWithTweets, CancellationToken.None);
 
@@ -108,11 +110,15 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 .Returns(Task.CompletedTask);
 
             var loggerMock = new Mock<ILogger<SaveProgressionProcessor>>();
+            var removeTwitterAccountActionMock = new Mock<IRemoveTwitterAccountAction>(
+                MockBehavior.Strict
+            );
             #endregion
 
             var processor = new SaveProgressionProcessor(
                 twitterUserDalMock.Object,
-                loggerMock.Object
+                loggerMock.Object,
+                removeTwitterAccountActionMock.Object
             );
             await processor.ProcessAsync(usersWithTweets, CancellationToken.None);
 
@@ -163,11 +169,15 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 .Returns(Task.CompletedTask);
 
             var loggerMock = new Mock<ILogger<SaveProgressionProcessor>>();
+            var removeTwitterAccountActionMock = new Mock<IRemoveTwitterAccountAction>(
+                MockBehavior.Strict
+            );
             #endregion
 
             var processor = new SaveProgressionProcessor(
                 twitterUserDalMock.Object,
-                loggerMock.Object
+                loggerMock.Object,
+                removeTwitterAccountActionMock.Object
             );
             await processor.ProcessAsync(usersWithTweets, CancellationToken.None);