format things, I think
continuous-integration/drone Build is failing Details

also prepare pipelines and stuff

Signed-off-by: Sam Therapy <sam@samtherapy.net>
This commit is contained in:
Sam Therapy 2022-11-15 17:25:54 +01:00
parent b3baf6e998
commit e2ec4a5857
Signed by: sam
GPG Key ID: 4D8B07C18F31ACBD
216 changed files with 4832 additions and 2685 deletions

65
.drone.yml Normal file
View File

@ -0,0 +1,65 @@
kind: pipeline
name: testing
type: docker
steps:
- name: Install Dependencies
image: mcr.microsoft.com/dotnet/sdk:6.0
commands:
- dotnet restore ./src
- name: Build
image: mcr.microsoft.com/dotnet/sdk:6.0
commands:
- dotnet build --configuration Release --no-restore ./src
- name: Test
image: mcr.microsoft.com/dotnet/sdk:6.0
commands:
- sed -i "s/127\.0\.0\.1/database/g" ./src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/Base/PostgresTestingBase.cs
- dotnet test --no-restore --verbosity minimal ./src
services:
- name: database
image: postgres:15
environment:
POSTGRES_USER: birdtest
POSTGRES_PASSWORD: mysecretpassword
POSTGRES_DB: birdsitetest
---
kind: pipeline
name: docker-publish
type: docker
depends_on:
- testing
steps:
- name: Build
image: plugins/docker
settings:
repo: git.froth.zone/sam/BirdsiteLIVE
dry_run: true
when:
event:
- pull_request
depends_on:
- "clone"
- name: Build & Publish
image: plugins/docker
settings:
registry: git.froth.zone
username: sam
password:
from_secret: password
repo: git.froth.zone/sam/BirdsiteLIVE
tags: latest
when:
branch:
- master
event:
- push
depends_on:
- "clone"

248
.editorconfig Normal file
View File

@ -0,0 +1,248 @@
# Remove the line below if you want to inherit .editorconfig settings from higher directories
root = true
# C# files
[*.cs]
#### Core EditorConfig Options ####
# Indentation and spacing
indent_size = 4
indent_style = space
tab_width = 4
# New line preferences
end_of_line = lf
insert_final_newline = true
#### .NET Coding Conventions ####
# Organize usings
dotnet_separate_import_directive_groups = true
dotnet_sort_system_directives_first = true
file_header_template = unset
# this. and Me. preferences
dotnet_style_qualification_for_event = false
dotnet_style_qualification_for_field = false
dotnet_style_qualification_for_method = false
dotnet_style_qualification_for_property = false
# Language keywords vs BCL types preferences
dotnet_style_predefined_type_for_locals_parameters_members = true
dotnet_style_predefined_type_for_member_access = true
# Parentheses preferences
dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity
dotnet_style_parentheses_in_other_binary_operators = always_for_clarity
dotnet_style_parentheses_in_other_operators = never_if_unnecessary
dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity
# Modifier preferences
dotnet_style_require_accessibility_modifiers = for_non_interface_members
# Expression-level preferences
dotnet_style_coalesce_expression = true
dotnet_style_collection_initializer = true
dotnet_style_explicit_tuple_names = true
dotnet_style_namespace_match_folder = false
dotnet_style_null_propagation = true
dotnet_style_object_initializer = true
dotnet_style_operator_placement_when_wrapping = beginning_of_line
dotnet_style_prefer_auto_properties = true
dotnet_style_prefer_compound_assignment = true
dotnet_style_prefer_conditional_expression_over_assignment = true
dotnet_style_prefer_conditional_expression_over_return = true
dotnet_style_prefer_foreach_explicit_cast_in_source = when_strongly_typed
dotnet_style_prefer_inferred_anonymous_type_member_names = true
dotnet_style_prefer_inferred_tuple_names = true
dotnet_style_prefer_is_null_check_over_reference_equality_method = true
dotnet_style_prefer_simplified_boolean_expressions = true
dotnet_style_prefer_simplified_interpolation = true
# Field preferences
dotnet_style_readonly_field = true
# Parameter preferences
dotnet_code_quality_unused_parameters = all
# Suppression preferences
dotnet_remove_unnecessary_suppression_exclusions = none
# New line preferences
dotnet_style_allow_multiple_blank_lines_experimental = false
dotnet_style_allow_statement_immediately_after_block_experimental = false
#### C# Coding Conventions ####
# var preferences
csharp_style_var_elsewhere = true
csharp_style_var_for_built_in_types = false
csharp_style_var_when_type_is_apparent = true
# Expression-bodied members
csharp_style_expression_bodied_accessors = true:silent
csharp_style_expression_bodied_constructors = when_on_single_line:silent
csharp_style_expression_bodied_indexers = true:silent
csharp_style_expression_bodied_lambdas = true:silent
csharp_style_expression_bodied_local_functions = when_on_single_line:silent
csharp_style_expression_bodied_methods = when_on_single_line:silent
csharp_style_expression_bodied_operators = when_on_single_line:silent
csharp_style_expression_bodied_properties = true:silent
# Pattern matching preferences
csharp_style_pattern_matching_over_as_with_null_check = true
csharp_style_pattern_matching_over_is_with_cast_check = true
csharp_style_prefer_extended_property_pattern = true
csharp_style_prefer_not_pattern = true
csharp_style_prefer_pattern_matching = true
csharp_style_prefer_switch_expression = true:suggestion
# Null-checking preferences
csharp_style_conditional_delegate_call = true
# Modifier preferences
csharp_prefer_static_local_function = true
csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,required,volatile,async
# Code-block preferences
csharp_prefer_braces = when_multiline:silent
csharp_prefer_simple_using_statement = true:suggestion
csharp_style_namespace_declarations = file_scoped:suggestion
csharp_style_prefer_method_group_conversion = true:silent
csharp_style_prefer_top_level_statements = true:suggestion
# Expression-level preferences
csharp_prefer_simple_default_expression = true:suggestion
csharp_style_deconstructed_variable_declaration = true:suggestion
csharp_style_implicit_object_creation_when_type_is_apparent = true:suggestion
csharp_style_inlined_variable_declaration = true:suggestion
csharp_style_prefer_index_operator = true:suggestion
csharp_style_prefer_local_over_anonymous_function = true:suggestion
csharp_style_prefer_null_check_over_type_check = true:suggestion
csharp_style_prefer_range_operator = true:suggestion
csharp_style_prefer_tuple_swap = true:suggestion
csharp_style_prefer_utf8_string_literals = true:suggestion
csharp_style_throw_expression = true:suggestion
csharp_style_unused_value_assignment_preference = discard_variable:suggestion
csharp_style_unused_value_expression_statement_preference = discard_variable:silent
# 'using' directive preferences
csharp_using_directive_placement = outside_namespace:silent
# New line preferences
csharp_style_allow_blank_line_after_colon_in_constructor_initializer_experimental = false
csharp_style_allow_blank_lines_between_consecutive_braces_experimental = false
csharp_style_allow_embedded_statements_on_same_line_experimental = true
#### C# Formatting Rules ####
# New line preferences
csharp_new_line_before_catch = true
csharp_new_line_before_else = true
csharp_new_line_before_finally = true
csharp_new_line_before_members_in_anonymous_types = true
csharp_new_line_before_members_in_object_initializers = true
csharp_new_line_before_open_brace = all
csharp_new_line_between_query_expression_clauses = true
# Indentation preferences
csharp_indent_block_contents = true
csharp_indent_braces = false
csharp_indent_case_contents = true
csharp_indent_case_contents_when_block = true
csharp_indent_labels = one_less_than_current
csharp_indent_switch_labels = true
# Space preferences
csharp_space_after_cast = false
csharp_space_after_colon_in_inheritance_clause = true
csharp_space_after_comma = true
csharp_space_after_dot = false
csharp_space_after_keywords_in_control_flow_statements = true
csharp_space_after_semicolon_in_for_statement = true
csharp_space_around_binary_operators = before_and_after
csharp_space_around_declaration_statements = false
csharp_space_before_colon_in_inheritance_clause = true
csharp_space_before_comma = false
csharp_space_before_dot = false
csharp_space_before_open_square_brackets = false
csharp_space_before_semicolon_in_for_statement = false
csharp_space_between_empty_square_brackets = false
csharp_space_between_method_call_empty_parameter_list_parentheses = false
csharp_space_between_method_call_name_and_opening_parenthesis = false
csharp_space_between_method_call_parameter_list_parentheses = false
csharp_space_between_method_declaration_empty_parameter_list_parentheses = false
csharp_space_between_method_declaration_name_and_open_parenthesis = false
csharp_space_between_method_declaration_parameter_list_parentheses = false
csharp_space_between_parentheses = false
csharp_space_between_square_brackets = false
# Wrapping preferences
csharp_preserve_single_line_blocks = true
csharp_preserve_single_line_statements = true
#### Naming styles ####
# Naming rules
dotnet_naming_rule.interface_should_be_begins_with_i.severity = suggestion
dotnet_naming_rule.interface_should_be_begins_with_i.symbols = interface
dotnet_naming_rule.interface_should_be_begins_with_i.style = begins_with_i
dotnet_naming_rule.types_should_be_pascal_case.severity = suggestion
dotnet_naming_rule.types_should_be_pascal_case.symbols = types
dotnet_naming_rule.types_should_be_pascal_case.style = pascal_case
dotnet_naming_rule.non_field_members_should_be_pascal_case.severity = suggestion
dotnet_naming_rule.non_field_members_should_be_pascal_case.symbols = non_field_members
dotnet_naming_rule.non_field_members_should_be_pascal_case.style = pascal_case
# Symbol specifications
dotnet_naming_symbols.interface.applicable_kinds = interface
dotnet_naming_symbols.interface.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected
dotnet_naming_symbols.interface.required_modifiers =
dotnet_naming_symbols.types.applicable_kinds = class, struct, interface, enum
dotnet_naming_symbols.types.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected
dotnet_naming_symbols.types.required_modifiers =
dotnet_naming_symbols.non_field_members.applicable_kinds = property, event, method
dotnet_naming_symbols.non_field_members.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected
dotnet_naming_symbols.non_field_members.required_modifiers =
# Naming styles
dotnet_naming_style.pascal_case.required_prefix =
dotnet_naming_style.pascal_case.required_suffix =
dotnet_naming_style.pascal_case.word_separator =
dotnet_naming_style.pascal_case.capitalization = pascal_case
dotnet_naming_style.begins_with_i.required_prefix = I
dotnet_naming_style.begins_with_i.required_suffix =
dotnet_naming_style.begins_with_i.word_separator =
dotnet_naming_style.begins_with_i.capitalization = pascal_case
dotnet_diagnostic.IDE0004.severity = suggestion
dotnet_diagnostic.IDE0005.severity = suggestion
[*.{cs,vb}]
dotnet_style_coalesce_expression = true:suggestion
dotnet_style_null_propagation = true:suggestion
dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion
dotnet_style_prefer_auto_properties = true:silent
dotnet_style_object_initializer = true:suggestion
dotnet_style_collection_initializer = true:suggestion
dotnet_style_prefer_simplified_boolean_expressions = true:suggestion
dotnet_style_operator_placement_when_wrapping = beginning_of_line
tab_width = 4
indent_size = 4
end_of_line = crlf
dotnet_style_prefer_conditional_expression_over_assignment = true:silent
dotnet_style_namespace_match_folder = false:silent
dotnet_style_explicit_tuple_names = true:suggestion
dotnet_style_prefer_conditional_expression_over_return = true:silent
dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion
dotnet_style_prefer_inferred_tuple_names = true:suggestion
dotnet_style_prefer_simplified_interpolation = true:suggestion
dotnet_style_prefer_compound_assignment = true:suggestion

1
.github/FUNDING.yml vendored
View File

@ -1 +0,0 @@
patreon: nicolasconstant

View File

@ -12,7 +12,7 @@ jobs:
steps:
- uses: actions/checkout@v2
- name: Launch Db for testing
run: docker run --name postgres -e POSTGRES_DB=mytestdb -e POSTGRES_PASSWORD=mysecretpassword -d -p 5432:5432 postgres
run: docker run --name postgres -e POSTGRES_USER=birdtest -e POSTGRES_DB=birdsitetest -e POSTGRES_PASSWORD=mysecretpassword -d -p 5432:5432 postgres
- name: Setup .NET Core
uses: actions/setup-dotnet@v1
with:

2
.gitignore vendored
View File

@ -352,3 +352,5 @@ MigrationBackup/
# Ionide (cross platform F# VS Code tools) working folder
.ionide/
/src/BSLManager/Properties/launchSettings.json
.dccache

View File

@ -1,14 +1,15 @@
#See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging.
FROM mcr.microsoft.com/dotnet/aspnet:3.1-buster-slim AS base
FROM mcr.microsoft.com/dotnet/aspnet:6.0-slim AS base
WORKDIR /app
EXPOSE 80
EXPOSE 443
FROM mcr.microsoft.com/dotnet/sdk:3.1-buster AS publish
FROM mcr.microsoft.com/dotnet/sdk:6.0 AS publish
WORKDIR /
COPY ./src/ ./src/
RUN dotnet publish "/src/BirdsiteLive/BirdsiteLive.csproj" -c Release -o /app/publish
RUN dotnet publish "/src/BSLManager/BSLManager.csproj" -r linux-x64 --self-contained true -p:PublishSingleFile=true -p:IncludeAllContentForSelfExtract=true -c Release -o /app/publish
RUN dotnet publish "/src/BirdsiteLive/BirdsiteLive.csproj" -c Release -o /app/publish \
&& dotnet publish "/src/BSLManager/BSLManager.csproj" -r linux-x64 --self-contained true -p:PublishSingleFile=true -p:IncludeAllContentForSelfExtract=true -c Release -o /app/publish
FROM base AS final
WORKDIR /app

View File

@ -1,13 +1,16 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Moderation.Actions;
using BSLManager.Domain;
using BSLManager.Tools;
using Terminal.Gui;
namespace BSLManager
@ -37,7 +40,6 @@ namespace BSLManager
{
X = 0,
Y = 1, // Leave one row for the toplevel menu
// By using Dim.Fill(), it will automatically resize without manual intervention
Width = Dim.Fill(),
Height = Dim.Fill()
@ -46,29 +48,46 @@ namespace BSLManager
top.Add(win);
// Creates a menubar, the item "New" has a help menu.
var menu = new MenuBar(new MenuBarItem[]
{
new MenuBarItem("_File", new MenuItem[]
var menu = new MenuBar(
new MenuBarItem[]
{
new MenuItem("_Quit", "", () =>
{
if (Quit()) top.Running = false;
})
}),
//new MenuBarItem ("_Edit", new MenuItem [] {
// new MenuItem ("_Copy", "", null),
// new MenuItem ("C_ut", "", null),
// new MenuItem ("_Paste", "", null)
//})
});
new MenuBarItem(
"_File",
new MenuItem[]
{
new MenuItem(
"_Quit",
"",
() =>
{
if (Quit())
top.Running = false;
}
)
}
),
//new MenuBarItem ("_Edit", new MenuItem [] {
// new MenuItem ("_Copy", "", null),
// new MenuItem ("C_ut", "", null),
// new MenuItem ("_Paste", "", null)
//})
}
);
top.Add(menu);
static bool Quit()
{
var n = MessageBox.Query(50, 7, "Quit BSL Manager", "Are you sure you want to quit?", "Yes", "No");
var n = MessageBox.Query(
50,
7,
"Quit BSL Manager",
"Are you sure you want to quit?",
"Yes",
"No"
);
return n == 0;
}
RetrieveUserList();
var list = new ListView(_state.GetDisplayableList())
@ -84,11 +103,13 @@ namespace BSLManager
if (_.KeyEvent.Key == Key.Enter)
{
OpenFollowerDialog(list.SelectedItem);
}
else if (_.KeyEvent.Key == Key.Delete
|| _.KeyEvent.Key == Key.DeleteChar
|| _.KeyEvent.Key == Key.Backspace
|| _.KeyEvent.Key == Key.D)
}
else if (
_.KeyEvent.Key == Key.Delete
|| _.KeyEvent.Key == Key.DeleteChar
|| _.KeyEvent.Key == Key.Backspace
|| _.KeyEvent.Key == Key.D
)
{
OpenDeleteDialog(list.SelectedItem);
}
@ -113,12 +134,7 @@ namespace BSLManager
}
};
win.Add(
listingFollowersLabel,
filterLabel,
filterText,
list
);
win.Add(listingFollowersLabel, filterLabel, filterText, list);
Application.Run();
}
@ -224,7 +240,7 @@ namespace BSLManager
try
{
var userToDelete = _state.GetElementAt(el);
BasicLogger.Log($"Delete {userToDelete.Acct}@{userToDelete.Host}");
await _removeFollowerAction.ProcessAsync(userToDelete);
BasicLogger.Log($"Remove user from list");
@ -249,4 +265,4 @@ namespace BSLManager
});
}
}
}
}

View File

@ -1,8 +1,8 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
<TargetFramework>net6.0</TargetFramework>
</PropertyGroup>
<ItemGroup>

View File

@ -1,12 +1,15 @@
using System;
using System;
using System.Net.Http;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.Common.Structs;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Postgres.DataAccessLayers;
using BirdsiteLive.DAL.Postgres.Settings;
using Lamar;
using Lamar.Scanning.Conventions;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
@ -33,13 +36,17 @@ namespace BSLManager
x.For<InstanceSettings>().Use(x => _instanceSettings);
if (string.Equals(_dbSettings.Type, DbTypes.Postgres, StringComparison.OrdinalIgnoreCase))
if (
string.Equals(
_dbSettings.Type,
DbTypes.Postgres,
StringComparison.OrdinalIgnoreCase
)
)
{
var connString = $"Host={_dbSettings.Host};Username={_dbSettings.User};Password={_dbSettings.Password};Database={_dbSettings.Name}";
var postgresSettings = new PostgresSettings
{
ConnString = connString
};
var connString =
$"Host={_dbSettings.Host};Username={_dbSettings.User};Password={_dbSettings.Password};Database={_dbSettings.Name}";
var postgresSettings = new PostgresSettings { ConnString = connString };
x.For<PostgresSettings>().Use(x => postgresSettings);
x.For<ITwitterUserDal>().Use<TwitterUserPostgresDal>().Singleton();
@ -51,8 +58,11 @@ namespace BSLManager
throw new NotImplementedException($"{_dbSettings.Type} is not supported");
}
var serviceProvider = new ServiceCollection().AddHttpClient().BuildServiceProvider();
x.For<IHttpClientFactory>().Use(_ => serviceProvider.GetService<IHttpClientFactory>());
var serviceProvider = new ServiceCollection()
.AddHttpClient()
.BuildServiceProvider();
x.For<IHttpClientFactory>()
.Use(_ => serviceProvider.GetService<IHttpClientFactory>());
x.For(typeof(ILogger<>)).Use(typeof(DummyLogger<>));
@ -76,9 +86,13 @@ namespace BSLManager
public class DummyLogger<T> : ILogger<T>
{
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter)
{
}
public void Log<TState>(
LogLevel logLevel,
EventId eventId,
TState state,
Exception exception,
Func<TState, Exception, string> formatter
) { }
public bool IsEnabled(LogLevel logLevel)
{
@ -91,4 +105,4 @@ namespace BSLManager
}
}
}
}
}

View File

@ -1,5 +1,6 @@
using System.Collections.Generic;
using System.Collections.Generic;
using System.Linq;
using BirdsiteLive.DAL.Models;
namespace BSLManager.Domain
@ -10,11 +11,11 @@ namespace BSLManager.Domain
private List<Follower> _sourceUserList = new List<Follower>();
private List<Follower> _filteredSourceUserList = new List<Follower>();
public void Load(List<Follower> followers)
{
_sourceUserList = followers.OrderByDescending(x => x.Followings.Count).ToList();
ResetLists();
}
@ -26,7 +27,8 @@ namespace BSLManager.Domain
foreach (var follower in _sourceUserList)
{
var displayedUser = $"{GetFullHandle(follower)} ({follower.Followings.Count}) (err:{follower.PostingErrorCount})";
var displayedUser =
$"{GetFullHandle(follower)} ({follower.Followings.Count}) (err:{follower.PostingErrorCount})";
_filteredDisplayableUserList.Add(displayedUser);
}
}
@ -50,8 +52,10 @@ namespace BSLManager.Domain
foreach (var el in elToRemove)
{
_filteredSourceUserList.Remove(el);
var dElToRemove = _filteredDisplayableUserList.First(x => x.Contains(GetFullHandle(el)));
var dElToRemove = _filteredDisplayableUserList.First(
x => x.Contains(GetFullHandle(el))
);
_filteredDisplayableUserList.Remove(dElToRemove);
}
}
@ -78,4 +82,4 @@ namespace BSLManager.Domain
return _filteredSourceUserList[index];
}
}
}
}

View File

@ -1,19 +1,12 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System;
using System.Text;
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BSLManager.Tools;
using Microsoft.Extensions.Configuration;
using NStack;
using Terminal.Gui;
namespace BSLManager
{
class Program
internal class Program
{
static async Task Main(string[] args)
{

View File

@ -1,4 +1,4 @@
using System;
using System;
using System.IO;
namespace BSLManager.Tools
@ -7,7 +7,7 @@ namespace BSLManager.Tools
{
public static void Log(string log)
{
File.AppendAllLines($"Log-{Guid.NewGuid()}.txt", new []{ log });
File.AppendAllLines($"Log-{Guid.NewGuid()}.txt", new[] { log });
}
}
}
}

View File

@ -1,4 +1,5 @@
using System.Reflection;
using System.Reflection;
using Terminal.Gui;
namespace BSLManager.Tools
@ -12,4 +13,4 @@ namespace BSLManager.Tools
.Invoke(null, null);
}
}
}
}

View File

@ -1,8 +1,11 @@
using System;
using System;
using System.IO;
using System.Runtime.CompilerServices;
using BirdsiteLive.Common.Settings;
using Newtonsoft.Json;
using Org.BouncyCastle.Asn1.IsisMtt.X509;
namespace BSLManager.Tools
@ -14,10 +17,13 @@ namespace BSLManager.Tools
public (DbSettings dbSettings, InstanceSettings instanceSettings) GetSettings()
{
var localSettingsData = GetLocalSettingsFile();
if (localSettingsData != null) return Convert(localSettingsData);
if (localSettingsData != null)
return Convert(localSettingsData);
Console.WriteLine("We need to set up the manager");
Console.WriteLine("Please provide the following information as provided in the docker-compose file");
Console.WriteLine(
"Please provide the following information as provided in the docker-compose file"
);
LocalSettingsData data;
do
@ -39,12 +45,11 @@ namespace BSLManager.Tools
Console.WriteLine("Is it valid? (yes, no)");
resp = Console.ReadLine()?.Trim().ToLowerInvariant();
if (resp == "n" || resp == "no") data = null;
if (resp == "n" || resp == "no")
data = null;
} while (resp != "y" && resp != "yes" && resp != "n" && resp != "no");
} while (data == null);
SaveLocalSettings(data);
return Convert(data);
}
@ -71,7 +76,9 @@ namespace BSLManager.Tools
return data;
}
private (DbSettings dbSettings, InstanceSettings instanceSettings) Convert(LocalSettingsData data)
private (DbSettings dbSettings, InstanceSettings instanceSettings) Convert(
LocalSettingsData data
)
{
var dbSettings = new DbSettings
{
@ -81,10 +88,7 @@ namespace BSLManager.Tools
User = data.DbUser,
Password = data.DbPassword
};
var instancesSettings = new InstanceSettings
{
Domain = data.InstanceDomain
};
var instancesSettings = new InstanceSettings { Domain = data.InstanceDomain };
return (dbSettings, instancesSettings);
}
@ -92,7 +96,8 @@ namespace BSLManager.Tools
{
try
{
if (!File.Exists(LocalFileName)) return null;
if (!File.Exists(LocalFileName))
return null;
var jsonContent = File.ReadAllText(LocalFileName);
var content = JsonConvert.DeserializeObject<LocalSettingsData>(jsonContent);
@ -121,4 +126,4 @@ namespace BSLManager.Tools
public string InstanceDomain { get; set; }
}
}
}

View File

@ -1,4 +1,5 @@
using System;
using System;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub
@ -16,7 +17,7 @@ namespace BirdsiteLive.ActivityPub
return JsonConvert.DeserializeObject<ActivityFollow>(json);
case "Undo":
var a = JsonConvert.DeserializeObject<ActivityUndo>(json);
if(a.apObject.type == "Follow")
if (a.apObject.type == "Follow")
return JsonConvert.DeserializeObject<ActivityUndoFollow>(json);
break;
case "Accept":
@ -59,4 +60,4 @@ namespace BirdsiteLive.ActivityPub
public Activity apObject { get; set; }
}
}
}
}

View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<TargetFramework>net6.0</TargetFramework>
</PropertyGroup>
<ItemGroup>

View File

@ -1,18 +1,28 @@
using System;
using System;
using System.Collections.Generic;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub.Converters
{
public class ContextArrayConverter : JsonConverter
{
public override bool CanWrite { get { return false; } }
public override bool CanWrite
{
get { return false; }
}
public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
{
throw new NotImplementedException();
}
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
public override object ReadJson(
JsonReader reader,
Type objectType,
object existingValue,
JsonSerializer serializer
)
{
var result = new List<string>();
@ -36,4 +46,4 @@ namespace BirdsiteLive.ActivityPub.Converters
throw new NotImplementedException();
}
}
}
}

View File

@ -1,4 +1,4 @@
using System.Runtime.CompilerServices;
using System.Runtime.CompilerServices;
namespace BirdsiteLive.ActivityPub.Converters
{
@ -14,4 +14,4 @@ namespace BirdsiteLive.ActivityPub.Converters
return $"https://{domain.ToLowerInvariant()}/users/{username.ToLowerInvariant()}/statuses/{noteId}";
}
}
}
}

View File

@ -1,5 +1,6 @@
using System.Collections.Generic;
using System.Collections.Generic;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub
@ -7,7 +8,8 @@ namespace BirdsiteLive.ActivityPub
public class Activity
{
[Newtonsoft.Json.JsonIgnore]
public static readonly object[] DefaultContext = new object[] {
public static readonly object[] DefaultContext = new object[]
{
"https://www.w3.org/ns/activitystreams",
"https://w3id.org/security/v1",
new Dictionary<string, string>
@ -18,7 +20,6 @@ namespace BirdsiteLive.ActivityPub
{ "value", "schema:value" },
{ "sensitive", "as:sensitive" },
{ "quoteUrl", "as:quoteUrl" },
{ "schema", "http://schema.org#" },
{ "toot", "https://joinmastodon.org/ns#" }
}
@ -33,4 +34,4 @@ namespace BirdsiteLive.ActivityPub
//[JsonProperty("object")]
//public string apObject { get; set; }
}
}
}

View File

@ -1,4 +1,4 @@
using Newtonsoft.Json;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub
{
@ -7,4 +7,4 @@ namespace BirdsiteLive.ActivityPub
[JsonProperty("object")]
public object apObject { get; set; }
}
}
}

View File

@ -1,4 +1,4 @@
using Newtonsoft.Json;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub
{
@ -7,4 +7,4 @@ namespace BirdsiteLive.ActivityPub
[JsonProperty("object")]
public ActivityFollow apObject { get; set; }
}
}
}

View File

@ -1,4 +1,4 @@
using Newtonsoft.Json;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub
{
@ -7,4 +7,4 @@ namespace BirdsiteLive.ActivityPub
[JsonProperty("object")]
public ActivityUndoFollow apObject { get; set; }
}
}
}

View File

@ -1,7 +1,4 @@
namespace BirdsiteLive.ActivityPub
namespace BirdsiteLive.ActivityPub
{
public class ActivityCreate
{
}
}
public class ActivityCreate { }
}

View File

@ -1,5 +1,7 @@
using System;
using System;
using BirdsiteLive.ActivityPub.Models;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub
@ -13,4 +15,4 @@ namespace BirdsiteLive.ActivityPub
[JsonProperty("object")]
public Note apObject { get; set; }
}
}
}

View File

@ -1,4 +1,4 @@
using Newtonsoft.Json;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub
{
@ -7,4 +7,4 @@ namespace BirdsiteLive.ActivityPub
[JsonProperty("object")]
public string apObject { get; set; }
}
}
}

View File

@ -1,4 +1,4 @@
using Newtonsoft.Json;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub
{
@ -7,4 +7,4 @@ namespace BirdsiteLive.ActivityPub
[JsonProperty("object")]
public ActivityFollow apObject { get; set; }
}
}
}

View File

@ -1,4 +1,4 @@
using Newtonsoft.Json;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub
{
@ -7,4 +7,4 @@ namespace BirdsiteLive.ActivityPub
[JsonProperty("object")]
public Activity apObject { get; set; }
}
}
}

View File

@ -1,4 +1,4 @@
using Newtonsoft.Json;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub
{
@ -7,4 +7,4 @@ namespace BirdsiteLive.ActivityPub
[JsonProperty("object")]
public ActivityFollow apObject { get; set; }
}
}
}

View File

@ -1,8 +1,10 @@
using System;
using System;
using System.Collections.Generic;
using System.Net;
using BirdsiteLive.ActivityPub.Converters;
using BirdsiteLive.ActivityPub.Models;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub

View File

@ -1,4 +1,4 @@
namespace BirdsiteLive.ActivityPub
namespace BirdsiteLive.ActivityPub
{
public class Attachment
{
@ -6,4 +6,4 @@
public string mediaType { get; set; }
public string url { get; set; }
}
}
}

View File

@ -1,7 +1,7 @@
namespace BirdsiteLive.ActivityPub
namespace BirdsiteLive.ActivityPub
{
public class EndPoints
{
public string sharedInbox { get; set; }
}
}
}

View File

@ -1,4 +1,5 @@
using BirdsiteLive.ActivityPub.Converters;
using BirdsiteLive.ActivityPub.Converters;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub.Models
@ -12,4 +13,4 @@ namespace BirdsiteLive.ActivityPub.Models
public string id { get; set; }
public string type { get; set; } = "OrderedCollection";
}
}
}

View File

@ -1,4 +1,4 @@
namespace BirdsiteLive.ActivityPub
namespace BirdsiteLive.ActivityPub
{
public class Image
{
@ -6,4 +6,4 @@
public string mediaType { get; set; }
public string url { get; set; }
}
}
}

View File

@ -1,7 +1,9 @@
using BirdsiteLive.ActivityPub.Converters;
using Newtonsoft.Json;
using System.Collections.Generic;
using BirdsiteLive.ActivityPub.Converters;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub.Models
{
public class Note
@ -20,13 +22,16 @@ namespace BirdsiteLive.ActivityPub.Models
public string[] to { get; set; }
public string[] cc { get; set; }
public bool sensitive { get; set; }
//public string conversation { get; set; }
public string content { get; set; }
//public Dictionary<string,string> contentMap { get; set; }
public Attachment[] attachment { get; set; }
public Tag[] tag { get; set; }
//public Dictionary<string, string> replies;
public string quoteUrl { get; set; }
}
}
}

View File

@ -1,4 +1,4 @@
namespace BirdsiteLive.ActivityPub
namespace BirdsiteLive.ActivityPub
{
public class PublicKey
{
@ -6,4 +6,4 @@
public string owner { get; set; }
public string publicKeyPem { get; set; }
}
}
}

View File

@ -1,8 +1,9 @@
using System;
using System;
namespace BirdsiteLive.ActivityPub.Models
{
public class Tag {
public class Tag
{
public TagResource icon { get; set; } = null;
public string id { get; set; }
public string type { get; set; } //Hashtag
@ -16,4 +17,4 @@ namespace BirdsiteLive.ActivityPub.Models
public string type { get; set; }
public string url { get; set; }
}
}
}

View File

@ -1,4 +1,4 @@
namespace BirdsiteLive.ActivityPub
namespace BirdsiteLive.ActivityPub
{
public class UserAttachment
{
@ -6,4 +6,4 @@
public string name { get; set; }
public string value { get; set; }
}
}
}

View File

@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Text;

View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<TargetFramework>net6.0</TargetFramework>
</PropertyGroup>
</Project>

View File

@ -1,4 +1,4 @@
using System.Collections.Generic;
using System.Collections.Generic;
using System.Linq;
namespace BirdsiteLive.Common.Extensions
@ -13,4 +13,4 @@ namespace BirdsiteLive.Common.Extensions
}
}
}
}
}

File diff suppressed because one or more lines are too long

View File

@ -1,10 +1,12 @@
using System.Text.RegularExpressions;
using System.Text.RegularExpressions;
namespace BirdsiteLive.Common.Regexes
{
public class HashtagRegexes
{
public static readonly Regex HashtagName = new Regex(@"^[a-zA-Z0-9_]+$");
public static readonly Regex Hashtag = new Regex(@"(.?)#([a-zA-Z0-9_]+)(\s|$|[\[\]<>.,;:!?/|-])");
public static readonly Regex Hashtag = new Regex(
@"(.?)#([a-zA-Z0-9_]+)(\s|$|[\[\]<>.,;:!?/|-])"
);
}
}
}

View File

@ -1,4 +1,4 @@
using System.Text.RegularExpressions;
using System.Text.RegularExpressions;
namespace BirdsiteLive.Common.Regexes
{
@ -6,4 +6,4 @@ namespace BirdsiteLive.Common.Regexes
{
public static readonly Regex HeaderSignature = new Regex(@"^([a-zA-Z0-9]+)=""(.+)""$");
}
}
}

View File

@ -1,9 +1,11 @@
using System.Text.RegularExpressions;
using System.Text.RegularExpressions;
namespace BirdsiteLive.Common.Regexes
{
public class UrlRegexes
{
public static readonly Regex Url = new Regex(@"(.?)(((http|ftp|https):\/\/)[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&amp;:/~\+#]*[\w\-\@?^=%&amp;/~\+#])?)");
public static readonly Regex Url = new Regex(
@"(.?)(((http|ftp|https):\/\/)[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&amp;:/~\+#]*[\w\-\@?^=%&amp;/~\+#])?)"
);
}
}
}

View File

@ -1,10 +1,12 @@
using System.Text.RegularExpressions;
using System.Text.RegularExpressions;
namespace BirdsiteLive.Common.Regexes
{
public class UserRegexes
{
public static readonly Regex TwitterAccount = new Regex(@"^[a-zA-Z0-9_]+$");
public static readonly Regex Mention = new Regex(@"(.?)@([a-zA-Z0-9_]+)(\s|$|[\[\]<>,;:!?/|-]|(. ))");
public static readonly Regex Mention = new Regex(
@"(.?)@([a-zA-Z0-9_]+)(\s|$|[\[\]<>,;:!?/|-]|(. ))"
);
}
}
}

View File

@ -1,4 +1,4 @@
namespace BirdsiteLive.Common.Settings
namespace BirdsiteLive.Common.Settings
{
public class DbSettings
{
@ -8,4 +8,4 @@
public string User { get; set; }
public string Password { get; set; }
}
}
}

View File

@ -1,4 +1,4 @@
namespace BirdsiteLive.Common.Settings
namespace BirdsiteLive.Common.Settings
{
public class InstanceSettings
{

View File

@ -1,8 +1,8 @@
namespace BirdsiteLive.Common.Settings
namespace BirdsiteLive.Common.Settings
{
public class LogsSettings
{
public string Type { get; set; }
public string InstrumentationKey { get; set; }
}
}
}

View File

@ -1,4 +1,4 @@
namespace BirdsiteLive.Common.Settings
namespace BirdsiteLive.Common.Settings
{
public class ModerationSettings
{
@ -7,4 +7,4 @@
public string TwitterAccountsWhiteListing { get; set; }
public string TwitterAccountsBlackListing { get; set; }
}
}
}

View File

@ -1,8 +1,8 @@
namespace BirdsiteLive.Common.Settings
namespace BirdsiteLive.Common.Settings
{
public class TwitterSettings
{
public string ConsumerKey { get; set; }
public string ConsumerSecret { get; set; }
}
}
}

View File

@ -1,7 +1,7 @@
namespace BirdsiteLive.Common.Structs
namespace BirdsiteLive.Common.Structs
{
public struct DbTypes
{
public static string Postgres = "postgres";
}
}
}

View File

@ -1,8 +1,10 @@
using System.Linq;
using System.Linq;
using System.Security.Cryptography;
using Asn1;
using Asn1Sequence = Asn1.Asn1Sequence;
using Asn1Null = Asn1.Asn1Null;
using Asn1Sequence = Asn1.Asn1Sequence;
namespace BirdsiteLive.Cryptography
{
@ -46,4 +48,4 @@ namespace BirdsiteLive.Cryptography
return result.GetBytes();
}
}
}
}

View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<TargetFramework>net6.0</TargetFramework>
</PropertyGroup>
<ItemGroup>

View File

@ -1,7 +1,8 @@
using System;
using System;
using System.Collections.Generic;
using System.Security.Cryptography;
using System.Text;
using Newtonsoft.Json;
namespace BirdsiteLive.Cryptography
@ -86,7 +87,8 @@ namespace BirdsiteLive.Cryptography
else
{
_parts = key.Split('.');
if (_parts[0] != "RSA") throw new Exception("Unknown magic key!");
if (_parts[0] != "RSA")
throw new Exception("Unknown magic key!");
var rsaParams = new RSAParameters();
rsaParams.Modulus = _decodeBase64Url(_parts[1]);
@ -102,18 +104,37 @@ namespace BirdsiteLive.Cryptography
var rsa = RSA.Create();
rsa.KeySize = 2048;
return new MagicKey(JsonConvert.SerializeObject(RSAKeyParms.From(rsa.ExportParameters(true))));
return new MagicKey(
JsonConvert.SerializeObject(RSAKeyParms.From(rsa.ExportParameters(true)))
);
}
public byte[] BuildSignedData(string data, string dataType, string encoding, string algorithm)
public byte[] BuildSignedData(
string data,
string dataType,
string encoding,
string algorithm
)
{
var sig = data + "." + _encodeBase64Url(Encoding.UTF8.GetBytes(dataType)) + "." + _encodeBase64Url(Encoding.UTF8.GetBytes(encoding)) + "." + _encodeBase64Url(Encoding.UTF8.GetBytes(algorithm));
var sig =
data
+ "."
+ _encodeBase64Url(Encoding.UTF8.GetBytes(dataType))
+ "."
+ _encodeBase64Url(Encoding.UTF8.GetBytes(encoding))
+ "."
+ _encodeBase64Url(Encoding.UTF8.GetBytes(algorithm));
return Encoding.UTF8.GetBytes(sig);
}
public bool Verify(string signature, byte[] data)
{
return _rsa.VerifyData(data, _decodeBase64Url(signature), HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
return _rsa.VerifyData(
data,
_decodeBase64Url(signature),
HashAlgorithmName.SHA256,
RSASignaturePadding.Pkcs1
);
}
public byte[] Sign(byte[] data)
@ -140,7 +161,10 @@ namespace BirdsiteLive.Cryptography
public string PrivateKey
{
get { return JsonConvert.SerializeObject(RSAKeyParms.From(_rsa.ExportParameters(true))); }
get
{
return JsonConvert.SerializeObject(RSAKeyParms.From(_rsa.ExportParameters(true)));
}
}
public string PublicKey
@ -149,8 +173,13 @@ namespace BirdsiteLive.Cryptography
{
var parms = _rsa.ExportParameters(false);
return string.Join(".", "RSA", _encodeBase64Url(parms.Modulus), _encodeBase64Url(parms.Exponent));
return string.Join(
".",
"RSA",
_encodeBase64Url(parms.Modulus),
_encodeBase64Url(parms.Exponent)
);
}
}
}
}
}

View File

@ -1,4 +1,4 @@
using System;
using System;
using System.IO;
using System.Security.Cryptography;
@ -13,7 +13,7 @@ namespace BirdsiteLive.Cryptography
public string GetRsa()
{
var rsa = RSA.Create();
var outputStream = new StringWriter();
var parameters = rsa.ExportParameters(true);
using (var stream = new MemoryStream())
@ -26,7 +26,18 @@ namespace BirdsiteLive.Cryptography
innerWriter.Write((byte)0x30); // SEQUENCE
EncodeLength(innerWriter, 13);
innerWriter.Write((byte)0x06); // OBJECT IDENTIFIER
var rsaEncryptionOid = new byte[] { 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01 };
var rsaEncryptionOid = new byte[]
{
0x2a,
0x86,
0x48,
0x86,
0xf7,
0x0d,
0x01,
0x01,
0x01
};
EncodeLength(innerWriter, rsaEncryptionOid.Length);
innerWriter.Write(rsaEncryptionOid);
innerWriter.Write((byte)0x05); // NULL
@ -55,7 +66,9 @@ namespace BirdsiteLive.Cryptography
writer.Write(innerStream.GetBuffer(), 0, length);
}
var base64 = Convert.ToBase64String(stream.GetBuffer(), 0, (int)stream.Length).ToCharArray();
var base64 = Convert
.ToBase64String(stream.GetBuffer(), 0, (int)stream.Length)
.ToCharArray();
// WriteLine terminates with \r\n, we want only \n
outputStream.Write("-----BEGIN PUBLIC KEY-----\n");
for (var i = 0; i < base64.Length; i += 64)
@ -71,7 +84,8 @@ namespace BirdsiteLive.Cryptography
private static void EncodeLength(BinaryWriter stream, int length)
{
if (length < 0) throw new ArgumentOutOfRangeException("length", "Length must be non-negative");
if (length < 0)
throw new ArgumentOutOfRangeException("length", "Length must be non-negative");
if (length < 0x80)
{
// Short form
@ -94,6 +108,5 @@ namespace BirdsiteLive.Cryptography
}
}
}
}
}

View File

@ -1,11 +1,12 @@
using Org.BouncyCastle.Crypto;
using Org.BouncyCastle.Crypto.Parameters;
using Org.BouncyCastle.OpenSsl;
using Org.BouncyCastle.Security;
using System;
using System.IO;
using System.Security.Cryptography;
using Org.BouncyCastle.Crypto;
using Org.BouncyCastle.Crypto.Parameters;
using Org.BouncyCastle.OpenSsl;
using Org.BouncyCastle.Security;
namespace MyProject.Data.Encryption
{
public class RSAKeys
@ -19,9 +20,11 @@ namespace MyProject.Data.Encryption
{
PemReader pr = new PemReader(new StringReader(pem));
AsymmetricCipherKeyPair KeyPair = (AsymmetricCipherKeyPair)pr.ReadObject();
RSAParameters rsaParams = DotNetUtilities.ToRSAParameters((RsaPrivateCrtKeyParameters)KeyPair.Private);
RSAParameters rsaParams = DotNetUtilities.ToRSAParameters(
(RsaPrivateCrtKeyParameters)KeyPair.Private
);
RSACryptoServiceProvider csp = new RSACryptoServiceProvider();// cspParams);
RSACryptoServiceProvider csp = new RSACryptoServiceProvider(); // cspParams);
csp.ImportParameters(rsaParams);
return csp;
}
@ -37,7 +40,7 @@ namespace MyProject.Data.Encryption
AsymmetricKeyParameter publicKey = (AsymmetricKeyParameter)pr.ReadObject();
RSAParameters rsaParams = DotNetUtilities.ToRSAParameters((RsaKeyParameters)publicKey);
RSACryptoServiceProvider csp = new RSACryptoServiceProvider();// cspParams);
RSACryptoServiceProvider csp = new RSACryptoServiceProvider(); // cspParams);
csp.ImportParameters(rsaParams);
return csp;
}
@ -51,7 +54,8 @@ namespace MyProject.Data.Encryption
public static string ExportPrivateKey(RSACryptoServiceProvider csp)
{
StringWriter outputStream = new StringWriter();
if (csp.PublicOnly) throw new ArgumentException("CSP does not contain a private key", "csp");
if (csp.PublicOnly)
throw new ArgumentException("CSP does not contain a private key", "csp");
var parameters = csp.ExportParameters(true);
using (var stream = new MemoryStream())
{
@ -74,7 +78,9 @@ namespace MyProject.Data.Encryption
writer.Write(innerStream.GetBuffer(), 0, length);
}
var base64 = Convert.ToBase64String(stream.GetBuffer(), 0, (int)stream.Length).ToCharArray();
var base64 = Convert
.ToBase64String(stream.GetBuffer(), 0, (int)stream.Length)
.ToCharArray();
// WriteLine terminates with \r\n, we want only \n
outputStream.Write("-----BEGIN RSA PRIVATE KEY-----\n");
// Output as Base64 with lines chopped at 64 characters
@ -109,7 +115,18 @@ namespace MyProject.Data.Encryption
innerWriter.Write((byte)0x30); // SEQUENCE
EncodeLength(innerWriter, 13);
innerWriter.Write((byte)0x06); // OBJECT IDENTIFIER
var rsaEncryptionOid = new byte[] { 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01 };
var rsaEncryptionOid = new byte[]
{
0x2a,
0x86,
0x48,
0x86,
0xf7,
0x0d,
0x01,
0x01,
0x01
};
EncodeLength(innerWriter, rsaEncryptionOid.Length);
innerWriter.Write(rsaEncryptionOid);
innerWriter.Write((byte)0x05); // NULL
@ -138,7 +155,9 @@ namespace MyProject.Data.Encryption
writer.Write(innerStream.GetBuffer(), 0, length);
}
var base64 = Convert.ToBase64String(stream.GetBuffer(), 0, (int)stream.Length).ToCharArray();
var base64 = Convert
.ToBase64String(stream.GetBuffer(), 0, (int)stream.Length)
.ToCharArray();
// WriteLine terminates with \r\n, we want only \n
outputStream.Write("-----BEGIN PUBLIC KEY-----\n");
for (var i = 0; i < base64.Length; i += 64)
@ -159,7 +178,8 @@ namespace MyProject.Data.Encryption
/// <param name="length"></param>
private static void EncodeLength(BinaryWriter stream, int length)
{
if (length < 0) throw new ArgumentOutOfRangeException("length", "Length must be non-negative");
if (length < 0)
throw new ArgumentOutOfRangeException("length", "Length must be non-negative");
if (length < 0x80)
{
// Short form
@ -189,13 +209,18 @@ namespace MyProject.Data.Encryption
/// <param name="stream"></param>
/// <param name="value"></param>
/// <param name="forceUnsigned"></param>
private static void EncodeIntegerBigEndian(BinaryWriter stream, byte[] value, bool forceUnsigned = true)
private static void EncodeIntegerBigEndian(
BinaryWriter stream,
byte[] value,
bool forceUnsigned = true
)
{
stream.Write((byte)0x02); // INTEGER
var prefixZeros = 0;
for (var i = 0; i < value.Length; i++)
{
if (value[i] != 0) break;
if (value[i] != 0)
break;
prefixZeros++;
}
if (value.Length - prefixZeros == 0)
@ -222,4 +247,4 @@ namespace MyProject.Data.Encryption
}
}
}
}
}

View File

@ -1,16 +1,20 @@
using System;
using System;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using BirdsiteLive.ActivityPub;
using BirdsiteLive.ActivityPub.Converters;
using BirdsiteLive.ActivityPub.Models;
using BirdsiteLive.Common.Settings;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Org.BouncyCastle.Bcpg;
namespace BirdsiteLive.Domain
@ -18,9 +22,19 @@ namespace BirdsiteLive.Domain
public interface IActivityPubService
{
Task<Actor> GetUser(string objectId);
Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null);
Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost,
string targetInbox);
Task<HttpStatusCode> PostDataAsync<T>(
T data,
string targetHost,
string actorUrl,
string inbox = null
);
Task PostNewNoteActivity(
Note note,
string username,
string noteId,
string targetHost,
string targetInbox
);
Task<WebFingerData> WebFinger(string account);
}
@ -32,7 +46,12 @@ namespace BirdsiteLive.Domain
private readonly ILogger<ActivityPubService> _logger;
#region Ctor
public ActivityPubService(ICryptoService cryptoService, InstanceSettings instanceSettings, IHttpClientFactory httpClientFactory, ILogger<ActivityPubService> logger)
public ActivityPubService(
ICryptoService cryptoService,
InstanceSettings instanceSettings,
IHttpClientFactory httpClientFactory,
ILogger<ActivityPubService> logger
)
{
_cryptoService = cryptoService;
_instanceSettings = instanceSettings;
@ -49,11 +68,18 @@ namespace BirdsiteLive.Domain
var content = await result.Content.ReadAsStringAsync();
var actor = JsonConvert.DeserializeObject<Actor>(content);
if (string.IsNullOrWhiteSpace(actor.url)) actor.url = objectId;
if (string.IsNullOrWhiteSpace(actor.url))
actor.url = objectId;
return actor;
}
public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox)
public async Task PostNewNoteActivity(
Note note,
string username,
string noteId,
string targetHost,
string targetInbox
)
{
try
{
@ -70,7 +96,6 @@ namespace BirdsiteLive.Domain
type = "Create",
actor = actor,
published = nowString,
to = note.to,
cc = note.cc,
apObject = note
@ -80,12 +105,24 @@ namespace BirdsiteLive.Domain
}
catch (Exception e)
{
_logger.LogError(e, "Error sending {Username} post ({NoteId}) to {Host}{Inbox}", username, noteId, targetHost, targetInbox);
_logger.LogError(
e,
"Error sending {Username} post ({NoteId}) to {Host}{Inbox}",
username,
noteId,
targetHost,
targetInbox
);
throw;
}
}
public async Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null)
public async Task<HttpStatusCode> PostDataAsync<T>(
T data,
string targetHost,
string actorUrl,
string inbox = null
)
{
var usedInbox = $"/inbox";
if (!string.IsNullOrWhiteSpace(inbox))
@ -98,7 +135,13 @@ namespace BirdsiteLive.Domain
var digest = _cryptoService.ComputeSha256Hash(json);
var signature = _cryptoService.SignAndGetSignatureHeader(date, actorUrl, targetHost, digest, usedInbox);
var signature = _cryptoService.SignAndGetSignatureHeader(
date,
actorUrl,
targetHost,
digest,
usedInbox
);
var client = _httpClientFactory.CreateClient();
var httpRequestMessage = new HttpRequestMessage
@ -107,10 +150,10 @@ namespace BirdsiteLive.Domain
RequestUri = new Uri($"https://{targetHost}{usedInbox}"),
Headers =
{
{"Host", targetHost},
{"Date", httpDate},
{"Signature", signature},
{"Digest", $"SHA-256={digest}"}
{ "Host", targetHost },
{ "Date", httpDate },
{ "Signature", signature },
{ "Digest", $"SHA-256={digest}" }
},
Content = new StringContent(json, Encoding.UTF8, "application/ld+json")
};
@ -123,10 +166,15 @@ namespace BirdsiteLive.Domain
public async Task<WebFingerData> WebFinger(string account)
{
var httpClient = _httpClientFactory.CreateClient();
var result = await httpClient.GetAsync("https://" + account.Split('@')[1] + "/.well-known/webfinger?resource=acct:" + account);
var result = await httpClient.GetAsync(
"https://"
+ account.Split('@')[1]
+ "/.well-known/webfinger?resource=acct:"
+ account
);
var content = await result.Content.ReadAsStringAsync();
return JsonConvert.DeserializeObject<WebFingerData>(content);
}
}
}
}

View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<TargetFramework>net6.0</TargetFramework>
</PropertyGroup>
<ItemGroup>

View File

@ -1,11 +1,19 @@
using System.Threading.Tasks;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
namespace BirdsiteLive.Domain.BusinessUseCases
{
public interface IProcessFollowUser
{
Task ExecuteAsync(string followerUsername, string followerDomain, string twitterUsername, string followerInbox, string sharedInbox, string followerActorId);
Task ExecuteAsync(
string followerUsername,
string followerDomain,
string twitterUsername,
string followerInbox,
string sharedInbox,
string followerActorId
);
}
public class ProcessFollowUser : IProcessFollowUser
@ -21,13 +29,26 @@ namespace BirdsiteLive.Domain.BusinessUseCases
}
#endregion
public async Task ExecuteAsync(string followerUsername, string followerDomain, string twitterUsername, string followerInbox, string sharedInbox, string followerActorId)
public async Task ExecuteAsync(
string followerUsername,
string followerDomain,
string twitterUsername,
string followerInbox,
string sharedInbox,
string followerActorId
)
{
// Get Follower and Twitter Users
var follower = await _followerDal.GetFollowerAsync(followerUsername, followerDomain);
if (follower == null)
{
await _followerDal.CreateFollowerAsync(followerUsername, followerDomain, followerInbox, sharedInbox, followerActorId);
await _followerDal.CreateFollowerAsync(
followerUsername,
followerDomain,
followerInbox,
sharedInbox,
followerActorId
);
follower = await _followerDal.GetFollowerAsync(followerUsername, followerDomain);
}
@ -40,14 +61,14 @@ namespace BirdsiteLive.Domain.BusinessUseCases
// Update Follower
var twitterUserId = twitterUser.Id;
if(!follower.Followings.Contains(twitterUserId))
if (!follower.Followings.Contains(twitterUserId))
follower.Followings.Add(twitterUserId);
if(!follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
if (!follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
follower.FollowingsSyncStatus.Add(twitterUserId, -1);
// Save Follower
await _followerDal.UpdateFollowerAsync(follower);
}
}
}
}

View File

@ -1,5 +1,6 @@
using System.Linq;
using System.Linq;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
namespace BirdsiteLive.Domain.BusinessUseCases
@ -22,14 +23,20 @@ namespace BirdsiteLive.Domain.BusinessUseCases
}
#endregion
public async Task ExecuteAsync(string followerUsername, string followerDomain, string twitterUsername)
public async Task ExecuteAsync(
string followerUsername,
string followerDomain,
string twitterUsername
)
{
// Get Follower and Twitter Users
var follower = await _followerDal.GetFollowerAsync(followerUsername, followerDomain);
if (follower == null) return;
if (follower == null)
return;
var twitterUser = await _twitterUserDal.GetTwitterUserAsync(twitterUsername);
if (twitterUser == null) return;
if (twitterUser == null)
return;
// Update Follower
var twitterUserId = twitterUser.Id;
@ -43,12 +50,12 @@ namespace BirdsiteLive.Domain.BusinessUseCases
if (follower.Followings.Any())
await _followerDal.UpdateFollowerAsync(follower);
else
await _followerDal.DeleteFollowerAsync(followerUsername, followerDomain);
await _followerDal.DeleteFollowerAsync(followerUsername, followerDomain);
// Check if TwitterUser has still followers
var followers = await _followerDal.GetFollowersAsync(twitterUser.Id);
if (!followers.Any())
await _twitterUserDal.DeleteTwitterUserAsync(twitterUsername);
}
}
}
}

View File

@ -1,6 +1,7 @@
using System;
using System;
using System.Security.Cryptography;
using System.Text;
using BirdsiteLive.Domain.Factories;
namespace BirdsiteLive.Domain
@ -8,7 +9,13 @@ namespace BirdsiteLive.Domain
public interface ICryptoService
{
string GetUserPem(string id);
string SignAndGetSignatureHeader(DateTime date, string actor, string host, string digest, string inbox);
string SignAndGetSignatureHeader(
DateTime date,
string actor,
string host,
string digest,
string inbox
);
string ComputeSha256Hash(string data);
}
@ -29,13 +36,19 @@ namespace BirdsiteLive.Domain
}
/// <summary>
///
///
/// </summary>
/// <param name="data"></param>
/// <param name="actor">in the form of https://domain.io/actor</param>
/// <param name="host">in the form of domain.io</param>
/// <returns></returns>
public string SignAndGetSignatureHeader(DateTime date, string actor, string targethost, string digest, string inbox)
public string SignAndGetSignatureHeader(
DateTime date,
string actor,
string targethost,
string digest,
string inbox
)
{
var usedInbox = "/inbox";
if (!string.IsNullOrWhiteSpace(inbox))
@ -43,24 +56,30 @@ namespace BirdsiteLive.Domain
var httpDate = date.ToString("r");
var signedString = $"(request-target): post {usedInbox}\nhost: {targethost}\ndate: {httpDate}\ndigest: SHA-256={digest}";
var signedString =
$"(request-target): post {usedInbox}\nhost: {targethost}\ndate: {httpDate}\ndigest: SHA-256={digest}";
var signedStringBytes = Encoding.UTF8.GetBytes(signedString);
var signature = _magicKeyFactory.GetMagicKey().Sign(signedStringBytes);
var sig64 = Convert.ToBase64String(signature);
var header = "keyId=\"" + actor + "\",algorithm=\"rsa-sha256\",headers=\"(request-target) host date digest\",signature=\"" + sig64 + "\"";
var header =
"keyId=\""
+ actor
+ "\",algorithm=\"rsa-sha256\",headers=\"(request-target) host date digest\",signature=\""
+ sig64
+ "\"";
return header;
}
public string ComputeSha256Hash(string data)
{
// Create a SHA256
// Create a SHA256
using (SHA256 sha256Hash = SHA256.Create())
{
// ComputeHash - returns byte array
// ComputeHash - returns byte array
byte[] bytes = sha256Hash.ComputeHash(Encoding.UTF8.GetBytes(data));
return Convert.ToBase64String(bytes);
}
}
}
}
}

View File

@ -1,4 +1,5 @@
using System.IO;
using System.IO;
using BirdsiteLive.Cryptography;
namespace BirdsiteLive.Domain.Factories
@ -14,16 +15,14 @@ namespace BirdsiteLive.Domain.Factories
private static MagicKey _magicKey;
#region Ctor
public MagicKeyFactory()
{
}
public MagicKeyFactory() { }
#endregion
public MagicKey GetMagicKey()
{
//Cached key
if (_magicKey != null) return _magicKey;
if (_magicKey != null)
return _magicKey;
//Generate key if needed
if (!File.Exists(Path))
@ -38,4 +37,4 @@ namespace BirdsiteLive.Domain.Factories
return _magicKey;
}
}
}
}

View File

@ -1,7 +1,8 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.Domain.Tools;
@ -38,8 +39,12 @@ namespace BirdsiteLive.Domain.Repository
var parsedFollowersWhiteListing = PatternsParser.Parse(settings.FollowersWhiteListing);
var parsedFollowersBlackListing = PatternsParser.Parse(settings.FollowersBlackListing);
var parsedTwitterAccountsWhiteListing = PatternsParser.Parse(settings.TwitterAccountsWhiteListing);
var parsedTwitterAccountsBlackListing = PatternsParser.Parse(settings.TwitterAccountsBlackListing);
var parsedTwitterAccountsWhiteListing = PatternsParser.Parse(
settings.TwitterAccountsWhiteListing
);
var parsedTwitterAccountsBlackListing = PatternsParser.Parse(
settings.TwitterAccountsBlackListing
);
_followersWhiteListing = parsedFollowersWhiteListing
.Select(x => ModerationRegexParser.Parse(ModerationEntityTypeEnum.Follower, x))
@ -48,10 +53,14 @@ namespace BirdsiteLive.Domain.Repository
.Select(x => ModerationRegexParser.Parse(ModerationEntityTypeEnum.Follower, x))
.ToArray();
_twitterAccountsWhiteListing = parsedTwitterAccountsWhiteListing
.Select(x => ModerationRegexParser.Parse(ModerationEntityTypeEnum.TwitterAccount, x))
.Select(
x => ModerationRegexParser.Parse(ModerationEntityTypeEnum.TwitterAccount, x)
)
.ToArray();
_twitterAccountsBlackListing = parsedTwitterAccountsBlackListing
.Select(x => ModerationRegexParser.Parse(ModerationEntityTypeEnum.TwitterAccount, x))
.Select(
x => ModerationRegexParser.Parse(ModerationEntityTypeEnum.TwitterAccount, x)
)
.ToArray();
// Set Follower moderation politic
@ -64,9 +73,15 @@ namespace BirdsiteLive.Domain.Repository
// Set Twitter account moderation politic
if (_twitterAccountsWhiteListing.Any())
_modMode.Add(ModerationEntityTypeEnum.TwitterAccount, ModerationTypeEnum.WhiteListing);
_modMode.Add(
ModerationEntityTypeEnum.TwitterAccount,
ModerationTypeEnum.WhiteListing
);
else if (_twitterAccountsBlackListing.Any())
_modMode.Add(ModerationEntityTypeEnum.TwitterAccount, ModerationTypeEnum.BlackListing);
_modMode.Add(
ModerationEntityTypeEnum.TwitterAccount,
ModerationTypeEnum.BlackListing
);
else
_modMode.Add(ModerationEntityTypeEnum.TwitterAccount, ModerationTypeEnum.None);
}
@ -79,7 +94,8 @@ namespace BirdsiteLive.Domain.Repository
public ModeratedTypeEnum CheckStatus(ModerationEntityTypeEnum type, string entity)
{
if (_modMode[type] == ModerationTypeEnum.None) return ModeratedTypeEnum.None;
if (_modMode[type] == ModerationTypeEnum.None)
return ModeratedTypeEnum.None;
switch (type)
{
@ -91,7 +107,7 @@ namespace BirdsiteLive.Domain.Repository
throw new NotImplementedException($"Type {type} is not supported");
}
private ModeratedTypeEnum ProcessFollower(string entity)
{
var politic = _modMode[ModerationEntityTypeEnum.Follower];
@ -137,30 +153,40 @@ namespace BirdsiteLive.Domain.Repository
private char GetSplitChar(string entry)
{
var separationChar = '|';
if (entry.Contains(";")) separationChar = ';';
else if (entry.Contains(",")) separationChar = ',';
if (entry.Contains(";"))
separationChar = ';';
else if (entry.Contains(","))
separationChar = ',';
return separationChar;
}
public IEnumerable<string> GetWhitelistedFollowers()
{
return _settings.FollowersWhiteListing.Split(GetSplitChar(_settings.FollowersWhiteListing));
return _settings.FollowersWhiteListing.Split(
GetSplitChar(_settings.FollowersWhiteListing)
);
}
public IEnumerable<string> GetBlacklistedFollowers()
{
return _settings.FollowersBlackListing.Split(GetSplitChar(_settings.FollowersBlackListing));
return _settings.FollowersBlackListing.Split(
GetSplitChar(_settings.FollowersBlackListing)
);
}
public IEnumerable<string> GetWhitelistedAccounts()
{
return _settings.TwitterAccountsWhiteListing.Split(GetSplitChar(_settings.TwitterAccountsWhiteListing));
return _settings.TwitterAccountsWhiteListing.Split(
GetSplitChar(_settings.TwitterAccountsWhiteListing)
);
}
public IEnumerable<string> GetBlacklistedAccounts()
{
return _settings.TwitterAccountsBlackListing.Split(GetSplitChar(_settings.TwitterAccountsBlackListing));
return _settings.TwitterAccountsBlackListing.Split(
GetSplitChar(_settings.TwitterAccountsBlackListing)
);
}
}
@ -184,4 +210,4 @@ namespace BirdsiteLive.Domain.Repository
BlackListed = 1,
WhiteListed = 2
}
}
}

View File

@ -1,4 +1,5 @@
using System.Linq;
using System.Linq;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.Domain.Tools;
@ -25,14 +26,16 @@ namespace BirdsiteLive.Domain.Repository
public bool IsUnlisted(string twitterAcct)
{
if (_unlistedAccounts == null || !_unlistedAccounts.Any()) return false;
if (_unlistedAccounts == null || !_unlistedAccounts.Any())
return false;
return _unlistedAccounts.Contains(twitterAcct.ToLowerInvariant());
}
public bool IsSensitive(string twitterAcct)
{
if (_sensitiveAccounts == null || !_sensitiveAccounts.Any()) return false;
if (_sensitiveAccounts == null || !_sensitiveAccounts.Any())
return false;
return _sensitiveAccounts.Contains(twitterAcct.ToLowerInvariant());
}

View File

@ -1,4 +1,4 @@
using System.Threading;
using System.Threading;
using System.Timers;
namespace BirdsiteLive.Domain.Statistics
@ -57,14 +57,24 @@ namespace BirdsiteLive.Domain.Statistics
public ExtractionStatistics GetStatistics()
{
return new ExtractionStatistics(_descriptionMentionsExtracted, _statusMentionsExtracted, _lastDescriptionMentionsExtracted, _lastStatusMentionsExtracted);
return new ExtractionStatistics(
_descriptionMentionsExtracted,
_statusMentionsExtracted,
_lastDescriptionMentionsExtracted,
_lastStatusMentionsExtracted
);
}
}
public class ExtractionStatistics
{
#region Ctor
public ExtractionStatistics(int mentionsInDescriptionsExtraction, int mentionsInStatusesExtraction, int lastMentionsInDescriptionsExtraction, int lastMentionsInStatusesExtraction)
public ExtractionStatistics(
int mentionsInDescriptionsExtraction,
int mentionsInStatusesExtraction,
int lastMentionsInDescriptionsExtraction,
int lastMentionsInStatusesExtraction
)
{
MentionsInDescriptionsExtraction = mentionsInDescriptionsExtraction;
MentionsInStatusesExtraction = mentionsInStatusesExtraction;
@ -79,4 +89,4 @@ namespace BirdsiteLive.Domain.Statistics
public int LastMentionsInDescriptionsExtraction { get; }
public int LastMentionsInStatusesExtraction { get; }
}
}
}

View File

@ -1,8 +1,9 @@
using System;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using BirdsiteLive.ActivityPub;
using BirdsiteLive.ActivityPub.Converters;
using BirdsiteLive.ActivityPub.Models;
@ -11,6 +12,7 @@ using BirdsiteLive.Domain.Repository;
using BirdsiteLive.Domain.Statistics;
using BirdsiteLive.Domain.Tools;
using BirdsiteLive.Twitter.Models;
using Tweetinvi.Models;
using Tweetinvi.Models.Entities;
@ -29,7 +31,12 @@ namespace BirdsiteLive.Domain
private readonly IPublicationRepository _publicationRepository;
#region Ctor
public StatusService(InstanceSettings instanceSettings, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler, IPublicationRepository publicationRepository)
public StatusService(
InstanceSettings instanceSettings,
IStatusExtractor statusExtractor,
IExtractionStatisticsHandler statisticsHandler,
IPublicationRepository publicationRepository
)
{
_instanceSettings = instanceSettings;
_statusExtractor = statusExtractor;
@ -41,15 +48,19 @@ namespace BirdsiteLive.Domain
public Note GetStatus(string username, ExtractedTweet tweet)
{
var actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, username);
var noteUrl = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, tweet.Id.ToString());
var noteUrl = UrlFactory.GetNoteUrl(
_instanceSettings.Domain,
username,
tweet.Id.ToString()
);
var to = $"{actorUrl}/followers";
var isUnlisted = _publicationRepository.IsUnlisted(username);
var cc = new string[0];
if (isUnlisted)
cc = new[] {"https://www.w3.org/ns/activitystreams#Public"};
cc = new[] { "https://www.w3.org/ns/activitystreams#Public" };
string summary = null;
var sensitive = _publicationRepository.IsSensitive(username);
if (sensitive || tweet.IsSensitive)
@ -63,38 +74,37 @@ namespace BirdsiteLive.Domain
if (content.Contains("{RT}") && tweet.IsRetweet)
{
if (!string.IsNullOrWhiteSpace(tweet.RetweetUrl))
content = content.Replace("{RT}",
$@"<a href=""{tweet.RetweetUrl}"" rel=""nofollow noopener noreferrer"" target=""_blank"">RT</a>");
content = content.Replace(
"{RT}",
$@"<a href=""{tweet.RetweetUrl}"" rel=""nofollow noopener noreferrer"" target=""_blank"">RT</a>"
);
else
content = content.Replace("{RT}", "RT");
}
string inReplyTo = null;
if (tweet.InReplyToStatusId != default)
inReplyTo = $"https://{_instanceSettings.Domain}/users/{tweet.InReplyToAccount.ToLowerInvariant()}/statuses/{tweet.InReplyToStatusId}";
inReplyTo =
$"https://{_instanceSettings.Domain}/users/{tweet.InReplyToAccount.ToLowerInvariant()}/statuses/{tweet.InReplyToStatusId}";
if( tweet.QuoteTweetUrl != null )
content += $@"<span class=""quote-inline""><br><br>RT: <a href=""{tweet.QuoteTweetUrl}"">{tweet.QuoteTweetUrl}</a></span>";
if (tweet.QuoteTweetUrl != null)
content +=
$@"<span class=""quote-inline""><br><br>RT: <a href=""{tweet.QuoteTweetUrl}"">{tweet.QuoteTweetUrl}</a></span>";
var note = new Note
{
id = noteUrl,
published = tweet.CreatedAt.ToString("s") + "Z",
url = noteUrl,
attributedTo = actorUrl,
inReplyTo = inReplyTo,
to = new[] { to },
cc = cc,
sensitive = tweet.IsSensitive || sensitive,
summary = summary,
content = $"<p>{content}</p>",
attachment = Convert(tweet.Media),
tag = extractedTags.tags,
quoteUrl = tweet.QuoteTweetUrl
};
@ -103,16 +113,19 @@ namespace BirdsiteLive.Domain
private Attachment[] Convert(ExtractedMedia[] media)
{
if(media == null) return new Attachment[0];
return media.Select(x =>
{
return new Attachment
if (media == null)
return new Attachment[0];
return media
.Select(x =>
{
type = "Document",
url = x.Url,
mediaType = x.MediaType
};
}).ToArray();
return new Attachment
{
type = "Document",
url = x.Url,
mediaType = x.MediaType
};
})
.ToArray();
}
}
}

View File

@ -1,6 +1,8 @@
using System;
using System;
using System.Text.RegularExpressions;
using BirdsiteLive.Domain.Repository;
using Org.BouncyCastle.Pkcs;
namespace BirdsiteLive.Domain.Tools
@ -25,4 +27,4 @@ namespace BirdsiteLive.Domain.Tools
return new Regex($@"^{data}$");
}
}
}
}

View File

@ -1,4 +1,4 @@
using System;
using System;
using System.Linq;
namespace BirdsiteLive.Domain.Tools
@ -7,17 +7,20 @@ namespace BirdsiteLive.Domain.Tools
{
public static string[] Parse(string entry)
{
if (string.IsNullOrWhiteSpace(entry)) return new string[0];
if (string.IsNullOrWhiteSpace(entry))
return new string[0];
var separationChar = '|';
if (entry.Contains(";")) separationChar = ';';
else if (entry.Contains(",")) separationChar = ',';
if (entry.Contains(";"))
separationChar = ';';
else if (entry.Contains(","))
separationChar = ',';
var splitEntries = entry
.Split(new[] {separationChar}, StringSplitOptions.RemoveEmptyEntries)
.Split(new[] { separationChar }, StringSplitOptions.RemoveEmptyEntries)
.Where(x => !string.IsNullOrWhiteSpace(x))
.Select(x => x.ToLowerInvariant().Trim());
return splitEntries.ToArray();
}
}
}
}

View File

@ -1,10 +1,12 @@
using System.Collections.Generic;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using BirdsiteLive.ActivityPub.Models;
using BirdsiteLive.Common.Regexes;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.Twitter;
using Microsoft.Extensions.Logging;
namespace BirdsiteLive.Domain.Tools
@ -27,7 +29,10 @@ namespace BirdsiteLive.Domain.Tools
}
#endregion
public (string content, Tag[] tags) Extract(string messageContent, bool extractMentions = true)
public (string content, Tag[] tags) Extract(
string messageContent,
bool extractMentions = true
)
{
var tags = new List<Tag>();
@ -64,8 +69,11 @@ namespace BirdsiteLive.Domain.Tools
secondPart = truncatedUrl.Substring(30);
}
messageContent = Regex.Replace(messageContent, Regex.Escape(m.ToString()),
$@"{m.Groups[1]}<a href=""{url}"" rel=""nofollow noopener noreferrer"" target=""_blank""><span class=""invisible"">{protocol}</span><span class=""ellipsis"">{firstPart}</span><span class=""invisible"">{secondPart}</span></a>");
messageContent = Regex.Replace(
messageContent,
Regex.Escape(m.ToString()),
$@"{m.Groups[1]}<a href=""{url}"" rel=""nofollow noopener noreferrer"" target=""_blank""><span class=""invisible"">{protocol}</span><span class=""ellipsis"">{firstPart}</span><span class=""invisible"">{secondPart}</span></a>"
);
}
// Extract Hashtags
@ -76,7 +84,11 @@ namespace BirdsiteLive.Domain.Tools
if (!HashtagRegexes.HashtagName.IsMatch(tag))
{
_logger.LogError("Parsing Hashtag failed: {Tag} on {Content}", tag, messageContent);
_logger.LogError(
"Parsing Hashtag failed: {Tag} on {Content}",
tag,
messageContent
);
continue;
}
@ -84,16 +96,21 @@ namespace BirdsiteLive.Domain.Tools
if (tags.All(x => x.href != url))
{
tags.Add(new Tag
{
name = $"#{tag}",
href = url,
type = "Hashtag"
});
tags.Add(
new Tag
{
name = $"#{tag}",
href = url,
type = "Hashtag"
}
);
}
messageContent = Regex.Replace(messageContent, Regex.Escape(m.Groups[0].ToString()),
$@"{m.Groups[1]}<a href=""{url}"" class=""mention hashtag"" rel=""tag"">#<span>{tag}</span></a>{m.Groups[3]}");
messageContent = Regex.Replace(
messageContent,
Regex.Escape(m.Groups[0].ToString()),
$@"{m.Groups[1]}<a href=""{url}"" class=""mention hashtag"" rel=""tag"">#<span>{tag}</span></a>{m.Groups[3]}"
);
}
// Extract Mentions
@ -106,7 +123,11 @@ namespace BirdsiteLive.Domain.Tools
if (!UserRegexes.TwitterAccount.IsMatch(mention))
{
_logger.LogError("Parsing Mention failed: {Mention} on {Content}", mention, messageContent);
_logger.LogError(
"Parsing Mention failed: {Mention} on {Content}",
mention,
messageContent
);
continue;
}
@ -115,26 +136,32 @@ namespace BirdsiteLive.Domain.Tools
if (tags.All(x => x.href != url))
{
tags.Add(new Tag
{
name = name,
href = url,
type = "Mention"
});
tags.Add(
new Tag
{
name = name,
href = url,
type = "Mention"
}
);
}
messageContent = Regex.Replace(messageContent, Regex.Escape(m.Groups[0].ToString()),
$@"{m.Groups[1]}<span class=""h-card""><a href=""https://{_instanceSettings.Domain}/@{mention}"" class=""u-url mention"">@<span>{mention}</span></a></span>{m.Groups[3]}");
messageContent = Regex.Replace(
messageContent,
Regex.Escape(m.Groups[0].ToString()),
$@"{m.Groups[1]}<span class=""h-card""><a href=""https://{_instanceSettings.Domain}/@{mention}"" class=""u-url mention"">@<span>{mention}</span></a></span>{m.Groups[3]}"
);
}
}
return (messageContent.Trim(), tags.ToArray());
}
private IEnumerable<Match> OrderByLength(MatchCollection matches)
{
var result = new List<Match>();
foreach (Match m in matches) result.Add(m);
foreach (Match m in matches)
result.Add(m);
result = result
.OrderBy(x => x.Length)
@ -145,4 +172,4 @@ namespace BirdsiteLive.Domain.Tools
return result;
}
}
}
}

View File

@ -1,10 +1,11 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using BirdsiteLive.ActivityPub;
using BirdsiteLive.ActivityPub.Converters;
using BirdsiteLive.ActivityPub.Models;
@ -18,6 +19,7 @@ using BirdsiteLive.Domain.Statistics;
using BirdsiteLive.Domain.Tools;
using BirdsiteLive.Twitter;
using BirdsiteLive.Twitter.Models;
using Tweetinvi.Core.Exceptions;
using Tweetinvi.Models;
@ -26,8 +28,24 @@ namespace BirdsiteLive.Domain
public interface IUserService
{
Actor GetUser(TwitterUser twitterUser);
Task<bool> FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders, ActivityFollow activity, string body);
Task<bool> UndoFollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders, ActivityUndoFollow activity, string body);
Task<bool> FollowRequestedAsync(
string signature,
string method,
string path,
string queryString,
Dictionary<string, string> requestHeaders,
ActivityFollow activity,
string body
);
Task<bool> UndoFollowRequestedAsync(
string signature,
string method,
string path,
string queryString,
Dictionary<string, string> requestHeaders,
ActivityUndoFollow activity,
string body
);
Task<bool> SendRejectFollowAsync(ActivityFollow activity, string followerHost);
}
@ -50,7 +68,18 @@ namespace BirdsiteLive.Domain
private readonly IFollowersDal _followerDal;
#region Ctor
public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler, ITwitterUserService twitterUserService, IModerationRepository moderationRepository, IFollowersDal followerDal)
public UserService(
InstanceSettings instanceSettings,
ICryptoService cryptoService,
IActivityPubService activityPubService,
IProcessFollowUser processFollowUser,
IProcessUndoFollowUser processUndoFollowUser,
IStatusExtractor statusExtractor,
IExtractionStatisticsHandler statisticsHandler,
ITwitterUserService twitterUserService,
IModerationRepository moderationRepository,
IFollowersDal followerDal
)
{
_instanceSettings = instanceSettings;
_cryptoService = cryptoService;
@ -74,44 +103,60 @@ namespace BirdsiteLive.Domain
var description = twitterUser.Description;
if (!string.IsNullOrWhiteSpace(description))
{
var extracted = _statusExtractor.Extract(description, _instanceSettings.ResolveMentionsInProfiles);
var extracted = _statusExtractor.Extract(
description,
_instanceSettings.ResolveMentionsInProfiles
);
description = extracted.content;
_statisticsHandler.ExtractedDescription(extracted.tags.Count(x => x.type == "Mention"));
_statisticsHandler.ExtractedDescription(
extracted.tags.Count(x => x.type == "Mention")
);
}
var attachments = new List<UserAttachment>();
attachments.Add(new UserAttachment
{
type = "PropertyValue",
name = _instanceSettings.TwitterDomainLabel != "" ? _instanceSettings.TwitterDomainLabel : _instanceSettings.TwitterDomain,
value = $"<a href=\"https://{_instanceSettings.TwitterDomain}/{acct}\" rel=\"me nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">{_instanceSettings.TwitterDomain}/{acct}</span></a>"
});
if(_instanceSettings.TwitterDomain != "twitter.com")
{
attachments.Add(new UserAttachment
attachments.Add(
new UserAttachment
{
type = "PropertyValue",
name = "Twitter",
value = $"twitter.com/{acct}"
});
name =
_instanceSettings.TwitterDomainLabel != ""
? _instanceSettings.TwitterDomainLabel
: _instanceSettings.TwitterDomain,
value =
$"<a href=\"https://{_instanceSettings.TwitterDomain}/{acct}\" rel=\"me nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">{_instanceSettings.TwitterDomain}/{acct}</span></a>"
}
);
if (_instanceSettings.TwitterDomain != "twitter.com")
{
attachments.Add(
new UserAttachment
{
type = "PropertyValue",
name = "Twitter",
value = $"twitter.com/{acct}"
}
);
}
if (_instanceSettings.ShowAboutInstanceOnProfiles)
{
attachments.Add(new UserAttachment
{
type = "PropertyValue",
name = $"About {_instanceSettings.Name}",
value = $"<a href=\"https://{_instanceSettings.Domain}/About\" rel=\"nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">{_instanceSettings.Domain}/About</span></a>"
});
attachments.Add(
new UserAttachment
{
type = "PropertyValue",
name = $"About {_instanceSettings.Name}",
value =
$"<a href=\"https://{_instanceSettings.Domain}/About\" rel=\"nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">{_instanceSettings.Domain}/About</span></a>"
}
);
}
var user = new Actor
{
id = actorUrl,
type = "Service",
type = "Service",
followers = $"{actorUrl}/followers",
preferredUsername = acct,
name = twitterUser.Name,
@ -125,16 +170,8 @@ namespace BirdsiteLive.Domain
owner = actorUrl,
publicKeyPem = _cryptoService.GetUserPem(acct)
},
icon = new Image
{
mediaType = "image/jpeg",
url = twitterUser.ProfileImageUrl
},
image = new Image
{
mediaType = "image/jpeg",
url = twitterUser.ProfileBannerURL
},
icon = new Image { mediaType = "image/jpeg", url = twitterUser.ProfileImageUrl },
image = new Image { mediaType = "image/jpeg", url = twitterUser.ProfileBannerURL },
attachment = attachments.ToArray(),
endpoints = new EndPoints
{
@ -165,49 +202,96 @@ namespace BirdsiteLive.Domain
return user;
}
public async Task<bool> FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders, ActivityFollow activity, string body)
public async Task<bool> FollowRequestedAsync(
string signature,
string method,
string path,
string queryString,
Dictionary<string, string> requestHeaders,
ActivityFollow activity,
string body
)
{
// Validate
var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body);
if (!sigValidation.SignatureIsValidated) return false;
var sigValidation = await ValidateSignature(
activity.actor,
signature,
method,
path,
queryString,
requestHeaders,
body
);
if (!sigValidation.SignatureIsValidated)
return false;
// Prepare data
var followerUserName = sigValidation.User.preferredUsername.ToLowerInvariant().Trim();
var followerHost = sigValidation.User.url.Replace("https://", string.Empty).Split('/').First();
var followerHost = sigValidation.User.url
.Replace("https://", string.Empty)
.Split('/')
.First();
var followerInbox = sigValidation.User.inbox;
var followerSharedInbox = sigValidation.User?.endpoints?.sharedInbox;
var twitterUser = activity.apObject.Split('/').Last().Replace("@", string.Empty).ToLowerInvariant().Trim();
var twitterUser = activity.apObject
.Split('/')
.Last()
.Replace("@", string.Empty)
.ToLowerInvariant()
.Trim();
// Make sure to only keep routes
followerInbox = OnlyKeepRoute(followerInbox, followerHost);
followerSharedInbox = OnlyKeepRoute(followerSharedInbox, followerHost);
// Validate Moderation status
var followerModPolicy = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.Follower);
var followerModPolicy = _moderationRepository.GetModerationType(
ModerationEntityTypeEnum.Follower
);
if (followerModPolicy != ModerationTypeEnum.None)
{
var followerStatus = _moderationRepository.CheckStatus(ModerationEntityTypeEnum.Follower, $"@{followerUserName}@{followerHost}");
if(followerModPolicy == ModerationTypeEnum.WhiteListing && followerStatus != ModeratedTypeEnum.WhiteListed ||
followerModPolicy == ModerationTypeEnum.BlackListing && followerStatus == ModeratedTypeEnum.BlackListed)
var followerStatus = _moderationRepository.CheckStatus(
ModerationEntityTypeEnum.Follower,
$"@{followerUserName}@{followerHost}"
);
if (
followerModPolicy == ModerationTypeEnum.WhiteListing
&& followerStatus != ModeratedTypeEnum.WhiteListed
|| followerModPolicy == ModerationTypeEnum.BlackListing
&& followerStatus == ModeratedTypeEnum.BlackListed
)
return await SendRejectFollowAsync(activity, followerHost);
}
// Validate TwitterAccount status
var twitterAccountModPolicy = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.TwitterAccount);
var twitterAccountModPolicy = _moderationRepository.GetModerationType(
ModerationEntityTypeEnum.TwitterAccount
);
if (twitterAccountModPolicy != ModerationTypeEnum.None)
{
var twitterUserStatus = _moderationRepository.CheckStatus(ModerationEntityTypeEnum.TwitterAccount, twitterUser);
if (twitterAccountModPolicy == ModerationTypeEnum.WhiteListing && twitterUserStatus != ModeratedTypeEnum.WhiteListed ||
twitterAccountModPolicy == ModerationTypeEnum.BlackListing && twitterUserStatus == ModeratedTypeEnum.BlackListed)
var twitterUserStatus = _moderationRepository.CheckStatus(
ModerationEntityTypeEnum.TwitterAccount,
twitterUser
);
if (
twitterAccountModPolicy == ModerationTypeEnum.WhiteListing
&& twitterUserStatus != ModeratedTypeEnum.WhiteListed
|| twitterAccountModPolicy == ModerationTypeEnum.BlackListing
&& twitterUserStatus == ModeratedTypeEnum.BlackListed
)
return await SendRejectFollowAsync(activity, followerHost);
}
// Validate follower count < MaxFollowsPerUser
if (_instanceSettings.MaxFollowsPerUser > 0) {
if (_instanceSettings.MaxFollowsPerUser > 0)
{
var follower = await _followerDal.GetFollowerAsync(followerUserName, followerHost);
if (follower != null && follower.Followings.Count + 1 > _instanceSettings.MaxFollowsPerUser)
if (
follower != null
&& follower.Followings.Count + 1 > _instanceSettings.MaxFollowsPerUser
)
{
return await SendRejectFollowAsync(activity, followerHost);
}
@ -218,7 +302,14 @@ namespace BirdsiteLive.Domain
if (!user.Protected)
{
// Execute
await _processFollowUser.ExecuteAsync(followerUserName, followerHost, twitterUser, followerInbox, followerSharedInbox, activity.actor);
await _processFollowUser.ExecuteAsync(
followerUserName,
followerHost,
twitterUser,
followerInbox,
followerSharedInbox,
activity.actor
);
return await SendAcceptFollowAsync(activity, followerHost);
}
@ -227,7 +318,7 @@ namespace BirdsiteLive.Domain
return await SendRejectFollowAsync(activity, followerHost);
}
}
private async Task<bool> SendAcceptFollowAsync(ActivityFollow activity, string followerHost)
{
var acceptFollow = new ActivityAcceptFollow()
@ -244,9 +335,12 @@ namespace BirdsiteLive.Domain
apObject = activity.apObject
}
};
var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject);
return result == HttpStatusCode.Accepted ||
result == HttpStatusCode.OK; //TODO: revamp this for better error handling
var result = await _activityPubService.PostDataAsync(
acceptFollow,
followerHost,
activity.apObject
);
return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; //TODO: revamp this for better error handling
}
public async Task<bool> SendRejectFollowAsync(ActivityFollow activity, string followerHost)
@ -265,14 +359,17 @@ namespace BirdsiteLive.Domain
apObject = activity.apObject
}
};
var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject);
return result == HttpStatusCode.Accepted ||
result == HttpStatusCode.OK; //TODO: revamp this for better error handling
var result = await _activityPubService.PostDataAsync(
acceptFollow,
followerHost,
activity.apObject
);
return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; //TODO: revamp this for better error handling
}
private string OnlyKeepRoute(string inbox, string host)
{
if (string.IsNullOrWhiteSpace(inbox))
if (string.IsNullOrWhiteSpace(inbox))
return null;
if (inbox.Contains(host))
@ -281,18 +378,40 @@ namespace BirdsiteLive.Domain
return inbox;
}
public async Task<bool> UndoFollowRequestedAsync(string signature, string method, string path, string queryString,
Dictionary<string, string> requestHeaders, ActivityUndoFollow activity, string body)
public async Task<bool> UndoFollowRequestedAsync(
string signature,
string method,
string path,
string queryString,
Dictionary<string, string> requestHeaders,
ActivityUndoFollow activity,
string body
)
{
// Validate
var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body);
if (!sigValidation.SignatureIsValidated) return false;
var sigValidation = await ValidateSignature(
activity.actor,
signature,
method,
path,
queryString,
requestHeaders,
body
);
if (!sigValidation.SignatureIsValidated)
return false;
// Save Follow in DB
var followerUserName = sigValidation.User.preferredUsername.ToLowerInvariant();
var followerHost = sigValidation.User.url.Replace("https://", string.Empty).Split('/').First();
var followerHost = sigValidation.User.url
.Replace("https://", string.Empty)
.Split('/')
.First();
//var followerInbox = sigValidation.User.inbox;
var twitterUser = activity.apObject.apObject.Split('/').Last().Replace("@", string.Empty);
var twitterUser = activity.apObject.apObject
.Split('/')
.Last()
.Replace("@", string.Empty);
await _processUndoFollowUser.ExecuteAsync(followerUserName, followerHost, twitterUser);
// Send Accept Activity
@ -310,24 +429,40 @@ namespace BirdsiteLive.Domain
apObject = activity.apObject
}
};
var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject.apObject);
var result = await _activityPubService.PostDataAsync(
acceptFollow,
followerHost,
activity.apObject.apObject
);
return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; //TODO: revamp this for better error handling
}
private async Task<SignatureValidationResult> ValidateSignature(string actor, string rawSig, string method, string path, string queryString, Dictionary<string, string> requestHeaders, string body)
private async Task<SignatureValidationResult> ValidateSignature(
string actor,
string rawSig,
string method,
string path,
string queryString,
Dictionary<string, string> requestHeaders,
string body
)
{
//Check Date Validity
var date = requestHeaders["date"];
var d = DateTime.Parse(date).ToUniversalTime();
var now = DateTime.UtcNow;
var delta = Math.Abs((d - now).TotalSeconds);
if (delta > 30) return new SignatureValidationResult { SignatureIsValidated = false };
if (delta > 30)
return new SignatureValidationResult { SignatureIsValidated = false };
//Check Digest
var digest = requestHeaders["digest"];
var digestHash = digest.Split(new [] {"SHA-256="},StringSplitOptions.RemoveEmptyEntries).LastOrDefault();
var digestHash = digest
.Split(new[] { "SHA-256=" }, StringSplitOptions.RemoveEmptyEntries)
.LastOrDefault();
var calculatedDigestHash = _cryptoService.ComputeSha256Hash(body);
if (digestHash != calculatedDigestHash) return new SignatureValidationResult { SignatureIsValidated = false };
if (digestHash != calculatedDigestHash)
return new SignatureValidationResult { SignatureIsValidated = false };
//Check Signature
var signatures = rawSig.Split(',');
@ -347,15 +482,19 @@ namespace BirdsiteLive.Domain
var remoteUser = await _activityPubService.GetUser(actor);
// Prepare Key data
var toDecode = remoteUser.publicKey.publicKeyPem.Trim().Remove(0, remoteUser.publicKey.publicKeyPem.IndexOf('\n'));
var toDecode = remoteUser.publicKey.publicKeyPem
.Trim()
.Remove(0, remoteUser.publicKey.publicKeyPem.IndexOf('\n'));
toDecode = toDecode.Remove(toDecode.LastIndexOf('\n')).Replace("\n", "");
var signKey = ASN1.ToRSA(Convert.FromBase64String(toDecode));
var toSign = new StringBuilder();
foreach (var headerKey in headers.Split(' '))
{
if (headerKey == "(request-target)") toSign.Append($"(request-target): {method.ToLower()} {path}{queryString}\n");
else toSign.Append($"{headerKey}: {string.Join(", ", requestHeaders[headerKey])}\n");
if (headerKey == "(request-target)")
toSign.Append($"(request-target): {method.ToLower()} {path}{queryString}\n");
else
toSign.Append($"{headerKey}: {string.Join(", ", requestHeaders[headerKey])}\n");
}
toSign.Remove(toSign.Length - 1, 1);
@ -366,7 +505,12 @@ namespace BirdsiteLive.Domain
key.ImportParameters(rsaKeyInfo);
// Trust and Verify
var result = signKey.VerifyData(Encoding.UTF8.GetBytes(toSign.ToString()), sig, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
var result = signKey.VerifyData(
Encoding.UTF8.GetBytes(toSign.ToString()),
sig,
HashAlgorithmName.SHA256,
RSASignaturePadding.Pkcs1
);
return new SignatureValidationResult()
{
@ -376,7 +520,7 @@ namespace BirdsiteLive.Domain
}
}
public class SignatureValidationResult
public class SignatureValidationResult
{
public bool SignatureIsValidated { get; set; }
public Actor User { get; set; }

View File

@ -1,5 +1,6 @@
using System;
using System;
using System.Threading.Tasks;
using BirdsiteLive.ActivityPub;
using BirdsiteLive.ActivityPub.Converters;
using BirdsiteLive.Common.Settings;
@ -21,7 +22,11 @@ namespace BirdsiteLive.Moderation.Actions
private readonly InstanceSettings _instanceSettings;
#region Ctor
public RejectAllFollowingsAction(ITwitterUserDal twitterUserDal, IUserService userService, InstanceSettings instanceSettings)
public RejectAllFollowingsAction(
ITwitterUserDal twitterUserDal,
IUserService userService,
InstanceSettings instanceSettings
)
{
_twitterUserDal = twitterUserDal;
_userService = userService;
@ -49,4 +54,4 @@ namespace BirdsiteLive.Moderation.Actions
}
}
}
}
}

View File

@ -1,5 +1,6 @@
using System;
using System;
using System.Threading.Tasks;
using BirdsiteLive.ActivityPub;
using BirdsiteLive.ActivityPub.Converters;
using BirdsiteLive.Common.Settings;
@ -17,7 +18,7 @@ namespace BirdsiteLive.Moderation.Actions
{
private readonly IUserService _userService;
private readonly InstanceSettings _instanceSettings;
#region Ctor
public RejectFollowingAction(IUserService userService, InstanceSettings instanceSettings)
{
@ -41,4 +42,4 @@ namespace BirdsiteLive.Moderation.Actions
catch (Exception) { }
}
}
}
}

View File

@ -1,6 +1,7 @@
using System;
using System;
using System.Linq;
using System.Threading.Tasks;
using BirdsiteLive.ActivityPub;
using BirdsiteLive.ActivityPub.Converters;
using BirdsiteLive.Common.Settings;
@ -22,7 +23,11 @@ namespace BirdsiteLive.Moderation.Actions
private readonly IRejectAllFollowingsAction _rejectAllFollowingsAction;
#region Ctor
public RemoveFollowerAction(IFollowersDal followersDal, ITwitterUserDal twitterUserDal, IRejectAllFollowingsAction rejectAllFollowingsAction)
public RemoveFollowerAction(
IFollowersDal followersDal,
ITwitterUserDal twitterUserDal,
IRejectAllFollowingsAction rejectAllFollowingsAction
)
{
_followersDal = followersDal;
_twitterUserDal = twitterUserDal;
@ -48,4 +53,4 @@ namespace BirdsiteLive.Moderation.Actions
await _followersDal.DeleteFollowerAsync(follower.Id);
}
}
}
}

View File

@ -1,5 +1,6 @@
using System.Linq;
using System.Linq;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
@ -17,7 +18,11 @@ namespace BirdsiteLive.Moderation.Actions
private readonly IRejectFollowingAction _rejectFollowingAction;
#region Ctor
public RemoveTwitterAccountAction(IFollowersDal followersDal, ITwitterUserDal twitterUserDal, IRejectFollowingAction rejectFollowingAction)
public RemoveTwitterAccountAction(
IFollowersDal followersDal,
ITwitterUserDal twitterUserDal,
IRejectFollowingAction rejectFollowingAction
)
{
_followersDal = followersDal;
_twitterUserDal = twitterUserDal;
@ -27,12 +32,12 @@ namespace BirdsiteLive.Moderation.Actions
public async Task ProcessAsync(SyncTwitterUser twitterUser)
{
// Check Followers
// Check Followers
var twitterUserId = twitterUser.Id;
var followers = await _followersDal.GetFollowersAsync(twitterUserId);
// Remove all Followers
foreach (var follower in followers)
foreach (var follower in followers)
{
// Perform undo following to user instance
await _rejectFollowingAction.ProcessAsync(follower, twitterUser);
@ -54,4 +59,4 @@ namespace BirdsiteLive.Moderation.Actions
await _twitterUserDal.DeleteTwitterUserAsync(twitterUserId);
}
}
}
}

View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<TargetFramework>net6.0</TargetFramework>
</PropertyGroup>
<ItemGroup>

View File

@ -1,7 +1,9 @@
using System;
using System;
using System.Threading.Tasks;
using BirdsiteLive.Domain.Repository;
using BirdsiteLive.Moderation.Processors;
using Microsoft.Extensions.Logging;
namespace BirdsiteLive.Moderation
@ -20,7 +22,12 @@ namespace BirdsiteLive.Moderation
private readonly ILogger<ModerationPipeline> _logger;
#region Ctor
public ModerationPipeline(IModerationRepository moderationRepository, IFollowerModerationProcessor followerModerationProcessor, ITwitterAccountModerationProcessor twitterAccountModerationProcessor, ILogger<ModerationPipeline> logger)
public ModerationPipeline(
IModerationRepository moderationRepository,
IFollowerModerationProcessor followerModerationProcessor,
ITwitterAccountModerationProcessor twitterAccountModerationProcessor,
ILogger<ModerationPipeline> logger
)
{
_moderationRepository = moderationRepository;
_followerModerationProcessor = followerModerationProcessor;
@ -44,16 +51,22 @@ namespace BirdsiteLive.Moderation
private async Task CheckFollowerModerationPolicyAsync()
{
var followerPolicy = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.Follower);
if (followerPolicy == ModerationTypeEnum.None) return;
var followerPolicy = _moderationRepository.GetModerationType(
ModerationEntityTypeEnum.Follower
);
if (followerPolicy == ModerationTypeEnum.None)
return;
await _followerModerationProcessor.ProcessAsync(followerPolicy);
}
private async Task CheckTwitterAccountModerationPolicyAsync()
{
var twitterAccountPolicy = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.TwitterAccount);
if (twitterAccountPolicy == ModerationTypeEnum.None) return;
var twitterAccountPolicy = _moderationRepository.GetModerationType(
ModerationEntityTypeEnum.TwitterAccount
);
if (twitterAccountPolicy == ModerationTypeEnum.None)
return;
await _twitterAccountModerationProcessor.ProcessAsync(twitterAccountPolicy);
}

View File

@ -1,5 +1,6 @@
using System;
using System;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.Domain.Repository;
using BirdsiteLive.Moderation.Actions;
@ -18,7 +19,11 @@ namespace BirdsiteLive.Moderation.Processors
private readonly IRemoveFollowerAction _removeFollowerAction;
#region Ctor
public FollowerModerationProcessor(IFollowersDal followersDal, IModerationRepository moderationRepository, IRemoveFollowerAction removeFollowerAction)
public FollowerModerationProcessor(
IFollowersDal followersDal,
IModerationRepository moderationRepository,
IRemoveFollowerAction removeFollowerAction
)
{
_followersDal = followersDal;
_moderationRepository = moderationRepository;
@ -28,17 +33,26 @@ namespace BirdsiteLive.Moderation.Processors
public async Task ProcessAsync(ModerationTypeEnum type)
{
if (type == ModerationTypeEnum.None) return;
if (type == ModerationTypeEnum.None)
return;
var followers = await _followersDal.GetAllFollowersAsync();
foreach (var follower in followers)
{
var followerHandle = $"@{follower.Acct.Trim()}@{follower.Host.Trim()}".ToLowerInvariant();
var status = _moderationRepository.CheckStatus(ModerationEntityTypeEnum.Follower, followerHandle);
var followerHandle =
$"@{follower.Acct.Trim()}@{follower.Host.Trim()}".ToLowerInvariant();
var status = _moderationRepository.CheckStatus(
ModerationEntityTypeEnum.Follower,
followerHandle
);
if (type == ModerationTypeEnum.WhiteListing && status != ModeratedTypeEnum.WhiteListed ||
type == ModerationTypeEnum.BlackListing && status == ModeratedTypeEnum.BlackListed)
if (
type == ModerationTypeEnum.WhiteListing
&& status != ModeratedTypeEnum.WhiteListed
|| type == ModerationTypeEnum.BlackListing
&& status == ModeratedTypeEnum.BlackListed
)
{
Console.WriteLine($"Remove {followerHandle}");
await _removeFollowerAction.ProcessAsync(follower);
@ -46,4 +60,4 @@ namespace BirdsiteLive.Moderation.Processors
}
}
}
}
}

View File

@ -1,4 +1,5 @@
using System.Threading.Tasks;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.Domain.Repository;
using BirdsiteLive.Moderation.Actions;
@ -15,9 +16,13 @@ namespace BirdsiteLive.Moderation.Processors
private readonly ITwitterUserDal _twitterUserDal;
private readonly IModerationRepository _moderationRepository;
private readonly IRemoveTwitterAccountAction _removeTwitterAccountAction;
#region Ctor
public TwitterAccountModerationProcessor(ITwitterUserDal twitterUserDal, IModerationRepository moderationRepository, IRemoveTwitterAccountAction removeTwitterAccountAction)
public TwitterAccountModerationProcessor(
ITwitterUserDal twitterUserDal,
IModerationRepository moderationRepository,
IRemoveTwitterAccountAction removeTwitterAccountAction
)
{
_twitterUserDal = twitterUserDal;
_moderationRepository = moderationRepository;
@ -27,19 +32,27 @@ namespace BirdsiteLive.Moderation.Processors
public async Task ProcessAsync(ModerationTypeEnum type)
{
if (type == ModerationTypeEnum.None) return;
if (type == ModerationTypeEnum.None)
return;
var twitterUsers = await _twitterUserDal.GetAllTwitterUsersAsync();
foreach (var user in twitterUsers)
{
var userHandle = user.Acct.ToLowerInvariant().Trim();
var status = _moderationRepository.CheckStatus(ModerationEntityTypeEnum.TwitterAccount, userHandle);
var status = _moderationRepository.CheckStatus(
ModerationEntityTypeEnum.TwitterAccount,
userHandle
);
if (type == ModerationTypeEnum.WhiteListing && status != ModeratedTypeEnum.WhiteListed ||
type == ModerationTypeEnum.BlackListing && status == ModeratedTypeEnum.BlackListed)
if (
type == ModerationTypeEnum.WhiteListing
&& status != ModeratedTypeEnum.WhiteListed
|| type == ModerationTypeEnum.BlackListing
&& status == ModeratedTypeEnum.BlackListed
)
await _removeTwitterAccountAction.ProcessAsync(user);
}
}
}
}
}

View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<TargetFramework>net6.0</TargetFramework>
<LangVersion>latest</LangVersion>
</PropertyGroup>

View File

@ -1,5 +1,6 @@
using System.Threading;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Pipeline.Models;
@ -7,6 +8,9 @@ namespace BirdsiteLive.Pipeline.Contracts
{
public interface IRefreshTwitterUserStatusProcessor
{
Task<UserWithDataToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct);
Task<UserWithDataToSync[]> ProcessAsync(
SyncTwitterUser[] syncTwitterUsers,
CancellationToken ct
);
}
}
}

View File

@ -1,13 +1,17 @@
using System.Collections.Generic;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.Pipeline.Models;
namespace BirdsiteLive.Pipeline.Contracts
{
public interface IRetrieveFollowersProcessor
{
Task<IEnumerable<UserWithDataToSync>> ProcessAsync(UserWithDataToSync[] userWithTweetsToSyncs, CancellationToken ct);
Task<IEnumerable<UserWithDataToSync>> ProcessAsync(
UserWithDataToSync[] userWithTweetsToSyncs,
CancellationToken ct
);
//IAsyncEnumerable<UserWithTweetsToSync> ProcessAsync(UserWithTweetsToSync[] userWithTweetsToSyncs, CancellationToken ct);
}
}
}

View File

@ -1,5 +1,6 @@
using System.Threading;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Pipeline.Models;
@ -7,6 +8,9 @@ namespace BirdsiteLive.Pipeline.Contracts
{
public interface IRetrieveTweetsProcessor
{
Task<UserWithDataToSync[]> ProcessAsync(UserWithDataToSync[] syncTwitterUsers, CancellationToken ct);
Task<UserWithDataToSync[]> ProcessAsync(
UserWithDataToSync[] syncTwitterUsers,
CancellationToken ct
);
}
}
}

View File

@ -1,12 +1,16 @@
using System.Threading;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using BirdsiteLive.DAL.Models;
namespace BirdsiteLive.Pipeline.Contracts
{
public interface IRetrieveTwitterUsersProcessor
{
Task GetTwitterUsersAsync(BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock, CancellationToken ct);
Task GetTwitterUsersAsync(
BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock,
CancellationToken ct
);
}
}
}

View File

@ -1,5 +1,6 @@
using System.Threading;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.Pipeline.Models;
namespace BirdsiteLive.Pipeline.Contracts
@ -8,4 +9,4 @@ namespace BirdsiteLive.Pipeline.Contracts
{
Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct);
}
}
}

View File

@ -1,11 +1,15 @@
using System.Threading;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.Pipeline.Models;
namespace BirdsiteLive.Pipeline.Contracts
{
public interface ISendTweetsToFollowersProcessor
{
Task<UserWithDataToSync> ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct);
Task<UserWithDataToSync> ProcessAsync(
UserWithDataToSync userWithTweetsToSync,
CancellationToken ct
);
}
}
}

View File

@ -1,5 +1,6 @@
using BirdsiteLive.DAL.Models;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Twitter.Models;
using Tweetinvi.Models;
namespace BirdsiteLive.Pipeline.Models
@ -10,4 +11,4 @@ namespace BirdsiteLive.Pipeline.Models
public ExtractedTweet[] Tweets { get; set; }
public Follower[] Followers { get; set; }
}
}
}

View File

@ -1,7 +1,8 @@
using System;
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
@ -20,7 +21,12 @@ namespace BirdsiteLive.Pipeline.Processors
private readonly InstanceSettings _instanceSettings;
#region Ctor
public RefreshTwitterUserStatusProcessor(ICachedTwitterUserService twitterUserService, ITwitterUserDal twitterUserDal, IRemoveTwitterAccountAction removeTwitterAccountAction, InstanceSettings instanceSettings)
public RefreshTwitterUserStatusProcessor(
ICachedTwitterUserService twitterUserService,
ITwitterUserDal twitterUserDal,
IRemoveTwitterAccountAction removeTwitterAccountAction,
InstanceSettings instanceSettings
)
{
_twitterUserService = twitterUserService;
_twitterUserDal = twitterUserDal;
@ -29,7 +35,10 @@ namespace BirdsiteLive.Pipeline.Processors
}
#endregion
public async Task<UserWithDataToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct)
public async Task<UserWithDataToSync[]> ProcessAsync(
SyncTwitterUser[] syncTwitterUsers,
CancellationToken ct
)
{
var usersWtData = new List<UserWithDataToSync>();
@ -43,10 +52,7 @@ namespace BirdsiteLive.Pipeline.Processors
else if (!userView.Protected)
{
user.FetchingErrorCount = 0;
var userWtData = new UserWithDataToSync
{
User = user
};
var userWtData = new UserWithDataToSync { User = user };
usersWtData.Add(userWtData);
}
}
@ -73,4 +79,4 @@ namespace BirdsiteLive.Pipeline.Processors
_twitterUserService.PurgeUser(user.Acct);
}
}
}
}

View File

@ -1,6 +1,7 @@
using System.Collections.Generic;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.Pipeline.Contracts;
using BirdsiteLive.Pipeline.Models;
@ -18,7 +19,10 @@ namespace BirdsiteLive.Pipeline.Processors
}
#endregion
public async Task<IEnumerable<UserWithDataToSync>> ProcessAsync(UserWithDataToSync[] userWithTweetsToSyncs, CancellationToken ct)
public async Task<IEnumerable<UserWithDataToSync>> ProcessAsync(
UserWithDataToSync[] userWithTweetsToSyncs,
CancellationToken ct
)
{
//TODO multithread this
foreach (var user in userWithTweetsToSyncs)
@ -30,4 +34,4 @@ namespace BirdsiteLive.Pipeline.Processors
return userWithTweetsToSyncs;
}
}
}
}

View File

@ -1,15 +1,18 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Pipeline.Contracts;
using BirdsiteLive.Pipeline.Models;
using BirdsiteLive.Twitter;
using BirdsiteLive.Twitter.Models;
using Microsoft.Extensions.Logging;
using Tweetinvi.Models;
namespace BirdsiteLive.Pipeline.Processors
@ -22,7 +25,12 @@ namespace BirdsiteLive.Pipeline.Processors
private readonly ILogger<RetrieveTweetsProcessor> _logger;
#region Ctor
public RetrieveTweetsProcessor(ITwitterTweetsService twitterTweetsService, ITwitterUserDal twitterUserDal, ICachedTwitterUserService twitterUserService, ILogger<RetrieveTweetsProcessor> logger)
public RetrieveTweetsProcessor(
ITwitterTweetsService twitterTweetsService,
ITwitterUserDal twitterUserDal,
ICachedTwitterUserService twitterUserService,
ILogger<RetrieveTweetsProcessor> logger
)
{
_twitterTweetsService = twitterTweetsService;
_twitterUserDal = twitterUserDal;
@ -31,7 +39,10 @@ namespace BirdsiteLive.Pipeline.Processors
}
#endregion
public async Task<UserWithDataToSync[]> ProcessAsync(UserWithDataToSync[] syncTwitterUsers, CancellationToken ct)
public async Task<UserWithDataToSync[]> ProcessAsync(
UserWithDataToSync[] syncTwitterUsers,
CancellationToken ct
)
{
var usersWtTweets = new List<UserWithDataToSync>();
@ -49,12 +60,24 @@ namespace BirdsiteLive.Pipeline.Processors
{
var tweetId = tweets.Last().Id;
var now = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now);
await _twitterUserDal.UpdateTwitterUserAsync(
user.Id,
tweetId,
tweetId,
user.FetchingErrorCount,
now
);
}
else
{
var now = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now);
await _twitterUserDal.UpdateTwitterUserAsync(
user.Id,
user.LastTweetPostedId,
user.LastTweetSynchronizedForAllFollowersId,
user.FetchingErrorCount,
now
);
}
}
@ -64,21 +87,30 @@ namespace BirdsiteLive.Pipeline.Processors
private ExtractedTweet[] RetrieveNewTweets(SyncTwitterUser user)
{
var tweets = new ExtractedTweet[0];
try
{
if (user.LastTweetPostedId == -1)
tweets = _twitterTweetsService.GetTimeline(user.Acct, 1);
else
tweets = _twitterTweetsService.GetTimeline(user.Acct, 200, user.LastTweetSynchronizedForAllFollowersId);
tweets = _twitterTweetsService.GetTimeline(
user.Acct,
200,
user.LastTweetSynchronizedForAllFollowersId
);
}
catch (Exception e)
{
_logger.LogError(e, "Error retrieving TL of {Username} from {LastTweetPostedId}, purging user from cache", user.Acct, user.LastTweetPostedId);
_logger.LogError(
e,
"Error retrieving TL of {Username} from {LastTweetPostedId}, purging user from cache",
user.Acct,
user.LastTweetPostedId
);
_twitterUserService.PurgeUser(user.Acct);
}
return tweets;
}
}
}
}

View File

@ -1,14 +1,16 @@
using System;
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using BirdsiteLive.Common.Extensions;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Pipeline.Contracts;
using BirdsiteLive.Pipeline.Tools;
using Microsoft.Extensions.Logging;
namespace BirdsiteLive.Pipeline.Processors
@ -18,11 +20,14 @@ namespace BirdsiteLive.Pipeline.Processors
private readonly ITwitterUserDal _twitterUserDal;
private readonly IMaxUsersNumberProvider _maxUsersNumberProvider;
private readonly ILogger<RetrieveTwitterUsersProcessor> _logger;
public int WaitFactor = 1000 * 60; //1 min
public int WaitFactor = 1000 * 60; //1 min
#region Ctor
public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IMaxUsersNumberProvider maxUsersNumberProvider, ILogger<RetrieveTwitterUsersProcessor> logger)
public RetrieveTwitterUsersProcessor(
ITwitterUserDal twitterUserDal,
IMaxUsersNumberProvider maxUsersNumberProvider,
ILogger<RetrieveTwitterUsersProcessor> logger
)
{
_twitterUserDal = twitterUserDal;
_maxUsersNumberProvider = maxUsersNumberProvider;
@ -30,7 +35,10 @@ namespace BirdsiteLive.Pipeline.Processors
}
#endregion
public async Task GetTwitterUsersAsync(BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock, CancellationToken ct)
public async Task GetTwitterUsersAsync(
BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock,
CancellationToken ct
)
{
for (; ; )
{
@ -42,7 +50,7 @@ namespace BirdsiteLive.Pipeline.Processors
var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber);
var userCount = users.Any() ? users.Length : 1;
var splitNumber = (int) Math.Ceiling(userCount / 15d);
var splitNumber = (int)Math.Ceiling(userCount / 15d);
var splitUsers = users.Split(splitNumber).ToList();
foreach (var u in splitUsers)
@ -55,7 +63,8 @@ namespace BirdsiteLive.Pipeline.Processors
}
var splitCount = splitUsers.Count();
if (splitCount < 15) await Task.Delay((15 - splitCount) * WaitFactor, ct); //Always wait 15min
if (splitCount < 15)
await Task.Delay((15 - splitCount) * WaitFactor, ct); //Always wait 15min
//// Extra wait time to fit 100.000/day limit
//var extraWaitTime = (int)Math.Ceiling((60 / ((100000d / 24) / userCount)) - 15);
@ -69,4 +78,4 @@ namespace BirdsiteLive.Pipeline.Processors
}
}
}
}
}

View File

@ -1,10 +1,12 @@
using System;
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.Pipeline.Contracts;
using BirdsiteLive.Pipeline.Models;
using Microsoft.Extensions.Logging;
namespace BirdsiteLive.Pipeline.Processors
@ -15,14 +17,20 @@ namespace BirdsiteLive.Pipeline.Processors
private readonly ILogger<SaveProgressionProcessor> _logger;
#region Ctor
public SaveProgressionProcessor(ITwitterUserDal twitterUserDal, ILogger<SaveProgressionProcessor> logger)
public SaveProgressionProcessor(
ITwitterUserDal twitterUserDal,
ILogger<SaveProgressionProcessor> logger
)
{
_twitterUserDal = twitterUserDal;
_logger = logger;
}
#endregion
public async Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
public async Task ProcessAsync(
UserWithDataToSync userWithTweetsToSync,
CancellationToken ct
)
{
try
{
@ -31,25 +39,40 @@ namespace BirdsiteLive.Pipeline.Processors
_logger.LogWarning("No tweets synchronized");
return;
}
if(userWithTweetsToSync.Followers.Length == 0)
if (userWithTweetsToSync.Followers.Length == 0)
{
_logger.LogWarning("No Followers found for {User}", userWithTweetsToSync.User.Acct);
_logger.LogWarning(
"No Followers found for {User}",
userWithTweetsToSync.User.Acct
);
return;
}
var userId = userWithTweetsToSync.User.Id;
var followingSyncStatuses = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).ToList();
var followingSyncStatuses = userWithTweetsToSync.Followers
.Select(x => x.FollowingsSyncStatus[userId])
.ToList();
if (followingSyncStatuses.Count == 0)
{
_logger.LogWarning("No Followers sync found for {User}, Id: {UserId}", userWithTweetsToSync.User.Acct, userId);
_logger.LogWarning(
"No Followers sync found for {User}, Id: {UserId}",
userWithTweetsToSync.User.Acct,
userId
);
return;
}
var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();
var minimumSync = followingSyncStatuses.Min();
var now = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now);
await _twitterUserDal.UpdateTwitterUserAsync(
userId,
lastPostedTweet,
minimumSync,
userWithTweetsToSync.User.FetchingErrorCount,
now
);
}
catch (Exception e)
{
@ -58,4 +81,4 @@ namespace BirdsiteLive.Pipeline.Processors
}
}
}
}
}

View File

@ -1,10 +1,11 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Domain;
@ -13,7 +14,9 @@ using BirdsiteLive.Pipeline.Models;
using BirdsiteLive.Pipeline.Processors.SubTasks;
using BirdsiteLive.Twitter;
using BirdsiteLive.Twitter.Models;
using Microsoft.Extensions.Logging;
using Tweetinvi.Models;
namespace BirdsiteLive.Pipeline.Processors
@ -26,7 +29,12 @@ namespace BirdsiteLive.Pipeline.Processors
private readonly ILogger<SendTweetsToFollowersProcessor> _logger;
#region Ctor
public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox, IFollowersDal followersDal, ILogger<SendTweetsToFollowersProcessor> logger)
public SendTweetsToFollowersProcessor(
ISendTweetsToInboxTask sendTweetsToInboxTask,
ISendTweetsToSharedInboxTask sendTweetsToSharedInbox,
IFollowersDal followersDal,
ILogger<SendTweetsToFollowersProcessor> logger
)
{
_sendTweetsToInboxTask = sendTweetsToInboxTask;
_sendTweetsToSharedInbox = sendTweetsToSharedInbox;
@ -35,7 +43,10 @@ namespace BirdsiteLive.Pipeline.Processors
}
#endregion
public async Task<UserWithDataToSync> ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
public async Task<UserWithDataToSync> ProcessAsync(
UserWithDataToSync userWithTweetsToSync,
CancellationToken ct
)
{
var user = userWithTweetsToSync.User;
@ -43,18 +54,30 @@ namespace BirdsiteLive.Pipeline.Processors
var followersWtSharedInbox = userWithTweetsToSync.Followers
.Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
.ToList();
await ProcessFollowersWithSharedInboxAsync(userWithTweetsToSync.Tweets, followersWtSharedInbox, user);
await ProcessFollowersWithSharedInboxAsync(
userWithTweetsToSync.Tweets,
followersWtSharedInbox,
user
);
// Process Inbox
var followerWtInbox = userWithTweetsToSync.Followers
.Where(x => string.IsNullOrWhiteSpace(x.SharedInboxRoute))
.ToList();
await ProcessFollowersWithInboxAsync(userWithTweetsToSync.Tweets, followerWtInbox, user);
await ProcessFollowersWithInboxAsync(
userWithTweetsToSync.Tweets,
followerWtInbox,
user
);
return userWithTweetsToSync;
}
private async Task ProcessFollowersWithSharedInboxAsync(ExtractedTweet[] tweets, List<Follower> followers, SyncTwitterUser user)
private async Task ProcessFollowersWithSharedInboxAsync(
ExtractedTweet[] tweets,
List<Follower> followers,
SyncTwitterUser user
)
{
var followersPerInstances = followers.GroupBy(x => x.Host);
@ -62,7 +85,12 @@ namespace BirdsiteLive.Pipeline.Processors
{
try
{
await _sendTweetsToSharedInbox.ExecuteAsync(tweets, user, followersPerInstance.Key, followersPerInstance.ToArray());
await _sendTweetsToSharedInbox.ExecuteAsync(
tweets,
user,
followersPerInstance.Key,
followersPerInstance.ToArray()
);
foreach (var f in followersPerInstance)
await ProcessWorkingUserAsync(f);
@ -70,15 +98,24 @@ namespace BirdsiteLive.Pipeline.Processors
catch (Exception e)
{
var follower = followersPerInstance.First();
_logger.LogError(e, "Posting to {Host}{Route} failed", follower.Host, follower.SharedInboxRoute);
_logger.LogError(
e,
"Posting to {Host}{Route} failed",
follower.Host,
follower.SharedInboxRoute
);
foreach (var f in followersPerInstance)
await ProcessFailingUserAsync(f);
}
}
}
private async Task ProcessFollowersWithInboxAsync(ExtractedTweet[] tweets, List<Follower> followerWtInbox, SyncTwitterUser user)
private async Task ProcessFollowersWithInboxAsync(
ExtractedTweet[] tweets,
List<Follower> followerWtInbox,
SyncTwitterUser user
)
{
foreach (var follower in followerWtInbox)
{
@ -89,7 +126,12 @@ namespace BirdsiteLive.Pipeline.Processors
}
catch (Exception e)
{
_logger.LogError(e, "Posting to {Host}{Route} failed", follower.Host, follower.InboxRoute);
_logger.LogError(
e,
"Posting to {Host}{Route} failed",
follower.Host,
follower.InboxRoute
);
await ProcessFailingUserAsync(follower);
}
}
@ -110,4 +152,4 @@ namespace BirdsiteLive.Pipeline.Processors
await _followersDal.UpdateFollowerAsync(follower);
}
}
}
}

View File

@ -1,20 +1,26 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Domain;
using BirdsiteLive.Twitter.Models;
using Microsoft.Extensions.Logging;
namespace BirdsiteLive.Pipeline.Processors.SubTasks
{
public interface ISendTweetsToInboxTask
{
Task ExecuteAsync(IEnumerable<ExtractedTweet> tweets, Follower follower, SyncTwitterUser user);
Task ExecuteAsync(
IEnumerable<ExtractedTweet> tweets,
Follower follower,
SyncTwitterUser user
);
}
public class SendTweetsToInboxTask : ISendTweetsToInboxTask
@ -25,9 +31,14 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
private readonly InstanceSettings _settings;
private readonly ILogger<SendTweetsToInboxTask> _logger;
#region Ctor
public SendTweetsToInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger<SendTweetsToInboxTask> logger)
public SendTweetsToInboxTask(
IActivityPubService activityPubService,
IStatusService statusService,
IFollowersDal followersDal,
InstanceSettings settings,
ILogger<SendTweetsToInboxTask> logger
)
{
_activityPubService = activityPubService;
_statusService = statusService;
@ -37,14 +48,15 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
}
#endregion
public async Task ExecuteAsync(IEnumerable<ExtractedTweet> tweets, Follower follower, SyncTwitterUser user)
public async Task ExecuteAsync(
IEnumerable<ExtractedTweet> tweets,
Follower follower,
SyncTwitterUser user
)
{
var userId = user.Id;
var fromStatusId = follower.FollowingsSyncStatus[userId];
var tweetsToSend = tweets
.Where(x => x.Id > fromStatusId)
.OrderBy(x => x.Id)
.ToList();
var tweetsToSend = tweets.Where(x => x.Id > fromStatusId).OrderBy(x => x.Id).ToList();
var inbox = follower.InboxRoute;
@ -55,19 +67,34 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
{
try
{
if (!tweet.IsReply ||
tweet.IsReply && tweet.IsThread ||
_settings.PublishReplies)
if (
!tweet.IsReply
|| tweet.IsReply && tweet.IsThread
|| _settings.PublishReplies
)
{
var note = _statusService.GetStatus(user.Acct, tweet);
await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox);
await _activityPubService.PostNewNoteActivity(
note,
user.Acct,
tweet.Id.ToString(),
follower.Host,
inbox
);
}
}
catch (ArgumentException e)
{
if (e.Message.Contains("Invalid pattern") && e.Message.Contains("at offset")) //Regex exception
if (
e.Message.Contains("Invalid pattern") && e.Message.Contains("at offset")
) //Regex exception
{
_logger.LogError(e, "Can't parse {MessageContent} from Tweet {Id}", tweet.MessageContent, tweet.Id);
_logger.LogError(
e,
"Can't parse {MessageContent} from Tweet {Id}",
tweet.MessageContent,
tweet.Id
);
}
else
{
@ -88,4 +115,4 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
}
}
}
}
}

View File

@ -1,19 +1,26 @@
using System;
using System;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Domain;
using BirdsiteLive.Twitter.Models;
using Microsoft.Extensions.Logging;
namespace BirdsiteLive.Pipeline.Processors.SubTasks
{
public interface ISendTweetsToSharedInboxTask
{
Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, string host, Follower[] followersPerInstance);
Task ExecuteAsync(
ExtractedTweet[] tweets,
SyncTwitterUser user,
string host,
Follower[] followersPerInstance
);
}
public class SendTweetsToSharedInboxTask : ISendTweetsToSharedInboxTask
@ -25,7 +32,13 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
private readonly ILogger<SendTweetsToSharedInboxTask> _logger;
#region Ctor
public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger<SendTweetsToSharedInboxTask> logger)
public SendTweetsToSharedInboxTask(
IActivityPubService activityPubService,
IStatusService statusService,
IFollowersDal followersDal,
InstanceSettings settings,
ILogger<SendTweetsToSharedInboxTask> logger
)
{
_activityPubService = activityPubService;
_statusService = statusService;
@ -35,18 +48,19 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
}
#endregion
public async Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, string host, Follower[] followersPerInstance)
public async Task ExecuteAsync(
ExtractedTweet[] tweets,
SyncTwitterUser user,
string host,
Follower[] followersPerInstance
)
{
var userId = user.Id;
var inbox = followersPerInstance.First().SharedInboxRoute;
var fromStatusId = followersPerInstance
.Max(x => x.FollowingsSyncStatus[userId]);
var fromStatusId = followersPerInstance.Max(x => x.FollowingsSyncStatus[userId]);
var tweetsToSend = tweets
.Where(x => x.Id > fromStatusId)
.OrderBy(x => x.Id)
.ToList();
var tweetsToSend = tweets.Where(x => x.Id > fromStatusId).OrderBy(x => x.Id).ToList();
var syncStatus = fromStatusId;
try
@ -55,19 +69,34 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
{
try
{
if (!tweet.IsReply ||
tweet.IsReply && tweet.IsThread ||
_settings.PublishReplies)
if (
!tweet.IsReply
|| tweet.IsReply && tweet.IsThread
|| _settings.PublishReplies
)
{
var note = _statusService.GetStatus(user.Acct, tweet);
await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox);
await _activityPubService.PostNewNoteActivity(
note,
user.Acct,
tweet.Id.ToString(),
host,
inbox
);
}
}
catch (ArgumentException e)
{
if (e.Message.Contains("Invalid pattern") && e.Message.Contains("at offset")) //Regex exception
if (
e.Message.Contains("Invalid pattern") && e.Message.Contains("at offset")
) //Regex exception
{
_logger.LogError(e, "Can't parse {MessageContent} from Tweet {Id}", tweet.MessageContent, tweet.Id);
_logger.LogError(
e,
"Can't parse {MessageContent} from Tweet {Id}",
tweet.MessageContent,
tweet.Id
);
}
else
{
@ -91,4 +120,4 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
}
}
}
}
}

View File

@ -1,11 +1,13 @@
using System;
using System;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Pipeline.Contracts;
using BirdsiteLive.Pipeline.Models;
using Microsoft.Extensions.Logging;
namespace BirdsiteLive.Pipeline
@ -26,7 +28,15 @@ namespace BirdsiteLive.Pipeline
private readonly ILogger<StatusPublicationPipeline> _logger;
#region Ctor
public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ISaveProgressionProcessor saveProgressionProcessor, IRefreshTwitterUserStatusProcessor refreshTwitterUserStatusProcessor, ILogger<StatusPublicationPipeline> logger)
public StatusPublicationPipeline(
IRetrieveTweetsProcessor retrieveTweetsProcessor,
IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor,
IRetrieveFollowersProcessor retrieveFollowersProcessor,
ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor,
ISaveProgressionProcessor saveProgressionProcessor,
IRefreshTwitterUserStatusProcessor refreshTwitterUserStatusProcessor,
ILogger<StatusPublicationPipeline> logger
)
{
_retrieveTweetsProcessor = retrieveTweetsProcessor;
_retrieveTwitterAccountsProcessor = retrieveTwitterAccountsProcessor;
@ -41,37 +51,107 @@ namespace BirdsiteLive.Pipeline
public async Task ExecuteAsync(CancellationToken ct)
{
// Create blocks
var twitterUserToRefreshBufferBlock = new BufferBlock<SyncTwitterUser[]>(new DataflowBlockOptions
{ BoundedCapacity = 1, CancellationToken = ct });
var twitterUserToRefreshBlock = new TransformBlock<SyncTwitterUser[], UserWithDataToSync[]>(async x => await _refreshTwitterUserStatusProcessor.ProcessAsync(x, ct));
var twitterUsersBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
var retrieveTweetsBlock = new TransformBlock<UserWithDataToSync[], UserWithDataToSync[]>(async x => await _retrieveTweetsProcessor.ProcessAsync(x, ct));
var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
var retrieveFollowersBlock = new TransformManyBlock<UserWithDataToSync[], UserWithDataToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct));
var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
var sendTweetsToFollowersBlock = new TransformBlock<UserWithDataToSync, UserWithDataToSync>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
var sendTweetsToFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
var saveProgressionBlock = new ActionBlock<UserWithDataToSync>(async x => await _saveProgressionProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
// Create blocks
var twitterUserToRefreshBufferBlock = new BufferBlock<SyncTwitterUser[]>(
new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct }
);
var twitterUserToRefreshBlock = new TransformBlock<
SyncTwitterUser[],
UserWithDataToSync[]
>(async x => await _refreshTwitterUserStatusProcessor.ProcessAsync(x, ct));
var twitterUsersBufferBlock = new BufferBlock<UserWithDataToSync[]>(
new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct }
);
var retrieveTweetsBlock = new TransformBlock<
UserWithDataToSync[],
UserWithDataToSync[]
>(async x => await _retrieveTweetsProcessor.ProcessAsync(x, ct));
var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(
new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct }
);
var retrieveFollowersBlock = new TransformManyBlock<
UserWithDataToSync[],
UserWithDataToSync
>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct));
var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(
new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct }
);
var sendTweetsToFollowersBlock = new TransformBlock<
UserWithDataToSync,
UserWithDataToSync
>(
async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 5,
CancellationToken = ct
}
);
var sendTweetsToFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(
new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct }
);
var saveProgressionBlock = new ActionBlock<UserWithDataToSync>(
async x => await _saveProgressionProcessor.ProcessAsync(x, ct),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 5,
CancellationToken = ct
}
);
// Link pipeline
twitterUserToRefreshBufferBlock.LinkTo(twitterUserToRefreshBlock, new DataflowLinkOptions { PropagateCompletion = true });
twitterUserToRefreshBlock.LinkTo(twitterUsersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
twitterUsersBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions { PropagateCompletion = true });
retrieveTweetsBlock.LinkTo(retrieveTweetsBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
retrieveTweetsBufferBlock.LinkTo(retrieveFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
retrieveFollowersBlock.LinkTo(retrieveFollowersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
retrieveFollowersBufferBlock.LinkTo(sendTweetsToFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
sendTweetsToFollowersBlock.LinkTo(sendTweetsToFollowersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
sendTweetsToFollowersBufferBlock.LinkTo(saveProgressionBlock, new DataflowLinkOptions { PropagateCompletion = true });
twitterUserToRefreshBufferBlock.LinkTo(
twitterUserToRefreshBlock,
new DataflowLinkOptions { PropagateCompletion = true }
);
twitterUserToRefreshBlock.LinkTo(
twitterUsersBufferBlock,
new DataflowLinkOptions { PropagateCompletion = true }
);
twitterUsersBufferBlock.LinkTo(
retrieveTweetsBlock,
new DataflowLinkOptions { PropagateCompletion = true }
);
retrieveTweetsBlock.LinkTo(
retrieveTweetsBufferBlock,
new DataflowLinkOptions { PropagateCompletion = true }
);
retrieveTweetsBufferBlock.LinkTo(
retrieveFollowersBlock,
new DataflowLinkOptions { PropagateCompletion = true }
);
retrieveFollowersBlock.LinkTo(
retrieveFollowersBufferBlock,
new DataflowLinkOptions { PropagateCompletion = true }
);
retrieveFollowersBufferBlock.LinkTo(
sendTweetsToFollowersBlock,
new DataflowLinkOptions { PropagateCompletion = true }
);
sendTweetsToFollowersBlock.LinkTo(
sendTweetsToFollowersBufferBlock,
new DataflowLinkOptions { PropagateCompletion = true }
);
sendTweetsToFollowersBufferBlock.LinkTo(
saveProgressionBlock,
new DataflowLinkOptions { PropagateCompletion = true }
);
// Launch twitter user retriever
var retrieveTwitterAccountsTask = _retrieveTwitterAccountsProcessor.GetTwitterUsersAsync(twitterUserToRefreshBufferBlock, ct);
var retrieveTwitterAccountsTask =
_retrieveTwitterAccountsProcessor.GetTwitterUsersAsync(
twitterUserToRefreshBufferBlock,
ct
);
// Wait
await Task.WhenAny(new[] { retrieveTwitterAccountsTask, saveProgressionBlock.Completion });
await Task.WhenAny(
new[] { retrieveTwitterAccountsTask, saveProgressionBlock.Completion }
);
var ex = retrieveTwitterAccountsTask.IsFaulted ? retrieveTwitterAccountsTask.Exception : saveProgressionBlock.Completion.Exception;
var ex = retrieveTwitterAccountsTask.IsFaulted
? retrieveTwitterAccountsTask.Exception
: saveProgressionBlock.Completion.Exception;
_logger.LogCritical(ex, "An error occurred, pipeline stopped");
}
}

View File

@ -1,4 +1,5 @@
using System.Threading.Tasks;
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
@ -13,13 +14,16 @@ namespace BirdsiteLive.Pipeline.Tools
{
private readonly InstanceSettings _instanceSettings;
private readonly ITwitterUserDal _twitterUserDal;
private int _totalUsersCount = -1;
private int _warmUpIterations;
private const int WarmUpMaxCapacity = 200;
#region Ctor
public MaxUsersNumberProvider(InstanceSettings instanceSettings, ITwitterUserDal twitterUserDal)
public MaxUsersNumberProvider(
InstanceSettings instanceSettings,
ITwitterUserDal twitterUserDal
)
{
_instanceSettings = instanceSettings;
_twitterUserDal = twitterUserDal;
@ -29,21 +33,21 @@ namespace BirdsiteLive.Pipeline.Tools
public async Task<int> GetMaxUsersNumberAsync()
{
// Init data
if (_totalUsersCount == -1)
if (_totalUsersCount == -1)
{
_totalUsersCount = await _twitterUserDal.GetTwitterUsersCountAsync();
_warmUpIterations = (int)(_totalUsersCount / (float)WarmUpMaxCapacity);
}
// Return if warm up ended
if (_warmUpIterations <= 0) return _instanceSettings.MaxUsersCapacity;
if (_warmUpIterations <= 0)
return _instanceSettings.MaxUsersCapacity;
// Calculate warm up value
var maxUsers = _warmUpIterations > 0
? WarmUpMaxCapacity
: _instanceSettings.MaxUsersCapacity;
var maxUsers =
_warmUpIterations > 0 ? WarmUpMaxCapacity : _instanceSettings.MaxUsersCapacity;
_warmUpIterations--;
return maxUsers;
}
}
}
}

View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<TargetFramework>net6.0</TargetFramework>
</PropertyGroup>
<ItemGroup>

View File

@ -1,5 +1,7 @@
using System;
using System;
using BirdsiteLive.Twitter.Models;
using Microsoft.Extensions.Caching.Memory;
namespace BirdsiteLive.Twitter
@ -13,13 +15,12 @@ namespace BirdsiteLive.Twitter
{
private readonly ITwitterTweetsService _twitterService;
private MemoryCache _tweetCache = new MemoryCache(new MemoryCacheOptions()
{
SizeLimit = 5000
});
private MemoryCache _tweetCache = new MemoryCache(
new MemoryCacheOptions() { SizeLimit = 5000 }
);
private MemoryCacheEntryOptions _cacheEntryOptions = new MemoryCacheEntryOptions()
.SetSize(1)//Size amount
//Priority on removing when reaching size limit (memory pressure)
.SetSize(1) //Size amount
//Priority on removing when reaching size limit (memory pressure)
.SetPriority(CacheItemPriority.High)
// Keep in cache for this time, reset time if accessed.
// We set this lower than a user's in case they delete this Tweet for some reason; we don't need that cached.
@ -42,13 +43,13 @@ namespace BirdsiteLive.Twitter
public ExtractedTweet GetTweet(long statusId)
{
if(!_tweetCache.TryGetValue(statusId, out ExtractedTweet tweet))
if (!_tweetCache.TryGetValue(statusId, out ExtractedTweet tweet))
{
tweet = _twitterService.GetTweet(statusId);
// Unlike with the user cache, save the null value anyway to prevent (quicker) API exhaustion.
// It's incredibly unlikely that a tweet with this ID is going to magickally appear within 2 hours.
_tweetCache.Set(statusId, tweet, _cacheEntryOptions);
_tweetCache.Set(statusId, tweet, _cacheEntryOptions);
}
return tweet;
@ -60,4 +61,4 @@ namespace BirdsiteLive.Twitter
_tweetCache.Remove(statusId);
}
}
}
}

View File

@ -1,5 +1,7 @@
using System;
using System;
using BirdsiteLive.Twitter.Models;
using Microsoft.Extensions.Caching.Memory;
namespace BirdsiteLive.Twitter
@ -13,12 +15,11 @@ namespace BirdsiteLive.Twitter
{
private readonly ITwitterUserService _twitterService;
private MemoryCache _userCache = new MemoryCache(new MemoryCacheOptions()
{
SizeLimit = 5000
});
private MemoryCache _userCache = new MemoryCache(
new MemoryCacheOptions() { SizeLimit = 5000 }
);
private MemoryCacheEntryOptions _cacheEntryOptions = new MemoryCacheEntryOptions()
.SetSize(1)//Size amount
.SetSize(1) //Size amount
//Priority on removing when reaching size limit (memory pressure)
.SetPriority(CacheItemPriority.High)
// Keep in cache for this time, reset time if accessed.
@ -38,7 +39,8 @@ namespace BirdsiteLive.Twitter
if (!_userCache.TryGetValue(username, out TwitterUser user))
{
user = _twitterService.GetUser(username);
if(user != null) _userCache.Set(username, user, _cacheEntryOptions);
if (user != null)
_userCache.Set(username, user, _cacheEntryOptions);
}
return user;
@ -49,4 +51,4 @@ namespace BirdsiteLive.Twitter
_userCache.Remove(username);
}
}
}
}

View File

@ -1,9 +1,11 @@
using System;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.Twitter.Models;
using Tweetinvi.Models;
using Tweetinvi.Models.Entities;
@ -36,11 +38,20 @@ namespace BirdsiteLive.Twitter.Extractors
Media = ExtractMedia(tweet),
CreatedAt = tweet.CreatedAt.ToUniversalTime(),
IsReply = tweet.InReplyToUserId != null,
IsThread = tweet.InReplyToUserId != null && tweet.InReplyToUserId == tweet.CreatedBy.Id,
IsThread =
tweet.InReplyToUserId != null && tweet.InReplyToUserId == tweet.CreatedBy.Id,
IsRetweet = tweet.IsRetweet || tweet.QuotedStatusId != null,
RetweetUrl = ExtractRetweetUrl(tweet),
IsSensitive = tweet.PossiblySensitive,
QuoteTweetUrl = tweet.QuotedStatusId != null ? "https://" + _instanceSettings.Domain + "/users/" + tweet.QuotedTweet.CreatedBy.ScreenName + "/statuses/" + tweet.QuotedStatusId : null
QuoteTweetUrl =
tweet.QuotedStatusId != null
? "https://"
+ _instanceSettings.Domain
+ "/users/"
+ tweet.QuotedTweet.CreatedBy.ScreenName
+ "/statuses/"
+ tweet.QuotedStatusId
: null
};
return extractedTweet;
@ -59,7 +70,9 @@ namespace BirdsiteLive.Twitter.Extractors
}
if (tweet.FullText.Contains("https://t.co/"))
{
var retweetId = tweet.FullText.Split(new[] { "https://t.co/" }, StringSplitOptions.RemoveEmptyEntries).Last();
var retweetId = tweet.FullText
.Split(new[] { "https://t.co/" }, StringSplitOptions.RemoveEmptyEntries)
.Last();
return $"https://t.co/{retweetId}";
}
}
@ -71,7 +84,7 @@ namespace BirdsiteLive.Twitter.Extractors
{
var message = tweet.FullText;
var tweetUrls = tweet.Media.Select(x => x.URL).Distinct();
if (tweet.IsRetweet && message.StartsWith("RT") && tweet.RetweetedTweet != null)
{
message = tweet.RetweetedTweet.FullText;
@ -80,13 +93,13 @@ namespace BirdsiteLive.Twitter.Extractors
foreach (var tweetUrl in tweetUrls)
{
if(tweet.IsRetweet)
if (tweet.IsRetweet)
message = tweet.RetweetedTweet.FullText.Replace(tweetUrl, string.Empty).Trim();
else
else
message = message.Replace(tweetUrl, string.Empty).Trim();
}
if (tweet.QuotedTweet != null && ! _instanceSettings.EnableQuoteRT)
if (tweet.QuotedTweet != null && !_instanceSettings.EnableQuoteRT)
{
message = $"[Quote {{RT}}]{Environment.NewLine}{message}";
}
@ -94,9 +107,16 @@ namespace BirdsiteLive.Twitter.Extractors
if (tweet.IsRetweet)
{
if (tweet.RetweetedTweet != null && !message.StartsWith("RT"))
message = $"[{{RT}} @{tweet.RetweetedTweet.CreatedBy.ScreenName}]{Environment.NewLine}{message}";
else if (tweet.RetweetedTweet != null && message.StartsWith($"RT @{tweet.RetweetedTweet.CreatedBy.ScreenName}:"))
message = message.Replace($"RT @{tweet.RetweetedTweet.CreatedBy.ScreenName}:", $"[{{RT}} @{tweet.RetweetedTweet.CreatedBy.ScreenName}]{Environment.NewLine}");
message =
$"[{{RT}} @{tweet.RetweetedTweet.CreatedBy.ScreenName}]{Environment.NewLine}{message}";
else if (
tweet.RetweetedTweet != null
&& message.StartsWith($"RT @{tweet.RetweetedTweet.CreatedBy.ScreenName}:")
)
message = message.Replace(
$"RT @{tweet.RetweetedTweet.CreatedBy.ScreenName}:",
$"[{{RT}} @{tweet.RetweetedTweet.CreatedBy.ScreenName}]{Environment.NewLine}"
);
else
message = message.Replace("RT", "[{{RT}}]");
}
@ -108,7 +128,8 @@ namespace BirdsiteLive.Twitter.Extractors
if (url.ExpandedURL == tweet.QuotedTweet?.Url && _instanceSettings.EnableQuoteRT)
{
url.ExpandedURL = "";
} else
}
else
{
var linkUri = new UriBuilder(url.ExpandedURL);
@ -138,14 +159,10 @@ namespace BirdsiteLive.Twitter.Extractors
{
var mediaUrl = GetMediaUrl(m);
var mediaType = GetMediaType(m.MediaType, mediaUrl);
if (mediaType == null) continue;
if (mediaType == null)
continue;
var att = new ExtractedMedia
{
MediaType = mediaType,
Url = mediaUrl
};
var att = new ExtractedMedia { MediaType = mediaType, Url = mediaUrl };
result.Add(att);
}
@ -156,10 +173,17 @@ namespace BirdsiteLive.Twitter.Extractors
{
switch (media.MediaType)
{
case "photo": return media.MediaURLHttps;
case "animated_gif": return media.VideoDetails.Variants[0].URL;
case "video": return media.VideoDetails.Variants.OrderByDescending(x => x.Bitrate).First().URL;
default: return null;
case "photo":
return media.MediaURLHttps;
case "animated_gif":
return media.VideoDetails.Variants[0].URL;
case "video":
return media.VideoDetails.Variants
.OrderByDescending(x => x.Bitrate)
.First()
.URL;
default:
return null;
}
}
@ -195,4 +219,4 @@ namespace BirdsiteLive.Twitter.Extractors
return null;
}
}
}
}

Some files were not shown because too many files have changed in this diff Show More