Compare commits

..

44 commits

Author SHA1 Message Date
61ccf93261 chore(deps): update dependency terminal.gui to v1.9.0
Some checks reported errors
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build was killed
2023-01-05 01:00:34 +00:00
a47011864a chore(deps): update mstest monorepo to v3.0.2
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2022-12-30 13:00:49 +00:00
9e5eab49b0 chore(deps): update dependency moq to v4.18.4
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2022-12-30 12:00:30 +00:00
16dda205b8 chore(deps): update dependency lamar.microsoft.dependencyinjection to v10.0.1
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2022-12-20 18:00:28 +00:00
ebe6303027 chore(deps): update mstest monorepo to v3.0.1
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2022-12-20 15:00:33 +00:00
77dbc056c6 chore(deps): update dependency npgsql to v7.0.1
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2022-12-17 16:00:31 +00:00
1babf16ce9 chore(deps): update dependency microsoft.net.test.sdk to v17.4.1
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2022-12-16 11:00:30 +00:00
9eba7dbdb0 chore(deps): update mstest monorepo to v3
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2022-12-06 18:01:00 +00:00
195e690646 chore(deps): update dependency lamar.microsoft.dependencyinjection to v10
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2022-12-06 17:00:31 +00:00
c8dead211c chore(deps): update dependency moq to v4.18.3
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2022-12-05 01:00:30 +00:00
6937523cd6 chore(deps): update dependency lamar.microsoft.dependencyinjection to v9
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2022-12-01 17:00:32 +00:00
e8b45110a0 chore(deps): update dependency newtonsoft.json to v13.0.2
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2022-11-24 04:00:31 +00:00
aca8b02f42
add an exception watch
All checks were successful
continuous-integration/drone/push Build is passing
Signed-off-by: Sam Therapy <sam@samtherapy.net>
2022-11-21 15:36:39 +01:00
c8f1e7e64e chore(deps): update dependency npgsql to v7
All checks were successful
continuous-integration/drone/push Build is passing
2022-11-16 13:58:00 +00:00
d165da516d chore(deps): update dependency newtonsoft.json to v13
Some checks are pending
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is running
2022-11-16 13:00:54 +00:00
49a0b05113 chore(deps): update dependency microsoft.net.test.sdk to v17
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2022-11-16 12:00:53 +00:00
44721fdbc1 chore(deps): update dependency lamar.microsoft.dependencyinjection to v8
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2022-11-16 11:00:58 +00:00
14983127c3 chore(deps): update dependency coverlet.collector to v3
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2022-11-16 10:00:56 +00:00
69f847713e chore(deps): update actions/setup-dotnet action to v3
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2022-11-16 09:00:46 +00:00
4634ad1bd0 chore(deps): update actions/checkout action to v3
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2022-11-16 08:00:48 +00:00
0df311f322 chore(deps): update mstest monorepo to v2.2.10
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2022-11-16 07:00:51 +00:00
90c0b02e8d chore(deps): update dependency portable.bouncycastle to v1.9.0
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2022-11-16 06:00:46 +00:00
3f4ac62edb chore(deps): update dependency moq to v4.18.2
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2022-11-16 05:00:48 +00:00
19b1126042 chore(deps): update dependency microsoft.applicationinsights.aspnetcore to v2.21.0
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2022-11-16 04:00:32 +00:00
Renovate Bot
0847bb75ae [SKIP CI] chore(deps): update dependency microsoft.visualstudio.azure.containers.tools.targets to v1.17.0 (#12)
Co-authored-by: Renovate Bot <renovate-bot@noreply.git.froth.zone>
Co-committed-by: Renovate Bot <renovate-bot@noreply.git.froth.zone>
2022-11-16 03:39:01 +00:00
Renovate Bot
b48b454e6a [SKIP CI] chore(deps): update dependency microsoft.net.test.sdk to v16.11.0 (#11)
Co-authored-by: Renovate Bot <renovate-bot@noreply.git.froth.zone>
Co-committed-by: Renovate Bot <renovate-bot@noreply.git.froth.zone>
2022-11-16 03:24:11 +00:00
Renovate Bot
feb7e43f20 [SKIP CI] chore(deps): update dependency coverlet.collector to v1.3.0 (#9)
Co-authored-by: Renovate Bot <renovate-bot@noreply.git.froth.zone>
Co-committed-by: Renovate Bot <renovate-bot@noreply.git.froth.zone>
2022-11-16 03:23:59 +00:00
Renovate Bot
0fc7489ec8 [SKIP CI] chore(deps): update dependency lamar.microsoft.dependencyinjection to v5.0.4 (#5)
Co-authored-by: Renovate Bot <renovate-bot@noreply.git.froth.zone>
Co-committed-by: Renovate Bot <renovate-bot@noreply.git.froth.zone>
2022-11-16 03:23:48 +00:00
Renovate Bot
25ecc935ed [SKIP CI] chore(deps): update dependency lamar to v5.0.4 (#3)
Co-authored-by: Renovate Bot <renovate-bot@noreply.git.froth.zone>
Co-committed-by: Renovate Bot <renovate-bot@noreply.git.froth.zone>
2022-11-16 03:23:26 +00:00
Renovate Bot
b439f54c10 [SKIP CI] chore(deps): update dependency npgsql to v4.1.12 (#7)
Co-authored-by: Renovate Bot <renovate-bot@noreply.git.froth.zone>
Co-committed-by: Renovate Bot <renovate-bot@noreply.git.froth.zone>
2022-11-16 03:23:13 +00:00
Renovate Bot
73db263d6e chore(deps): update dependency terminal.gui to v1.8.2 (#8)
All checks were successful
continuous-integration/drone/push Build is passing
This PR contains the following updates:

| Package | Type | Update | Change |
|---|---|---|---|
| [Terminal.Gui](https://github.com/gui-cs/Terminal.Gui) | nuget | patch | `1.8.0` -> `1.8.2` |

---

### Release Notes

<details>
<summary>gui-cs/Terminal.Gui</summary>

### [`v1.8.2`](https://github.com/gui-cs/Terminal.Gui/releases/tag/v1.8.2)

This was supposed to be v1.8.1, but a deployment mistake required a new push.

#### What's Changed

-   Bump crazy-max/ghaction-chocolatey from 1 to 2 by [@&#8203;dependabot](https://github.com/dependabot) in https://github.com/gui-cs/Terminal.Gui/pull/2050
-   Fixes [#&#8203;2053](https://github.com/gui-cs/Terminal.Gui/issues/2053). MessageBox.Query not wrapping correctly by [@&#8203;BDisp](https://github.com/BDisp) in https://github.com/gui-cs/Terminal.Gui/pull/2054

**Full Changelog**: https://github.com/gui-cs/Terminal.Gui/compare/v1.8.0...v1.8.2

</details>

---

### Configuration

📅 **Schedule**: Branch creation - At any time (no schedule defined), Automerge - At any time (no schedule defined).

🚦 **Automerge**: Enabled.

♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox.

🔕 **Ignore**: Close this PR and you won't be reminded about this update again.

---

 - [ ] <!-- rebase-check -->If you want to rebase/retry this PR, check this box

---

This PR has been generated by [Renovate Bot](https://github.com/renovatebot/renovate).
<!--renovate-debug:eyJjcmVhdGVkSW5WZXIiOiIzNC4yNS4xIiwidXBkYXRlZEluVmVyIjoiMzQuMjUuMSJ9-->

Co-authored-by: Renovate Bot <renovate@whitesourcesoftware.com>
Reviewed-on: #8
Co-authored-by: Renovate Bot <renovate-bot@noreply.git.froth.zone>
Co-committed-by: Renovate Bot <renovate-bot@noreply.git.froth.zone>
2022-11-16 02:06:37 +00:00
19d36545c5 chore(deps): update dependency dapper to v2.0.123
Some checks are pending
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is running
2022-11-16 01:00:50 +00:00
9098e53617 chore(deps): update dependency microsoft.visualstudio.web.codegeneration.design to v3.1.5
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2022-11-16 00:00:34 +00:00
59296b83d5 chore(deps): add renovate.json
Some checks reported errors
continuous-integration/drone/push Build was killed
2022-11-15 22:00:19 +00:00
652434c42a
[SKIP CI] fix(docs): update README
Signed-off-by: Sam Therapy <sam@samtherapy.net>
2022-11-15 22:38:27 +01:00
5d6ee7c5d3
fix(ci): this space intentionally left blank
All checks were successful
continuous-integration/drone/push Build is passing
WHY BUILDX, WHY

Signed-off-by: Sam Therapy <sam@samtherapy.net>
2022-11-15 22:03:47 +01:00
f22df41d49
fix(ci): I hate docker
All checks were successful
continuous-integration/drone/push Build is passing
Signed-off-by: Sam Therapy <sam@samtherapy.net>
2022-11-15 21:53:46 +01:00
7398ce6880
make it more clear this is actually a fork
Some checks reported errors
continuous-integration/drone/push Build encountered an error
Signed-off-by: Sam Therapy <sam@samtherapy.net>
2022-11-15 21:42:29 +01:00
097b5316e8
fix(typo): remove , from log
All checks were successful
continuous-integration/drone/push Build is passing
Signed-off-by: Sam Therapy <sam@samtherapy.net>
2022-11-15 19:10:27 +01:00
990778dfb3
feat(db): Purge user when no one follows them
All checks were successful
continuous-integration/drone/push Build is passing
Signed-off-by: Sam Therapy <sam@samtherapy.net>
2022-11-15 18:45:02 +01:00
1e2be0e5b5
fix(ci): merde
All checks were successful
continuous-integration/drone/push Build is passing
Of course it is lowercase only duh

Signed-off-by: Sam Therapy <sam@samtherapy.net>
2022-11-15 17:45:37 +01:00
0b2559724a
fix(docker): use proper base
Some checks failed
continuous-integration/drone/push Build is failing
Old one doesn't exist anymore, I think

Signed-off-by: Sam Therapy <sam@samtherapy.net>
2022-11-15 17:39:44 +01:00
01df5c6139
fix(ci): Remove restore
Some checks failed
continuous-integration/drone/push Build is failing
This is a temporary measure basically just to test the pipeline

Signed-off-by: Sam Therapy <sam@samtherapy.net>
2022-11-15 17:35:38 +01:00
e2ec4a5857
format things, I think
Some checks failed
continuous-integration/drone Build is failing
also prepare pipelines and stuff

Signed-off-by: Sam Therapy <sam@samtherapy.net>
2022-11-15 17:25:54 +01:00
231 changed files with 5023 additions and 5640 deletions

View file

@ -37,6 +37,21 @@ depends_on:
- testing
steps:
- name: Build
privileged: true
image: quay.io/thegeeklab/drone-docker-buildx
settings:
repo: git.froth.zone/sam/birdsitelive
dry_run: true
platforms:
- linux/amd64
- linux/arm64
when:
event:
- pull_request
depends_on:
- "clone"
- name: Build & Publish
privileged: true
image: quay.io/thegeeklab/drone-docker-buildx

248
.editorconfig Normal file
View file

@ -0,0 +1,248 @@
# Remove the line below if you want to inherit .editorconfig settings from higher directories
root = true
# C# files
[*.cs]
#### Core EditorConfig Options ####
# Indentation and spacing
indent_size = 4
indent_style = space
tab_width = 4
# New line preferences
end_of_line = lf
insert_final_newline = true
#### .NET Coding Conventions ####
# Organize usings
dotnet_separate_import_directive_groups = true
dotnet_sort_system_directives_first = true
file_header_template = unset
# this. and Me. preferences
dotnet_style_qualification_for_event = false
dotnet_style_qualification_for_field = false
dotnet_style_qualification_for_method = false
dotnet_style_qualification_for_property = false
# Language keywords vs BCL types preferences
dotnet_style_predefined_type_for_locals_parameters_members = true
dotnet_style_predefined_type_for_member_access = true
# Parentheses preferences
dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity
dotnet_style_parentheses_in_other_binary_operators = always_for_clarity
dotnet_style_parentheses_in_other_operators = never_if_unnecessary
dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity
# Modifier preferences
dotnet_style_require_accessibility_modifiers = for_non_interface_members
# Expression-level preferences
dotnet_style_coalesce_expression = true
dotnet_style_collection_initializer = true
dotnet_style_explicit_tuple_names = true
dotnet_style_namespace_match_folder = false
dotnet_style_null_propagation = true
dotnet_style_object_initializer = true
dotnet_style_operator_placement_when_wrapping = beginning_of_line
dotnet_style_prefer_auto_properties = true
dotnet_style_prefer_compound_assignment = true
dotnet_style_prefer_conditional_expression_over_assignment = true
dotnet_style_prefer_conditional_expression_over_return = true
dotnet_style_prefer_foreach_explicit_cast_in_source = when_strongly_typed
dotnet_style_prefer_inferred_anonymous_type_member_names = true
dotnet_style_prefer_inferred_tuple_names = true
dotnet_style_prefer_is_null_check_over_reference_equality_method = true
dotnet_style_prefer_simplified_boolean_expressions = true
dotnet_style_prefer_simplified_interpolation = true
# Field preferences
dotnet_style_readonly_field = true
# Parameter preferences
dotnet_code_quality_unused_parameters = all
# Suppression preferences
dotnet_remove_unnecessary_suppression_exclusions = none
# New line preferences
dotnet_style_allow_multiple_blank_lines_experimental = false
dotnet_style_allow_statement_immediately_after_block_experimental = false
#### C# Coding Conventions ####
# var preferences
csharp_style_var_elsewhere = true
csharp_style_var_for_built_in_types = false
csharp_style_var_when_type_is_apparent = true
# Expression-bodied members
csharp_style_expression_bodied_accessors = true:silent
csharp_style_expression_bodied_constructors = when_on_single_line:silent
csharp_style_expression_bodied_indexers = true:silent
csharp_style_expression_bodied_lambdas = true:silent
csharp_style_expression_bodied_local_functions = when_on_single_line:silent
csharp_style_expression_bodied_methods = when_on_single_line:silent
csharp_style_expression_bodied_operators = when_on_single_line:silent
csharp_style_expression_bodied_properties = true:silent
# Pattern matching preferences
csharp_style_pattern_matching_over_as_with_null_check = true
csharp_style_pattern_matching_over_is_with_cast_check = true
csharp_style_prefer_extended_property_pattern = true
csharp_style_prefer_not_pattern = true
csharp_style_prefer_pattern_matching = true
csharp_style_prefer_switch_expression = true:suggestion
# Null-checking preferences
csharp_style_conditional_delegate_call = true
# Modifier preferences
csharp_prefer_static_local_function = true
csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,required,volatile,async
# Code-block preferences
csharp_prefer_braces = when_multiline:silent
csharp_prefer_simple_using_statement = true:suggestion
csharp_style_namespace_declarations = file_scoped:suggestion
csharp_style_prefer_method_group_conversion = true:silent
csharp_style_prefer_top_level_statements = true:suggestion
# Expression-level preferences
csharp_prefer_simple_default_expression = true:suggestion
csharp_style_deconstructed_variable_declaration = true:suggestion
csharp_style_implicit_object_creation_when_type_is_apparent = true:suggestion
csharp_style_inlined_variable_declaration = true:suggestion
csharp_style_prefer_index_operator = true:suggestion
csharp_style_prefer_local_over_anonymous_function = true:suggestion
csharp_style_prefer_null_check_over_type_check = true:suggestion
csharp_style_prefer_range_operator = true:suggestion
csharp_style_prefer_tuple_swap = true:suggestion
csharp_style_prefer_utf8_string_literals = true:suggestion
csharp_style_throw_expression = true:suggestion
csharp_style_unused_value_assignment_preference = discard_variable:suggestion
csharp_style_unused_value_expression_statement_preference = discard_variable:silent
# 'using' directive preferences
csharp_using_directive_placement = outside_namespace:silent
# New line preferences
csharp_style_allow_blank_line_after_colon_in_constructor_initializer_experimental = false
csharp_style_allow_blank_lines_between_consecutive_braces_experimental = false
csharp_style_allow_embedded_statements_on_same_line_experimental = true
#### C# Formatting Rules ####
# New line preferences
csharp_new_line_before_catch = true
csharp_new_line_before_else = true
csharp_new_line_before_finally = true
csharp_new_line_before_members_in_anonymous_types = true
csharp_new_line_before_members_in_object_initializers = true
csharp_new_line_before_open_brace = all
csharp_new_line_between_query_expression_clauses = true
# Indentation preferences
csharp_indent_block_contents = true
csharp_indent_braces = false
csharp_indent_case_contents = true
csharp_indent_case_contents_when_block = true
csharp_indent_labels = one_less_than_current
csharp_indent_switch_labels = true
# Space preferences
csharp_space_after_cast = false
csharp_space_after_colon_in_inheritance_clause = true
csharp_space_after_comma = true
csharp_space_after_dot = false
csharp_space_after_keywords_in_control_flow_statements = true
csharp_space_after_semicolon_in_for_statement = true
csharp_space_around_binary_operators = before_and_after
csharp_space_around_declaration_statements = false
csharp_space_before_colon_in_inheritance_clause = true
csharp_space_before_comma = false
csharp_space_before_dot = false
csharp_space_before_open_square_brackets = false
csharp_space_before_semicolon_in_for_statement = false
csharp_space_between_empty_square_brackets = false
csharp_space_between_method_call_empty_parameter_list_parentheses = false
csharp_space_between_method_call_name_and_opening_parenthesis = false
csharp_space_between_method_call_parameter_list_parentheses = false
csharp_space_between_method_declaration_empty_parameter_list_parentheses = false
csharp_space_between_method_declaration_name_and_open_parenthesis = false
csharp_space_between_method_declaration_parameter_list_parentheses = false
csharp_space_between_parentheses = false
csharp_space_between_square_brackets = false
# Wrapping preferences
csharp_preserve_single_line_blocks = true
csharp_preserve_single_line_statements = true
#### Naming styles ####
# Naming rules
dotnet_naming_rule.interface_should_be_begins_with_i.severity = suggestion
dotnet_naming_rule.interface_should_be_begins_with_i.symbols = interface
dotnet_naming_rule.interface_should_be_begins_with_i.style = begins_with_i
dotnet_naming_rule.types_should_be_pascal_case.severity = suggestion
dotnet_naming_rule.types_should_be_pascal_case.symbols = types
dotnet_naming_rule.types_should_be_pascal_case.style = pascal_case
dotnet_naming_rule.non_field_members_should_be_pascal_case.severity = suggestion
dotnet_naming_rule.non_field_members_should_be_pascal_case.symbols = non_field_members
dotnet_naming_rule.non_field_members_should_be_pascal_case.style = pascal_case
# Symbol specifications
dotnet_naming_symbols.interface.applicable_kinds = interface
dotnet_naming_symbols.interface.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected
dotnet_naming_symbols.interface.required_modifiers =
dotnet_naming_symbols.types.applicable_kinds = class, struct, interface, enum
dotnet_naming_symbols.types.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected
dotnet_naming_symbols.types.required_modifiers =
dotnet_naming_symbols.non_field_members.applicable_kinds = property, event, method
dotnet_naming_symbols.non_field_members.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected
dotnet_naming_symbols.non_field_members.required_modifiers =
# Naming styles
dotnet_naming_style.pascal_case.required_prefix =
dotnet_naming_style.pascal_case.required_suffix =
dotnet_naming_style.pascal_case.word_separator =
dotnet_naming_style.pascal_case.capitalization = pascal_case
dotnet_naming_style.begins_with_i.required_prefix = I
dotnet_naming_style.begins_with_i.required_suffix =
dotnet_naming_style.begins_with_i.word_separator =
dotnet_naming_style.begins_with_i.capitalization = pascal_case
dotnet_diagnostic.IDE0004.severity = suggestion
dotnet_diagnostic.IDE0005.severity = suggestion
[*.{cs,vb}]
dotnet_style_coalesce_expression = true:suggestion
dotnet_style_null_propagation = true:suggestion
dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion
dotnet_style_prefer_auto_properties = true:silent
dotnet_style_object_initializer = true:suggestion
dotnet_style_collection_initializer = true:suggestion
dotnet_style_prefer_simplified_boolean_expressions = true:suggestion
dotnet_style_operator_placement_when_wrapping = beginning_of_line
tab_width = 4
indent_size = 4
end_of_line = crlf
dotnet_style_prefer_conditional_expression_over_assignment = true:silent
dotnet_style_namespace_match_folder = false:silent
dotnet_style_explicit_tuple_names = true:suggestion
dotnet_style_prefer_conditional_expression_over_return = true:silent
dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion
dotnet_style_prefer_inferred_tuple_names = true:suggestion
dotnet_style_prefer_simplified_interpolation = true:suggestion
dotnet_style_prefer_compound_assignment = true:suggestion

1
.github/FUNDING.yml vendored
View file

@ -1 +0,0 @@
patreon: nicolasconstant

View file

@ -12,7 +12,7 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: Launch Db for testing
run: docker run --name postgres -e POSTGRES_DB=mytestdb -e POSTGRES_PASSWORD=mysecretpassword -d -p 5432:5432 postgres
run: docker run --name postgres -e POSTGRES_USER=birdtest -e POSTGRES_DB=birdsitetest -e POSTGRES_PASSWORD=mysecretpassword -d -p 5432:5432 postgres
- name: Setup .NET Core
uses: actions/setup-dotnet@v3
with:

View file

@ -53,13 +53,12 @@ If both whitelisting and blacklisting are set, only the whitelisting will be act
* `Instance:ShowAboutInstanceOnProfiles` (default: true) show "About [instance name]" on profiles with a link to /About
* `Instance:MaxFollowsPerUser` (default: 0 - no limit) limit the number of follows per user - any follow count above this number will be Rejected
* `Instance:DiscloseInstanceRestrictions` (default: false) disclose your instance's restrictions on its About page
* `Instance:UnlistedTwitterAccounts` (default: null) to enable unlisted publication for selected twitter accounts, separated by `;` (please limit this to brands and other public profiles).
* `Instance:SensitiveTwitterAccounts` (default: null) mark all media from given accounts as sensitive by default, separated by `;`.
* `Instance:UnlistedTwitterAccounts` (default: null) to enable unlisted publication for selected twitter accounts, separated by `;` (please limit this to brands and other public profiles).
* `Instance:SensitiveTwitterAccounts` (default: null) mark all media from given accounts as sensitive by default, separated by `;`.
* `Instance:FailingTwitterUserCleanUpThreshold` (default: 700) set the max allowed errors (due to a banned/deleted/private account) from a Twitter Account retrieval before auto-removal. (by default an account is called every 15 mins)
* `Instance:FailingFollowerCleanUpThreshold` (default: 30000) set the max allowed errors from a Follower (Fediverse) Account before auto-removal. (often due to account suppression, instance issues, etc)
* `Instance:MaxStatusFetchAge` (default: 0 - no limit) statuses with a Snowflake older than this age in days will not be fetched by the service and will instead return 410 Gone
* `Instance:EnableQuoteRT` (default: false) enable Soapbox-style quote-RTs
* `Instance:FailingFollowerCleanUpThreshold` (default: 30000) set the max allowed errors from a Follower (Fediverse) Account before auto-removal. (often due to account suppression, instance issues, etc)
* `Instance:UserCacheCapacity` (default: 10000) set the caching limit of the Twitter User retrieval. Must be higher than the number of synchronized accounts on the instance.
# Docker Compose full example

View file

@ -1,13 +1,16 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Moderation.Actions;
using BSLManager.Domain;
using BSLManager.Tools;
using Terminal.Gui;
namespace BSLManager
@ -29,8 +32,6 @@ namespace BSLManager
public void Run()
{
Application.UseSystemConsole = true;
Application.Init();
var top = Application.Top;
@ -39,7 +40,6 @@ namespace BSLManager
{
X = 0,
Y = 1, // Leave one row for the toplevel menu
// By using Dim.Fill(), it will automatically resize without manual intervention
Width = Dim.Fill(),
Height = Dim.Fill()
@ -48,29 +48,46 @@ namespace BSLManager
top.Add(win);
// Creates a menubar, the item "New" has a help menu.
var menu = new MenuBar(new MenuBarItem[]
{
new MenuBarItem("_File", new MenuItem[]
var menu = new MenuBar(
new MenuBarItem[]
{
new MenuItem("_Quit", "", () =>
{
if (Quit()) top.Running = false;
})
}),
//new MenuBarItem ("_Edit", new MenuItem [] {
// new MenuItem ("_Copy", "", null),
// new MenuItem ("C_ut", "", null),
// new MenuItem ("_Paste", "", null)
//})
});
new MenuBarItem(
"_File",
new MenuItem[]
{
new MenuItem(
"_Quit",
"",
() =>
{
if (Quit())
top.Running = false;
}
)
}
),
//new MenuBarItem ("_Edit", new MenuItem [] {
// new MenuItem ("_Copy", "", null),
// new MenuItem ("C_ut", "", null),
// new MenuItem ("_Paste", "", null)
//})
}
);
top.Add(menu);
static bool Quit()
{
var n = MessageBox.Query(50, 7, "Quit BSL Manager", "Are you sure you want to quit?", "Yes", "No");
var n = MessageBox.Query(
50,
7,
"Quit BSL Manager",
"Are you sure you want to quit?",
"Yes",
"No"
);
return n == 0;
}
RetrieveUserList();
var list = new ListView(_state.GetDisplayableList())
@ -86,11 +103,13 @@ namespace BSLManager
if (_.KeyEvent.Key == Key.Enter)
{
OpenFollowerDialog(list.SelectedItem);
}
else if (_.KeyEvent.Key == Key.Delete
|| _.KeyEvent.Key == Key.DeleteChar
|| _.KeyEvent.Key == Key.Backspace
|| _.KeyEvent.Key == Key.D)
}
else if (
_.KeyEvent.Key == Key.Delete
|| _.KeyEvent.Key == Key.DeleteChar
|| _.KeyEvent.Key == Key.Backspace
|| _.KeyEvent.Key == Key.D
)
{
OpenDeleteDialog(list.SelectedItem);
}
@ -115,12 +134,7 @@ namespace BSLManager
}
};
win.Add(
listingFollowersLabel,
filterLabel,
filterText,
list
);
win.Add(listingFollowersLabel, filterLabel, filterText, list);
Application.Run();
}
@ -226,7 +240,7 @@ namespace BSLManager
try
{
var userToDelete = _state.GetElementAt(el);
BasicLogger.Log($"Delete {userToDelete.Acct}@{userToDelete.Host}");
await _removeFollowerAction.ProcessAsync(userToDelete);
BasicLogger.Log($"Remove user from list");
@ -251,4 +265,4 @@ namespace BSLManager
});
}
}
}
}

View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
@ -10,7 +10,7 @@
<PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="5.0.0" />
<PackageReference Include="Terminal.Gui" Version="1.12.1" />
<PackageReference Include="Terminal.Gui" Version="1.9.0" />
</ItemGroup>
<ItemGroup>

View file

@ -1,12 +1,15 @@
using System;
using System;
using System.Net.Http;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.Common.Structs;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Postgres.DataAccessLayers;
using BirdsiteLive.DAL.Postgres.Settings;
using Lamar;
using Lamar.Scanning.Conventions;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
@ -33,13 +36,17 @@ namespace BSLManager
x.For<InstanceSettings>().Use(x => _instanceSettings);
if (string.Equals(_dbSettings.Type, DbTypes.Postgres, StringComparison.OrdinalIgnoreCase))
if (
string.Equals(
_dbSettings.Type,
DbTypes.Postgres,
StringComparison.OrdinalIgnoreCase
)
)
{
var connString = $"Host={_dbSettings.Host};Username={_dbSettings.User};Password={_dbSettings.Password};Database={_dbSettings.Name}";
var postgresSettings = new PostgresSettings
{
ConnString = connString
};
var connString =
$"Host={_dbSettings.Host};Username={_dbSettings.User};Password={_dbSettings.Password};Database={_dbSettings.Name}";
var postgresSettings = new PostgresSettings { ConnString = connString };
x.For<PostgresSettings>().Use(x => postgresSettings);
x.For<ITwitterUserDal>().Use<TwitterUserPostgresDal>().Singleton();
@ -51,8 +58,11 @@ namespace BSLManager
throw new NotImplementedException($"{_dbSettings.Type} is not supported");
}
var serviceProvider = new ServiceCollection().AddHttpClient().BuildServiceProvider();
x.For<IHttpClientFactory>().Use(_ => serviceProvider.GetService<IHttpClientFactory>());
var serviceProvider = new ServiceCollection()
.AddHttpClient()
.BuildServiceProvider();
x.For<IHttpClientFactory>()
.Use(_ => serviceProvider.GetService<IHttpClientFactory>());
x.For(typeof(ILogger<>)).Use(typeof(DummyLogger<>));
@ -76,9 +86,13 @@ namespace BSLManager
public class DummyLogger<T> : ILogger<T>
{
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter)
{
}
public void Log<TState>(
LogLevel logLevel,
EventId eventId,
TState state,
Exception exception,
Func<TState, Exception, string> formatter
) { }
public bool IsEnabled(LogLevel logLevel)
{
@ -91,4 +105,4 @@ namespace BSLManager
}
}
}
}
}

View file

@ -1,5 +1,6 @@
using System.Collections.Generic;
using System.Collections.Generic;
using System.Linq;
using BirdsiteLive.DAL.Models;
namespace BSLManager.Domain
@ -10,11 +11,11 @@ namespace BSLManager.Domain
private List<Follower> _sourceUserList = new List<Follower>();
private List<Follower> _filteredSourceUserList = new List<Follower>();
public void Load(List<Follower> followers)
{
_sourceUserList = followers.OrderByDescending(x => x.Followings.Count).ToList();
ResetLists();
}
@ -26,7 +27,8 @@ namespace BSLManager.Domain
foreach (var follower in _sourceUserList)
{
var displayedUser = $"{GetFullHandle(follower)} ({follower.Followings.Count}) (err:{follower.PostingErrorCount})";
var displayedUser =
$"{GetFullHandle(follower)} ({follower.Followings.Count}) (err:{follower.PostingErrorCount})";
_filteredDisplayableUserList.Add(displayedUser);
}
}
@ -50,8 +52,10 @@ namespace BSLManager.Domain
foreach (var el in elToRemove)
{
_filteredSourceUserList.Remove(el);
var dElToRemove = _filteredDisplayableUserList.First(x => x.Contains(GetFullHandle(el)));
var dElToRemove = _filteredDisplayableUserList.First(
x => x.Contains(GetFullHandle(el))
);
_filteredDisplayableUserList.Remove(dElToRemove);
}
}
@ -78,4 +82,4 @@ namespace BSLManager.Domain
return _filteredSourceUserList[index];
}
}
}
}

View file

@ -1,19 +1,12 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System;
using System.Text;
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BSLManager.Tools;
using Microsoft.Extensions.Configuration;
using NStack;
using Terminal.Gui;
namespace BSLManager
{
class Program
internal class Program
{
static async Task Main(string[] args)
{

View file

@ -1,4 +1,4 @@
using System;
using System;
using System.IO;
namespace BSLManager.Tools
@ -7,7 +7,7 @@ namespace BSLManager.Tools
{
public static void Log(string log)
{
File.AppendAllLines($"Log-{Guid.NewGuid()}.txt", new []{ log });
File.AppendAllLines($"Log-{Guid.NewGuid()}.txt", new[] { log });
}
}
}
}

View file

@ -1,4 +1,5 @@
using System.Reflection;
using System.Reflection;
using Terminal.Gui;
namespace BSLManager.Tools
@ -12,4 +13,4 @@ namespace BSLManager.Tools
.Invoke(null, null);
}
}
}
}

View file

@ -1,8 +1,11 @@
using System;
using System;
using System.IO;
using System.Runtime.CompilerServices;
using BirdsiteLive.Common.Settings;
using Newtonsoft.Json;
using Org.BouncyCastle.Asn1.IsisMtt.X509;
namespace BSLManager.Tools
@ -14,10 +17,13 @@ namespace BSLManager.Tools
public (DbSettings dbSettings, InstanceSettings instanceSettings) GetSettings()
{
var localSettingsData = GetLocalSettingsFile();
if (localSettingsData != null) return Convert(localSettingsData);
if (localSettingsData != null)
return Convert(localSettingsData);
Console.WriteLine("We need to set up the manager");
Console.WriteLine("Please provide the following information as provided in the docker-compose file");
Console.WriteLine(
"Please provide the following information as provided in the docker-compose file"
);
LocalSettingsData data;
do
@ -39,12 +45,11 @@ namespace BSLManager.Tools
Console.WriteLine("Is it valid? (yes, no)");
resp = Console.ReadLine()?.Trim().ToLowerInvariant();
if (resp == "n" || resp == "no") data = null;
if (resp == "n" || resp == "no")
data = null;
} while (resp != "y" && resp != "yes" && resp != "n" && resp != "no");
} while (data == null);
SaveLocalSettings(data);
return Convert(data);
}
@ -71,7 +76,9 @@ namespace BSLManager.Tools
return data;
}
private (DbSettings dbSettings, InstanceSettings instanceSettings) Convert(LocalSettingsData data)
private (DbSettings dbSettings, InstanceSettings instanceSettings) Convert(
LocalSettingsData data
)
{
var dbSettings = new DbSettings
{
@ -81,10 +88,7 @@ namespace BSLManager.Tools
User = data.DbUser,
Password = data.DbPassword
};
var instancesSettings = new InstanceSettings
{
Domain = data.InstanceDomain
};
var instancesSettings = new InstanceSettings { Domain = data.InstanceDomain };
return (dbSettings, instancesSettings);
}
@ -92,7 +96,8 @@ namespace BSLManager.Tools
{
try
{
if (!File.Exists(LocalFileName)) return null;
if (!File.Exists(LocalFileName))
return null;
var jsonContent = File.ReadAllText(LocalFileName);
var content = JsonConvert.DeserializeObject<LocalSettingsData>(jsonContent);
@ -121,4 +126,4 @@ namespace BSLManager.Tools
public string InstanceDomain { get; set; }
}
}
}

View file

@ -1,5 +1,5 @@
using System;
using BirdsiteLive.ActivityPub.Models;
using System;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub
@ -17,11 +17,9 @@ namespace BirdsiteLive.ActivityPub
return JsonConvert.DeserializeObject<ActivityFollow>(json);
case "Undo":
var a = JsonConvert.DeserializeObject<ActivityUndo>(json);
if(a.apObject.type == "Follow")
if (a.apObject.type == "Follow")
return JsonConvert.DeserializeObject<ActivityUndoFollow>(json);
break;
case "Delete":
return JsonConvert.DeserializeObject<ActivityDelete>(json);
case "Accept":
var accept = JsonConvert.DeserializeObject<ActivityAccept>(json);
//var acceptType = JsonConvert.DeserializeObject<Activity>(accept.apObject);
@ -62,4 +60,4 @@ namespace BirdsiteLive.ActivityPub
public Activity apObject { get; set; }
}
}
}
}

View file

@ -6,7 +6,7 @@
<ItemGroup>
<PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
<PackageReference Include="System.Text.Json" Version="4.7.2" />
</ItemGroup>

View file

@ -1,18 +1,28 @@
using System;
using System;
using System.Collections.Generic;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub.Converters
{
public class ContextArrayConverter : JsonConverter
{
public override bool CanWrite { get { return false; } }
public override bool CanWrite
{
get { return false; }
}
public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
{
throw new NotImplementedException();
}
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
public override object ReadJson(
JsonReader reader,
Type objectType,
object existingValue,
JsonSerializer serializer
)
{
var result = new List<string>();
@ -36,4 +46,4 @@ namespace BirdsiteLive.ActivityPub.Converters
throw new NotImplementedException();
}
}
}
}

View file

@ -1,4 +1,4 @@
using System.Runtime.CompilerServices;
using System.Runtime.CompilerServices;
namespace BirdsiteLive.ActivityPub.Converters
{
@ -14,4 +14,4 @@ namespace BirdsiteLive.ActivityPub.Converters
return $"https://{domain.ToLowerInvariant()}/users/{username.ToLowerInvariant()}/statuses/{noteId}";
}
}
}
}

View file

@ -1,5 +1,6 @@
using System.Collections.Generic;
using System.Collections.Generic;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub
@ -7,7 +8,8 @@ namespace BirdsiteLive.ActivityPub
public class Activity
{
[Newtonsoft.Json.JsonIgnore]
public static readonly object[] DefaultContext = new object[] {
public static readonly object[] DefaultContext = new object[]
{
"https://www.w3.org/ns/activitystreams",
"https://w3id.org/security/v1",
new Dictionary<string, string>
@ -18,7 +20,6 @@ namespace BirdsiteLive.ActivityPub
{ "value", "schema:value" },
{ "sensitive", "as:sensitive" },
{ "quoteUrl", "as:quoteUrl" },
{ "schema", "http://schema.org#" },
{ "toot", "https://joinmastodon.org/ns#" }
}
@ -33,4 +34,4 @@ namespace BirdsiteLive.ActivityPub
//[JsonProperty("object")]
//public string apObject { get; set; }
}
}
}

View file

@ -1,4 +1,4 @@
using Newtonsoft.Json;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub
{
@ -7,4 +7,4 @@ namespace BirdsiteLive.ActivityPub
[JsonProperty("object")]
public object apObject { get; set; }
}
}
}

View file

@ -1,4 +1,4 @@
using Newtonsoft.Json;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub
{
@ -7,4 +7,4 @@ namespace BirdsiteLive.ActivityPub
[JsonProperty("object")]
public ActivityFollow apObject { get; set; }
}
}
}

View file

@ -1,4 +1,4 @@
using Newtonsoft.Json;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub
{
@ -7,4 +7,4 @@ namespace BirdsiteLive.ActivityPub
[JsonProperty("object")]
public ActivityUndoFollow apObject { get; set; }
}
}
}

View file

@ -1,7 +1,4 @@
namespace BirdsiteLive.ActivityPub
namespace BirdsiteLive.ActivityPub
{
public class ActivityCreate
{
}
}
public class ActivityCreate { }
}

View file

@ -1,5 +1,7 @@
using System;
using System;
using BirdsiteLive.ActivityPub.Models;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub
@ -13,4 +15,4 @@ namespace BirdsiteLive.ActivityPub
[JsonProperty("object")]
public Note apObject { get; set; }
}
}
}

View file

@ -1,11 +0,0 @@
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub.Models
{
public class ActivityDelete : Activity
{
public string[] to { get; set; }
[JsonProperty("object")]
public object apObject { get; set; }
}
}

View file

@ -1,4 +1,4 @@
using Newtonsoft.Json;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub
{
@ -7,4 +7,4 @@ namespace BirdsiteLive.ActivityPub
[JsonProperty("object")]
public string apObject { get; set; }
}
}
}

View file

@ -1,4 +1,4 @@
using Newtonsoft.Json;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub
{
@ -7,4 +7,4 @@ namespace BirdsiteLive.ActivityPub
[JsonProperty("object")]
public ActivityFollow apObject { get; set; }
}
}
}

View file

@ -1,4 +1,4 @@
using Newtonsoft.Json;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub
{
@ -7,4 +7,4 @@ namespace BirdsiteLive.ActivityPub
[JsonProperty("object")]
public Activity apObject { get; set; }
}
}
}

View file

@ -1,4 +1,4 @@
using Newtonsoft.Json;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub
{
@ -7,4 +7,4 @@ namespace BirdsiteLive.ActivityPub
[JsonProperty("object")]
public ActivityFollow apObject { get; set; }
}
}
}

View file

@ -1,8 +1,10 @@
using System;
using System;
using System.Collections.Generic;
using System.Net;
using BirdsiteLive.ActivityPub.Converters;
using BirdsiteLive.ActivityPub.Models;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub
@ -20,7 +22,6 @@ namespace BirdsiteLive.ActivityPub
public string name { get; set; }
public string summary { get; set; }
public string url { get; set; }
public string movedTo { get; set; }
public bool manuallyApprovesFollowers { get; set; }
public string inbox { get; set; }
public bool? discoverable { get; set; } = true;

View file

@ -1,4 +1,4 @@
namespace BirdsiteLive.ActivityPub
namespace BirdsiteLive.ActivityPub
{
public class Attachment
{
@ -6,4 +6,4 @@
public string mediaType { get; set; }
public string url { get; set; }
}
}
}

View file

@ -1,7 +1,7 @@
namespace BirdsiteLive.ActivityPub
namespace BirdsiteLive.ActivityPub
{
public class EndPoints
{
public string sharedInbox { get; set; }
}
}
}

View file

@ -1,4 +1,5 @@
using BirdsiteLive.ActivityPub.Converters;
using BirdsiteLive.ActivityPub.Converters;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub.Models
@ -12,4 +13,4 @@ namespace BirdsiteLive.ActivityPub.Models
public string id { get; set; }
public string type { get; set; } = "OrderedCollection";
}
}
}

View file

@ -1,4 +1,4 @@
namespace BirdsiteLive.ActivityPub
namespace BirdsiteLive.ActivityPub
{
public class Image
{
@ -6,4 +6,4 @@
public string mediaType { get; set; }
public string url { get; set; }
}
}
}

View file

@ -1,7 +1,9 @@
using BirdsiteLive.ActivityPub.Converters;
using Newtonsoft.Json;
using System.Collections.Generic;
using BirdsiteLive.ActivityPub.Converters;
using Newtonsoft.Json;
namespace BirdsiteLive.ActivityPub.Models
{
public class Note
@ -20,13 +22,16 @@ namespace BirdsiteLive.ActivityPub.Models
public string[] to { get; set; }
public string[] cc { get; set; }
public bool sensitive { get; set; }
//public string conversation { get; set; }
public string content { get; set; }
//public Dictionary<string,string> contentMap { get; set; }
public Attachment[] attachment { get; set; }
public Tag[] tag { get; set; }
//public Dictionary<string, string> replies;
public string quoteUrl { get; set; }
}
}
}

View file

@ -1,4 +1,4 @@
namespace BirdsiteLive.ActivityPub
namespace BirdsiteLive.ActivityPub
{
public class PublicKey
{
@ -6,4 +6,4 @@
public string owner { get; set; }
public string publicKeyPem { get; set; }
}
}
}

View file

@ -1,8 +1,9 @@
using System;
using System;
namespace BirdsiteLive.ActivityPub.Models
{
public class Tag {
public class Tag
{
public TagResource icon { get; set; } = null;
public string id { get; set; }
public string type { get; set; } //Hashtag
@ -16,4 +17,4 @@ namespace BirdsiteLive.ActivityPub.Models
public string type { get; set; }
public string url { get; set; }
}
}
}

View file

@ -1,4 +1,4 @@
namespace BirdsiteLive.ActivityPub
namespace BirdsiteLive.ActivityPub
{
public class UserAttachment
{
@ -6,4 +6,4 @@
public string name { get; set; }
public string value { get; set; }
}
}
}

View file

@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Text;

View file

@ -1,4 +1,4 @@
using System.Collections.Generic;
using System.Collections.Generic;
using System.Linq;
namespace BirdsiteLive.Common.Extensions
@ -13,4 +13,4 @@ namespace BirdsiteLive.Common.Extensions
}
}
}
}
}

File diff suppressed because one or more lines are too long

View file

@ -1,10 +1,12 @@
using System.Text.RegularExpressions;
using System.Text.RegularExpressions;
namespace BirdsiteLive.Common.Regexes
{
public class HashtagRegexes
{
public static readonly Regex HashtagName = new Regex(@"^[a-zA-Z0-9_]+$");
public static readonly Regex Hashtag = new Regex(@"(.?)#([a-zA-Z0-9_]+)(\s|$|[\[\]<>.,;:!?/|-])");
public static readonly Regex Hashtag = new Regex(
@"(.?)#([a-zA-Z0-9_]+)(\s|$|[\[\]<>.,;:!?/|-])"
);
}
}
}

View file

@ -1,4 +1,4 @@
using System.Text.RegularExpressions;
using System.Text.RegularExpressions;
namespace BirdsiteLive.Common.Regexes
{
@ -6,4 +6,4 @@ namespace BirdsiteLive.Common.Regexes
{
public static readonly Regex HeaderSignature = new Regex(@"^([a-zA-Z0-9]+)=""(.+)""$");
}
}
}

View file

@ -1,11 +1,11 @@
using System.Text.RegularExpressions;
using System.Text.RegularExpressions;
namespace BirdsiteLive.Common.Regexes
{
public class UrlRegexes
{
public static readonly Regex Url = new Regex(@"(.?)(((http|ftp|https):\/\/)[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&amp;:/~\+#]*[\w\-\@?^=%&amp;/~\+#])?)");
public static readonly Regex Domain = new Regex(@"^[a-zA-Z0-9\-_]+(\.[a-zA-Z0-9\-_]+)+$");
public static readonly Regex Url = new Regex(
@"(.?)(((http|ftp|https):\/\/)[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&amp;:/~\+#]*[\w\-\@?^=%&amp;/~\+#])?)"
);
}
}
}

View file

@ -1,10 +1,12 @@
using System.Text.RegularExpressions;
using System.Text.RegularExpressions;
namespace BirdsiteLive.Common.Regexes
{
public class UserRegexes
{
public static readonly Regex TwitterAccount = new Regex(@"^[a-zA-Z0-9_]+$");
public static readonly Regex Mention = new Regex(@"(.?)@([a-zA-Z0-9_]+)(\s|$|[\[\]<>,;:!?/|-]|(. ))");
public static readonly Regex Mention = new Regex(
@"(.?)@([a-zA-Z0-9_]+)(\s|$|[\[\]<>,;:!?/|-]|(. ))"
);
}
}
}

View file

@ -1,4 +1,4 @@
namespace BirdsiteLive.Common.Settings
namespace BirdsiteLive.Common.Settings
{
public class DbSettings
{
@ -8,4 +8,4 @@
public string User { get; set; }
public string Password { get; set; }
}
}
}

View file

@ -1,4 +1,4 @@
namespace BirdsiteLive.Common.Settings
namespace BirdsiteLive.Common.Settings
{
public class InstanceSettings
{
@ -30,8 +30,5 @@
public int MaxStatusFetchAge { get; set; }
public bool EnableQuoteRT { get; set; }
public int FailingFollowerCleanUpThreshold { get; set; } = -1;
public int UserCacheCapacity { get; set; }
}
}

View file

@ -1,8 +1,8 @@
namespace BirdsiteLive.Common.Settings
namespace BirdsiteLive.Common.Settings
{
public class LogsSettings
{
public string Type { get; set; }
public string InstrumentationKey { get; set; }
}
}
}

View file

@ -1,4 +1,4 @@
namespace BirdsiteLive.Common.Settings
namespace BirdsiteLive.Common.Settings
{
public class ModerationSettings
{
@ -7,4 +7,4 @@
public string TwitterAccountsWhiteListing { get; set; }
public string TwitterAccountsBlackListing { get; set; }
}
}
}

View file

@ -1,8 +1,8 @@
namespace BirdsiteLive.Common.Settings
namespace BirdsiteLive.Common.Settings
{
public class TwitterSettings
{
public string ConsumerKey { get; set; }
public string ConsumerSecret { get; set; }
}
}
}

View file

@ -1,7 +1,7 @@
namespace BirdsiteLive.Common.Structs
namespace BirdsiteLive.Common.Structs
{
public struct DbTypes
{
public static string Postgres = "postgres";
}
}
}

View file

@ -1,8 +1,10 @@
using System.Linq;
using System.Linq;
using System.Security.Cryptography;
using Asn1;
using Asn1Sequence = Asn1.Asn1Sequence;
using Asn1Null = Asn1.Asn1Null;
using Asn1Sequence = Asn1.Asn1Sequence;
namespace BirdsiteLive.Cryptography
{
@ -46,4 +48,4 @@ namespace BirdsiteLive.Cryptography
return result.GetBytes();
}
}
}
}

View file

@ -6,7 +6,7 @@
<ItemGroup>
<PackageReference Include="Asn1" Version="1.0.9" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
<PackageReference Include="Portable.BouncyCastle" Version="1.9.0" />
</ItemGroup>

View file

@ -1,7 +1,8 @@
using System;
using System;
using System.Collections.Generic;
using System.Security.Cryptography;
using System.Text;
using Newtonsoft.Json;
namespace BirdsiteLive.Cryptography
@ -86,7 +87,8 @@ namespace BirdsiteLive.Cryptography
else
{
_parts = key.Split('.');
if (_parts[0] != "RSA") throw new Exception("Unknown magic key!");
if (_parts[0] != "RSA")
throw new Exception("Unknown magic key!");
var rsaParams = new RSAParameters();
rsaParams.Modulus = _decodeBase64Url(_parts[1]);
@ -102,18 +104,37 @@ namespace BirdsiteLive.Cryptography
var rsa = RSA.Create();
rsa.KeySize = 2048;
return new MagicKey(JsonConvert.SerializeObject(RSAKeyParms.From(rsa.ExportParameters(true))));
return new MagicKey(
JsonConvert.SerializeObject(RSAKeyParms.From(rsa.ExportParameters(true)))
);
}
public byte[] BuildSignedData(string data, string dataType, string encoding, string algorithm)
public byte[] BuildSignedData(
string data,
string dataType,
string encoding,
string algorithm
)
{
var sig = data + "." + _encodeBase64Url(Encoding.UTF8.GetBytes(dataType)) + "." + _encodeBase64Url(Encoding.UTF8.GetBytes(encoding)) + "." + _encodeBase64Url(Encoding.UTF8.GetBytes(algorithm));
var sig =
data
+ "."
+ _encodeBase64Url(Encoding.UTF8.GetBytes(dataType))
+ "."
+ _encodeBase64Url(Encoding.UTF8.GetBytes(encoding))
+ "."
+ _encodeBase64Url(Encoding.UTF8.GetBytes(algorithm));
return Encoding.UTF8.GetBytes(sig);
}
public bool Verify(string signature, byte[] data)
{
return _rsa.VerifyData(data, _decodeBase64Url(signature), HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
return _rsa.VerifyData(
data,
_decodeBase64Url(signature),
HashAlgorithmName.SHA256,
RSASignaturePadding.Pkcs1
);
}
public byte[] Sign(byte[] data)
@ -140,7 +161,10 @@ namespace BirdsiteLive.Cryptography
public string PrivateKey
{
get { return JsonConvert.SerializeObject(RSAKeyParms.From(_rsa.ExportParameters(true))); }
get
{
return JsonConvert.SerializeObject(RSAKeyParms.From(_rsa.ExportParameters(true)));
}
}
public string PublicKey
@ -149,8 +173,13 @@ namespace BirdsiteLive.Cryptography
{
var parms = _rsa.ExportParameters(false);
return string.Join(".", "RSA", _encodeBase64Url(parms.Modulus), _encodeBase64Url(parms.Exponent));
return string.Join(
".",
"RSA",
_encodeBase64Url(parms.Modulus),
_encodeBase64Url(parms.Exponent)
);
}
}
}
}
}

View file

@ -1,4 +1,4 @@
using System;
using System;
using System.IO;
using System.Security.Cryptography;
@ -13,7 +13,7 @@ namespace BirdsiteLive.Cryptography
public string GetRsa()
{
var rsa = RSA.Create();
var outputStream = new StringWriter();
var parameters = rsa.ExportParameters(true);
using (var stream = new MemoryStream())
@ -26,7 +26,18 @@ namespace BirdsiteLive.Cryptography
innerWriter.Write((byte)0x30); // SEQUENCE
EncodeLength(innerWriter, 13);
innerWriter.Write((byte)0x06); // OBJECT IDENTIFIER
var rsaEncryptionOid = new byte[] { 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01 };
var rsaEncryptionOid = new byte[]
{
0x2a,
0x86,
0x48,
0x86,
0xf7,
0x0d,
0x01,
0x01,
0x01
};
EncodeLength(innerWriter, rsaEncryptionOid.Length);
innerWriter.Write(rsaEncryptionOid);
innerWriter.Write((byte)0x05); // NULL
@ -55,7 +66,9 @@ namespace BirdsiteLive.Cryptography
writer.Write(innerStream.GetBuffer(), 0, length);
}
var base64 = Convert.ToBase64String(stream.GetBuffer(), 0, (int)stream.Length).ToCharArray();
var base64 = Convert
.ToBase64String(stream.GetBuffer(), 0, (int)stream.Length)
.ToCharArray();
// WriteLine terminates with \r\n, we want only \n
outputStream.Write("-----BEGIN PUBLIC KEY-----\n");
for (var i = 0; i < base64.Length; i += 64)
@ -71,7 +84,8 @@ namespace BirdsiteLive.Cryptography
private static void EncodeLength(BinaryWriter stream, int length)
{
if (length < 0) throw new ArgumentOutOfRangeException("length", "Length must be non-negative");
if (length < 0)
throw new ArgumentOutOfRangeException("length", "Length must be non-negative");
if (length < 0x80)
{
// Short form
@ -94,6 +108,5 @@ namespace BirdsiteLive.Cryptography
}
}
}
}
}

View file

@ -1,11 +1,12 @@
using Org.BouncyCastle.Crypto;
using Org.BouncyCastle.Crypto.Parameters;
using Org.BouncyCastle.OpenSsl;
using Org.BouncyCastle.Security;
using System;
using System.IO;
using System.Security.Cryptography;
using Org.BouncyCastle.Crypto;
using Org.BouncyCastle.Crypto.Parameters;
using Org.BouncyCastle.OpenSsl;
using Org.BouncyCastle.Security;
namespace MyProject.Data.Encryption
{
public class RSAKeys
@ -19,9 +20,11 @@ namespace MyProject.Data.Encryption
{
PemReader pr = new PemReader(new StringReader(pem));
AsymmetricCipherKeyPair KeyPair = (AsymmetricCipherKeyPair)pr.ReadObject();
RSAParameters rsaParams = DotNetUtilities.ToRSAParameters((RsaPrivateCrtKeyParameters)KeyPair.Private);
RSAParameters rsaParams = DotNetUtilities.ToRSAParameters(
(RsaPrivateCrtKeyParameters)KeyPair.Private
);
RSACryptoServiceProvider csp = new RSACryptoServiceProvider();// cspParams);
RSACryptoServiceProvider csp = new RSACryptoServiceProvider(); // cspParams);
csp.ImportParameters(rsaParams);
return csp;
}
@ -37,7 +40,7 @@ namespace MyProject.Data.Encryption
AsymmetricKeyParameter publicKey = (AsymmetricKeyParameter)pr.ReadObject();
RSAParameters rsaParams = DotNetUtilities.ToRSAParameters((RsaKeyParameters)publicKey);
RSACryptoServiceProvider csp = new RSACryptoServiceProvider();// cspParams);
RSACryptoServiceProvider csp = new RSACryptoServiceProvider(); // cspParams);
csp.ImportParameters(rsaParams);
return csp;
}
@ -51,7 +54,8 @@ namespace MyProject.Data.Encryption
public static string ExportPrivateKey(RSACryptoServiceProvider csp)
{
StringWriter outputStream = new StringWriter();
if (csp.PublicOnly) throw new ArgumentException("CSP does not contain a private key", "csp");
if (csp.PublicOnly)
throw new ArgumentException("CSP does not contain a private key", "csp");
var parameters = csp.ExportParameters(true);
using (var stream = new MemoryStream())
{
@ -74,7 +78,9 @@ namespace MyProject.Data.Encryption
writer.Write(innerStream.GetBuffer(), 0, length);
}
var base64 = Convert.ToBase64String(stream.GetBuffer(), 0, (int)stream.Length).ToCharArray();
var base64 = Convert
.ToBase64String(stream.GetBuffer(), 0, (int)stream.Length)
.ToCharArray();
// WriteLine terminates with \r\n, we want only \n
outputStream.Write("-----BEGIN RSA PRIVATE KEY-----\n");
// Output as Base64 with lines chopped at 64 characters
@ -109,7 +115,18 @@ namespace MyProject.Data.Encryption
innerWriter.Write((byte)0x30); // SEQUENCE
EncodeLength(innerWriter, 13);
innerWriter.Write((byte)0x06); // OBJECT IDENTIFIER
var rsaEncryptionOid = new byte[] { 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01 };
var rsaEncryptionOid = new byte[]
{
0x2a,
0x86,
0x48,
0x86,
0xf7,
0x0d,
0x01,
0x01,
0x01
};
EncodeLength(innerWriter, rsaEncryptionOid.Length);
innerWriter.Write(rsaEncryptionOid);
innerWriter.Write((byte)0x05); // NULL
@ -138,7 +155,9 @@ namespace MyProject.Data.Encryption
writer.Write(innerStream.GetBuffer(), 0, length);
}
var base64 = Convert.ToBase64String(stream.GetBuffer(), 0, (int)stream.Length).ToCharArray();
var base64 = Convert
.ToBase64String(stream.GetBuffer(), 0, (int)stream.Length)
.ToCharArray();
// WriteLine terminates with \r\n, we want only \n
outputStream.Write("-----BEGIN PUBLIC KEY-----\n");
for (var i = 0; i < base64.Length; i += 64)
@ -159,7 +178,8 @@ namespace MyProject.Data.Encryption
/// <param name="length"></param>
private static void EncodeLength(BinaryWriter stream, int length)
{
if (length < 0) throw new ArgumentOutOfRangeException("length", "Length must be non-negative");
if (length < 0)
throw new ArgumentOutOfRangeException("length", "Length must be non-negative");
if (length < 0x80)
{
// Short form
@ -189,13 +209,18 @@ namespace MyProject.Data.Encryption
/// <param name="stream"></param>
/// <param name="value"></param>
/// <param name="forceUnsigned"></param>
private static void EncodeIntegerBigEndian(BinaryWriter stream, byte[] value, bool forceUnsigned = true)
private static void EncodeIntegerBigEndian(
BinaryWriter stream,
byte[] value,
bool forceUnsigned = true
)
{
stream.Write((byte)0x02); // INTEGER
var prefixZeros = 0;
for (var i = 0; i < value.Length; i++)
{
if (value[i] != 0) break;
if (value[i] != 0)
break;
prefixZeros++;
}
if (value.Length - prefixZeros == 0)
@ -222,4 +247,4 @@ namespace MyProject.Data.Encryption
}
}
}
}
}

View file

@ -1,35 +1,41 @@
using System;
using System;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Runtime.InteropServices;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using BirdsiteLive.ActivityPub;
using BirdsiteLive.ActivityPub.Converters;
using BirdsiteLive.ActivityPub.Models;
using BirdsiteLive.Common.Settings;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Org.BouncyCastle.Bcpg;
namespace BirdsiteLive.Domain
{
public interface IActivityPubService
{
Task<string> GetUserIdAsync(string acct);
Task<Actor> GetUser(string objectId);
Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null);
Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost,
string targetInbox);
Task DeleteUserAsync(string username, string targetHost, string targetInbox);
Task<WebFingerData> WebFinger(string account);
}
public class WebFinger
{
public string subject { get; set; }
public string[] aliases { get; set; }
Task<HttpStatusCode> PostDataAsync<T>(
T data,
string targetHost,
string actorUrl,
string inbox = null
);
Task PostNewNoteActivity(
Note note,
string username,
string noteId,
string targetHost,
string targetInbox
);
Task<WebFingerData> WebFinger(string account);
}
public class ActivityPubService : IActivityPubService
@ -40,7 +46,12 @@ namespace BirdsiteLive.Domain
private readonly ILogger<ActivityPubService> _logger;
#region Ctor
public ActivityPubService(ICryptoService cryptoService, InstanceSettings instanceSettings, IHttpClientFactory httpClientFactory, ILogger<ActivityPubService> logger)
public ActivityPubService(
ICryptoService cryptoService,
InstanceSettings instanceSettings,
IHttpClientFactory httpClientFactory,
ILogger<ActivityPubService> logger
)
{
_cryptoService = cryptoService;
_instanceSettings = instanceSettings;
@ -49,68 +60,26 @@ namespace BirdsiteLive.Domain
}
#endregion
public async Task<string> GetUserIdAsync(string acct)
{
var splittedAcct = acct.Trim('@').Split('@');
var url = $"https://{splittedAcct[1]}/.well-known/webfinger?resource=acct:{splittedAcct[0]}@{splittedAcct[1]}";
var httpClient = _httpClientFactory.CreateClient("BirdsiteLIVE");
httpClient.DefaultRequestHeaders.Add("Accept", "application/json");
var result = await httpClient.GetAsync(url);
result.EnsureSuccessStatusCode();
var content = await result.Content.ReadAsStringAsync();
var actor = JsonConvert.DeserializeObject<WebFinger>(content);
return actor.aliases.FirstOrDefault();
}
public async Task<Actor> GetUser(string objectId)
{
var httpClient = _httpClientFactory.CreateClient("BirdsiteLIVE");
var httpClient = _httpClientFactory.CreateClient();
httpClient.DefaultRequestHeaders.Add("Accept", "application/activity+json");
var result = await httpClient.GetAsync(objectId);
if (result.StatusCode == HttpStatusCode.Gone)
throw new FollowerIsGoneException();
result.EnsureSuccessStatusCode();
var content = await result.Content.ReadAsStringAsync();
var actor = JsonConvert.DeserializeObject<Actor>(content);
if (string.IsNullOrWhiteSpace(actor.url)) actor.url = objectId;
if (string.IsNullOrWhiteSpace(actor.url))
actor.url = objectId;
return actor;
}
public async Task DeleteUserAsync(string username, string targetHost, string targetInbox)
{
try
{
var actor = UrlFactory.GetActorUrl(_instanceSettings.Domain, username);
var deleteUser = new ActivityDelete
{
context = "https://www.w3.org/ns/activitystreams",
id = $"{actor}#delete",
type = "Delete",
actor = actor,
to = new [] { "https://www.w3.org/ns/activitystreams#Public" },
apObject = actor
};
await PostDataAsync(deleteUser, targetHost, actor, targetInbox);
}
catch (Exception e)
{
_logger.LogError(e, "Error deleting {Username} to {Host}{Inbox}", username, targetHost, targetInbox);
throw;
}
}
public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox)
public async Task PostNewNoteActivity(
Note note,
string username,
string noteId,
string targetHost,
string targetInbox
)
{
try
{
@ -127,7 +96,6 @@ namespace BirdsiteLive.Domain
type = "Create",
actor = actor,
published = nowString,
to = note.to,
cc = note.cc,
apObject = note
@ -137,12 +105,24 @@ namespace BirdsiteLive.Domain
}
catch (Exception e)
{
_logger.LogError(e, "Error sending {Username} post ({NoteId}) to {Host}{Inbox}", username, noteId, targetHost, targetInbox);
_logger.LogError(
e,
"Error sending {Username} post ({NoteId}) to {Host}{Inbox}",
username,
noteId,
targetHost,
targetInbox
);
throw;
}
}
public async Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null)
public async Task<HttpStatusCode> PostDataAsync<T>(
T data,
string targetHost,
string actorUrl,
string inbox = null
)
{
var usedInbox = $"/inbox";
if (!string.IsNullOrWhiteSpace(inbox))
@ -155,19 +135,25 @@ namespace BirdsiteLive.Domain
var digest = _cryptoService.ComputeSha256Hash(json);
var signature = _cryptoService.SignAndGetSignatureHeader(date, actorUrl, targetHost, digest, usedInbox);
var signature = _cryptoService.SignAndGetSignatureHeader(
date,
actorUrl,
targetHost,
digest,
usedInbox
);
var client = _httpClientFactory.CreateClient("BirdsiteLIVE");
var client = _httpClientFactory.CreateClient();
var httpRequestMessage = new HttpRequestMessage
{
Method = HttpMethod.Post,
RequestUri = new Uri($"https://{targetHost}{usedInbox}"),
Headers =
{
{"Host", targetHost},
{"Date", httpDate},
{"Signature", signature},
{"Digest", $"SHA-256={digest}"}
{ "Host", targetHost },
{ "Date", httpDate },
{ "Signature", signature },
{ "Digest", $"SHA-256={digest}" }
},
Content = new StringContent(json, Encoding.UTF8, "application/ld+json")
};
@ -179,11 +165,16 @@ namespace BirdsiteLive.Domain
public async Task<WebFingerData> WebFinger(string account)
{
var httpClient = _httpClientFactory.CreateClient("BirdsiteLIVE");
var result = await httpClient.GetAsync("https://" + account.Split('@')[1] + "/.well-known/webfinger?resource=acct:" + account);
var httpClient = _httpClientFactory.CreateClient();
var result = await httpClient.GetAsync(
"https://"
+ account.Split('@')[1]
+ "/.well-known/webfinger?resource=acct:"
+ account
);
var content = await result.Content.ReadAsStringAsync();
return JsonConvert.DeserializeObject<WebFingerData>(content);
}
}
}
}

View file

@ -15,8 +15,4 @@
<ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
</ItemGroup>
<ItemGroup>
<Folder Include="Enum\" />
</ItemGroup>
</Project>

View file

@ -1,51 +0,0 @@
using System.Linq;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
namespace BirdsiteLive.Domain.BusinessUseCases
{
public interface IProcessDeleteUser
{
Task ExecuteAsync(Follower follower);
Task ExecuteAsync(string followerUsername, string followerDomain);
}
public class ProcessDeleteUser : IProcessDeleteUser
{
private readonly IFollowersDal _followersDal;
private readonly ITwitterUserDal _twitterUserDal;
#region Ctor
public ProcessDeleteUser(IFollowersDal followersDal, ITwitterUserDal twitterUserDal)
{
_followersDal = followersDal;
_twitterUserDal = twitterUserDal;
}
#endregion
public async Task ExecuteAsync(string followerUsername, string followerDomain)
{
// Get Follower and Twitter Users
var follower = await _followersDal.GetFollowerAsync(followerUsername, followerDomain);
if (follower == null) return;
await ExecuteAsync(follower);
}
public async Task ExecuteAsync(Follower follower)
{
// Remove twitter users if no more followers
var followings = follower.Followings;
foreach (var following in followings)
{
var followers = await _followersDal.GetFollowersAsync(following);
if (followers.Length == 1 && followers.First().Id == follower.Id)
await _twitterUserDal.DeleteTwitterUserAsync(following);
}
// Remove follower from DB
await _followersDal.DeleteFollowerAsync(follower.Id);
}
}
}

View file

@ -1,11 +1,19 @@
using System.Threading.Tasks;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
namespace BirdsiteLive.Domain.BusinessUseCases
{
public interface IProcessFollowUser
{
Task ExecuteAsync(string followerUsername, string followerDomain, string twitterUsername, string followerInbox, string sharedInbox, string followerActorId);
Task ExecuteAsync(
string followerUsername,
string followerDomain,
string twitterUsername,
string followerInbox,
string sharedInbox,
string followerActorId
);
}
public class ProcessFollowUser : IProcessFollowUser
@ -21,13 +29,26 @@ namespace BirdsiteLive.Domain.BusinessUseCases
}
#endregion
public async Task ExecuteAsync(string followerUsername, string followerDomain, string twitterUsername, string followerInbox, string sharedInbox, string followerActorId)
public async Task ExecuteAsync(
string followerUsername,
string followerDomain,
string twitterUsername,
string followerInbox,
string sharedInbox,
string followerActorId
)
{
// Get Follower and Twitter Users
var follower = await _followerDal.GetFollowerAsync(followerUsername, followerDomain);
if (follower == null)
{
await _followerDal.CreateFollowerAsync(followerUsername, followerDomain, followerInbox, sharedInbox, followerActorId);
await _followerDal.CreateFollowerAsync(
followerUsername,
followerDomain,
followerInbox,
sharedInbox,
followerActorId
);
follower = await _followerDal.GetFollowerAsync(followerUsername, followerDomain);
}
@ -40,14 +61,14 @@ namespace BirdsiteLive.Domain.BusinessUseCases
// Update Follower
var twitterUserId = twitterUser.Id;
if(!follower.Followings.Contains(twitterUserId))
if (!follower.Followings.Contains(twitterUserId))
follower.Followings.Add(twitterUserId);
if(!follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
if (!follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
follower.FollowingsSyncStatus.Add(twitterUserId, -1);
// Save Follower
await _followerDal.UpdateFollowerAsync(follower);
}
}
}
}

View file

@ -1,5 +1,6 @@
using System.Linq;
using System.Linq;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
namespace BirdsiteLive.Domain.BusinessUseCases
@ -22,14 +23,20 @@ namespace BirdsiteLive.Domain.BusinessUseCases
}
#endregion
public async Task ExecuteAsync(string followerUsername, string followerDomain, string twitterUsername)
public async Task ExecuteAsync(
string followerUsername,
string followerDomain,
string twitterUsername
)
{
// Get Follower and Twitter Users
var follower = await _followerDal.GetFollowerAsync(followerUsername, followerDomain);
if (follower == null) return;
if (follower == null)
return;
var twitterUser = await _twitterUserDal.GetTwitterUserAsync(twitterUsername);
if (twitterUser == null) return;
if (twitterUser == null)
return;
// Update Follower
var twitterUserId = twitterUser.Id;
@ -43,12 +50,12 @@ namespace BirdsiteLive.Domain.BusinessUseCases
if (follower.Followings.Any())
await _followerDal.UpdateFollowerAsync(follower);
else
await _followerDal.DeleteFollowerAsync(followerUsername, followerDomain);
await _followerDal.DeleteFollowerAsync(followerUsername, followerDomain);
// Check if TwitterUser has still followers
var followers = await _followerDal.GetFollowersAsync(twitterUser.Id);
if (!followers.Any())
await _twitterUserDal.DeleteTwitterUserAsync(twitterUsername);
}
}
}
}

View file

@ -1,6 +1,7 @@
using System;
using System;
using System.Security.Cryptography;
using System.Text;
using BirdsiteLive.Domain.Factories;
namespace BirdsiteLive.Domain
@ -8,7 +9,13 @@ namespace BirdsiteLive.Domain
public interface ICryptoService
{
string GetUserPem(string id);
string SignAndGetSignatureHeader(DateTime date, string actor, string host, string digest, string inbox);
string SignAndGetSignatureHeader(
DateTime date,
string actor,
string host,
string digest,
string inbox
);
string ComputeSha256Hash(string data);
}
@ -29,13 +36,19 @@ namespace BirdsiteLive.Domain
}
/// <summary>
///
///
/// </summary>
/// <param name="data"></param>
/// <param name="actor">in the form of https://domain.io/actor</param>
/// <param name="host">in the form of domain.io</param>
/// <returns></returns>
public string SignAndGetSignatureHeader(DateTime date, string actor, string targethost, string digest, string inbox)
public string SignAndGetSignatureHeader(
DateTime date,
string actor,
string targethost,
string digest,
string inbox
)
{
var usedInbox = "/inbox";
if (!string.IsNullOrWhiteSpace(inbox))
@ -43,24 +56,30 @@ namespace BirdsiteLive.Domain
var httpDate = date.ToString("r");
var signedString = $"(request-target): post {usedInbox}\nhost: {targethost}\ndate: {httpDate}\ndigest: SHA-256={digest}";
var signedString =
$"(request-target): post {usedInbox}\nhost: {targethost}\ndate: {httpDate}\ndigest: SHA-256={digest}";
var signedStringBytes = Encoding.UTF8.GetBytes(signedString);
var signature = _magicKeyFactory.GetMagicKey().Sign(signedStringBytes);
var sig64 = Convert.ToBase64String(signature);
var header = "keyId=\"" + actor + "\",algorithm=\"rsa-sha256\",headers=\"(request-target) host date digest\",signature=\"" + sig64 + "\"";
var header =
"keyId=\""
+ actor
+ "\",algorithm=\"rsa-sha256\",headers=\"(request-target) host date digest\",signature=\""
+ sig64
+ "\"";
return header;
}
public string ComputeSha256Hash(string data)
{
// Create a SHA256
// Create a SHA256
using (SHA256 sha256Hash = SHA256.Create())
{
// ComputeHash - returns byte array
// ComputeHash - returns byte array
byte[] bytes = sha256Hash.ComputeHash(Encoding.UTF8.GetBytes(data));
return Convert.ToBase64String(bytes);
}
}
}
}
}

View file

@ -1,9 +0,0 @@
namespace BirdsiteLive.Domain.Enum
{
public enum MigrationTypeEnum
{
Unknown = 0,
Migration = 1,
Deletion = 2
}
}

View file

@ -1,8 +0,0 @@
using System;
namespace BirdsiteLive.Domain
{
public class FollowerIsGoneException : Exception
{
}
}

View file

@ -1,4 +1,5 @@
using System.IO;
using System.IO;
using BirdsiteLive.Cryptography;
namespace BirdsiteLive.Domain.Factories
@ -14,16 +15,14 @@ namespace BirdsiteLive.Domain.Factories
private static MagicKey _magicKey;
#region Ctor
public MagicKeyFactory()
{
}
public MagicKeyFactory() { }
#endregion
public MagicKey GetMagicKey()
{
//Cached key
if (_magicKey != null) return _magicKey;
if (_magicKey != null)
return _magicKey;
//Generate key if needed
if (!File.Exists(Path))
@ -38,4 +37,4 @@ namespace BirdsiteLive.Domain.Factories
return _magicKey;
}
}
}
}

View file

@ -1,352 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using BirdsiteLive.Twitter;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using BirdsiteLive.ActivityPub;
using BirdsiteLive.ActivityPub.Models;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.ActivityPub.Converters;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Domain.Enum;
using System.Net.Http;
using BirdsiteLive.Common.Regexes;
using Microsoft.Extensions.Logging;
namespace BirdsiteLive.Domain
{
public class MigrationService
{
private readonly InstanceSettings _instanceSettings;
private readonly ITheFedInfoService _theFedInfoService;
private readonly ITwitterTweetsService _twitterTweetsService;
private readonly IActivityPubService _activityPubService;
private readonly ITwitterUserDal _twitterUserDal;
private readonly IFollowersDal _followersDal;
private readonly IHttpClientFactory _httpClientFactory;
private readonly ILogger<MigrationService> _logger;
#region Ctor
public MigrationService(ITwitterTweetsService twitterTweetsService, IActivityPubService activityPubService, ITwitterUserDal twitterUserDal, IFollowersDal followersDal, InstanceSettings instanceSettings, ITheFedInfoService theFedInfoService, IHttpClientFactory httpClientFactory, ILogger<MigrationService> logger)
{
_twitterTweetsService = twitterTweetsService;
_activityPubService = activityPubService;
_twitterUserDal = twitterUserDal;
_followersDal = followersDal;
_instanceSettings = instanceSettings;
_theFedInfoService = theFedInfoService;
_httpClientFactory = httpClientFactory;
_logger = logger;
}
#endregion
public string GetMigrationCode(string acct)
{
var hash = GetHashString(acct);
return $"[[BirdsiteLIVE-MigrationCode|{hash.Substring(0, 10)}]]";
}
public string GetDeletionCode(string acct)
{
var hash = GetHashString(acct);
return $"[[BirdsiteLIVE-DeletionCode|{hash.Substring(0, 10)}]]";
}
public bool ValidateTweet(string acct, string tweetId, MigrationTypeEnum type)
{
string code;
if (type == MigrationTypeEnum.Migration)
code = GetMigrationCode(acct);
else if (type == MigrationTypeEnum.Deletion)
code = GetDeletionCode(acct);
else
throw new NotImplementedException();
var castedTweetId = ExtractedTweetId(tweetId);
var tweet = _twitterTweetsService.GetTweet(castedTweetId);
if (tweet == null)
throw new Exception("Tweet not found");
if (tweet.CreatorName.Trim().ToLowerInvariant() != acct.Trim().ToLowerInvariant())
throw new Exception($"Tweet not published by @{acct}");
if (!tweet.MessageContent.Contains(code))
{
var message = "Tweet don't have migration code";
if (type == MigrationTypeEnum.Deletion)
message = "Tweet don't have deletion code";
throw new Exception(message);
}
return true;
}
private long ExtractedTweetId(string tweetId)
{
if (string.IsNullOrWhiteSpace(tweetId))
throw new ArgumentException("No provided Tweet ID");
long castedId;
if (long.TryParse(tweetId, out castedId))
return castedId;
var urlPart = tweetId.Split('/').LastOrDefault();
if (long.TryParse(urlPart, out castedId))
return castedId;
throw new ArgumentException("Unvalid Tweet ID");
}
public async Task<ValidatedFediverseUser> ValidateFediverseAcctAsync(string fediverseAcct)
{
if (string.IsNullOrWhiteSpace(fediverseAcct))
throw new ArgumentException("Please provide Fediverse account");
if (!fediverseAcct.Contains('@') || !fediverseAcct.StartsWith("@") || fediverseAcct.Trim('@').Split('@').Length != 2)
throw new ArgumentException("Please provide valid Fediverse handle");
var objectId = await _activityPubService.GetUserIdAsync(fediverseAcct);
var user = await _activityPubService.GetUser(objectId);
var result = new ValidatedFediverseUser
{
FediverseAcct = fediverseAcct,
ObjectId = objectId,
User = user,
IsValid = user != null
};
return result;
}
public async Task MigrateAccountAsync(ValidatedFediverseUser validatedUser, string acct)
{
// Apply moved to
var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct);
if (twitterAccount == null)
{
await _twitterUserDal.CreateTwitterUserAsync(acct, -1, validatedUser.ObjectId, validatedUser.FediverseAcct);
twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct);
}
twitterAccount.MovedTo = validatedUser.User.id;
twitterAccount.MovedToAcct = validatedUser.FediverseAcct;
twitterAccount.LastSync = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount);
// Notify Followers
var message = $@"<p>[BSL MIRROR SERVICE NOTIFICATION]<br/>This bot has been disabled by its original owner.<br/>It has been redirected to {validatedUser.FediverseAcct}.</p>";
NotifyFollowers(acct, twitterAccount, message);
}
private void NotifyFollowers(string acct, SyncTwitterUser twitterAccount, string message)
{
var t = Task.Run(async () =>
{
var followers = await _followersDal.GetFollowersAsync(twitterAccount.Id);
foreach (var follower in followers)
{
try
{
var noteId = Guid.NewGuid().ToString();
var actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, acct);
var noteUrl = UrlFactory.GetNoteUrl(_instanceSettings.Domain, acct, noteId);
//var to = validatedUser.ObjectId;
var to = follower.ActorId;
var cc = new string[0];
var note = new Note
{
id = noteUrl,
published = DateTime.UtcNow.ToString("s") + "Z",
url = noteUrl,
attributedTo = actorUrl,
to = new[] { to },
cc = cc,
content = message,
tag = new Tag[]{
new Tag()
{
type = "Mention",
href = follower.ActorId,
name = $"@{follower.Acct}@{follower.Host}"
}
},
};
if (!string.IsNullOrWhiteSpace(follower.SharedInboxRoute))
await _activityPubService.PostNewNoteActivity(note, acct, noteId, follower.Host, follower.SharedInboxRoute);
else
await _activityPubService.PostNewNoteActivity(note, acct, noteId, follower.Host, follower.InboxRoute);
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
}
}
});
}
public async Task DeleteAccountAsync(string acct)
{
// Apply deleted state
var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct);
if (twitterAccount == null)
{
await _twitterUserDal.CreateTwitterUserAsync(acct, -1);
twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct);
}
twitterAccount.Deleted = true;
twitterAccount.LastSync = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount);
// Notify Followers
var message = $@"<p>[BSL MIRROR SERVICE NOTIFICATION]<br/>This bot has been deleted by its original owner.<br/></p>";
NotifyFollowers(acct, twitterAccount, message);
// Delete remote accounts
DeleteRemoteAccounts(acct);
}
private void DeleteRemoteAccounts(string acct)
{
var t = Task.Run(async () =>
{
var allUsers = await _followersDal.GetAllFollowersAsync();
var followersWtSharedInbox = allUsers
.Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
.GroupBy(x => x.Host)
.ToList();
foreach (var followerGroup in followersWtSharedInbox)
{
var host = followerGroup.First().Host;
var sharedInbox = followerGroup.First().SharedInboxRoute;
var t1 = Task.Run(async () =>
{
try
{
await _activityPubService.DeleteUserAsync(acct, host, sharedInbox);
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
}
});
}
var followerWtInbox = allUsers
.Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
.ToList();
foreach (var followerGroup in followerWtInbox)
{
var host = followerGroup.Host;
var sharedInbox = followerGroup.InboxRoute;
var t1 = Task.Run(async () =>
{
try
{
await _activityPubService.DeleteUserAsync(acct, host, sharedInbox);
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
}
});
}
});
}
public async Task TriggerRemoteMigrationAsync(string id, string tweetIdStg, string handle)
{
var url = $"https://{{0}}/migration/move/{{1}}/{{2}}/{handle}";
await ProcessRemoteMigrationAsync(id, tweetIdStg, url);
}
public async Task TriggerRemoteDeleteAsync(string id, string tweetIdStg)
{
var url = $"https://{{0}}/migration/delete/{{1}}/{{2}}";
await ProcessRemoteMigrationAsync(id, tweetIdStg, url);
}
private async Task ProcessRemoteMigrationAsync(string id, string tweetIdStg, string urlPattern)
{
try
{
var instances = await RetrieveCompatibleBslInstancesAsync();
var tweetId = ExtractedTweetId(tweetIdStg);
foreach (var instance in instances)
{
try
{
var host = instance.Host;
if(!UrlRegexes.Domain.IsMatch(host)) continue;
var url = string.Format(urlPattern, host, id, tweetId);
var client = _httpClientFactory.CreateClient("BirdsiteLIVE");
var result = await client.PostAsync(url, null);
result.EnsureSuccessStatusCode();
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
}
}
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
}
}
private async Task<List<BslInstanceInfo>> RetrieveCompatibleBslInstancesAsync()
{
var instances = await _theFedInfoService.GetBslInstanceListAsync();
var filteredInstances = instances
.Where(x => x.Version >= new Version(0, 21, 0))
.Where(x => string.Compare(x.Host,
_instanceSettings.Domain,
StringComparison.InvariantCultureIgnoreCase) != 0)
.ToList();
return filteredInstances;
}
private byte[] GetHash(string inputString)
{
using (HashAlgorithm algorithm = SHA256.Create())
return algorithm.ComputeHash(Encoding.UTF8.GetBytes(inputString));
}
private string GetHashString(string inputString)
{
StringBuilder sb = new StringBuilder();
foreach (byte b in GetHash(inputString))
sb.Append(b.ToString("X2"));
return sb.ToString();
}
}
public class ValidatedFediverseUser
{
public string FediverseAcct { get; set; }
public string ObjectId { get; set; }
public Actor User { get; set; }
public bool IsValid { get; set; }
}
}

View file

@ -1,7 +1,8 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.Domain.Tools;
@ -38,8 +39,12 @@ namespace BirdsiteLive.Domain.Repository
var parsedFollowersWhiteListing = PatternsParser.Parse(settings.FollowersWhiteListing);
var parsedFollowersBlackListing = PatternsParser.Parse(settings.FollowersBlackListing);
var parsedTwitterAccountsWhiteListing = PatternsParser.Parse(settings.TwitterAccountsWhiteListing);
var parsedTwitterAccountsBlackListing = PatternsParser.Parse(settings.TwitterAccountsBlackListing);
var parsedTwitterAccountsWhiteListing = PatternsParser.Parse(
settings.TwitterAccountsWhiteListing
);
var parsedTwitterAccountsBlackListing = PatternsParser.Parse(
settings.TwitterAccountsBlackListing
);
_followersWhiteListing = parsedFollowersWhiteListing
.Select(x => ModerationRegexParser.Parse(ModerationEntityTypeEnum.Follower, x))
@ -48,10 +53,14 @@ namespace BirdsiteLive.Domain.Repository
.Select(x => ModerationRegexParser.Parse(ModerationEntityTypeEnum.Follower, x))
.ToArray();
_twitterAccountsWhiteListing = parsedTwitterAccountsWhiteListing
.Select(x => ModerationRegexParser.Parse(ModerationEntityTypeEnum.TwitterAccount, x))
.Select(
x => ModerationRegexParser.Parse(ModerationEntityTypeEnum.TwitterAccount, x)
)
.ToArray();
_twitterAccountsBlackListing = parsedTwitterAccountsBlackListing
.Select(x => ModerationRegexParser.Parse(ModerationEntityTypeEnum.TwitterAccount, x))
.Select(
x => ModerationRegexParser.Parse(ModerationEntityTypeEnum.TwitterAccount, x)
)
.ToArray();
// Set Follower moderation politic
@ -64,9 +73,15 @@ namespace BirdsiteLive.Domain.Repository
// Set Twitter account moderation politic
if (_twitterAccountsWhiteListing.Any())
_modMode.Add(ModerationEntityTypeEnum.TwitterAccount, ModerationTypeEnum.WhiteListing);
_modMode.Add(
ModerationEntityTypeEnum.TwitterAccount,
ModerationTypeEnum.WhiteListing
);
else if (_twitterAccountsBlackListing.Any())
_modMode.Add(ModerationEntityTypeEnum.TwitterAccount, ModerationTypeEnum.BlackListing);
_modMode.Add(
ModerationEntityTypeEnum.TwitterAccount,
ModerationTypeEnum.BlackListing
);
else
_modMode.Add(ModerationEntityTypeEnum.TwitterAccount, ModerationTypeEnum.None);
}
@ -79,7 +94,8 @@ namespace BirdsiteLive.Domain.Repository
public ModeratedTypeEnum CheckStatus(ModerationEntityTypeEnum type, string entity)
{
if (_modMode[type] == ModerationTypeEnum.None) return ModeratedTypeEnum.None;
if (_modMode[type] == ModerationTypeEnum.None)
return ModeratedTypeEnum.None;
switch (type)
{
@ -91,7 +107,7 @@ namespace BirdsiteLive.Domain.Repository
throw new NotImplementedException($"Type {type} is not supported");
}
private ModeratedTypeEnum ProcessFollower(string entity)
{
var politic = _modMode[ModerationEntityTypeEnum.Follower];
@ -137,30 +153,40 @@ namespace BirdsiteLive.Domain.Repository
private char GetSplitChar(string entry)
{
var separationChar = '|';
if (entry.Contains(";")) separationChar = ';';
else if (entry.Contains(",")) separationChar = ',';
if (entry.Contains(";"))
separationChar = ';';
else if (entry.Contains(","))
separationChar = ',';
return separationChar;
}
public IEnumerable<string> GetWhitelistedFollowers()
{
return _settings.FollowersWhiteListing.Split(GetSplitChar(_settings.FollowersWhiteListing));
return _settings.FollowersWhiteListing.Split(
GetSplitChar(_settings.FollowersWhiteListing)
);
}
public IEnumerable<string> GetBlacklistedFollowers()
{
return _settings.FollowersBlackListing.Split(GetSplitChar(_settings.FollowersBlackListing));
return _settings.FollowersBlackListing.Split(
GetSplitChar(_settings.FollowersBlackListing)
);
}
public IEnumerable<string> GetWhitelistedAccounts()
{
return _settings.TwitterAccountsWhiteListing.Split(GetSplitChar(_settings.TwitterAccountsWhiteListing));
return _settings.TwitterAccountsWhiteListing.Split(
GetSplitChar(_settings.TwitterAccountsWhiteListing)
);
}
public IEnumerable<string> GetBlacklistedAccounts()
{
return _settings.TwitterAccountsBlackListing.Split(GetSplitChar(_settings.TwitterAccountsBlackListing));
return _settings.TwitterAccountsBlackListing.Split(
GetSplitChar(_settings.TwitterAccountsBlackListing)
);
}
}
@ -184,4 +210,4 @@ namespace BirdsiteLive.Domain.Repository
BlackListed = 1,
WhiteListed = 2
}
}
}

View file

@ -1,4 +1,5 @@
using System.Linq;
using System.Linq;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.Domain.Tools;
@ -25,14 +26,16 @@ namespace BirdsiteLive.Domain.Repository
public bool IsUnlisted(string twitterAcct)
{
if (_unlistedAccounts == null || !_unlistedAccounts.Any()) return false;
if (_unlistedAccounts == null || !_unlistedAccounts.Any())
return false;
return _unlistedAccounts.Contains(twitterAcct.ToLowerInvariant());
}
public bool IsSensitive(string twitterAcct)
{
if (_sensitiveAccounts == null || !_sensitiveAccounts.Any()) return false;
if (_sensitiveAccounts == null || !_sensitiveAccounts.Any())
return false;
return _sensitiveAccounts.Contains(twitterAcct.ToLowerInvariant());
}

View file

@ -1,4 +1,4 @@
using System.Threading;
using System.Threading;
using System.Timers;
namespace BirdsiteLive.Domain.Statistics
@ -57,14 +57,24 @@ namespace BirdsiteLive.Domain.Statistics
public ExtractionStatistics GetStatistics()
{
return new ExtractionStatistics(_descriptionMentionsExtracted, _statusMentionsExtracted, _lastDescriptionMentionsExtracted, _lastStatusMentionsExtracted);
return new ExtractionStatistics(
_descriptionMentionsExtracted,
_statusMentionsExtracted,
_lastDescriptionMentionsExtracted,
_lastStatusMentionsExtracted
);
}
}
public class ExtractionStatistics
{
#region Ctor
public ExtractionStatistics(int mentionsInDescriptionsExtraction, int mentionsInStatusesExtraction, int lastMentionsInDescriptionsExtraction, int lastMentionsInStatusesExtraction)
public ExtractionStatistics(
int mentionsInDescriptionsExtraction,
int mentionsInStatusesExtraction,
int lastMentionsInDescriptionsExtraction,
int lastMentionsInStatusesExtraction
)
{
MentionsInDescriptionsExtraction = mentionsInDescriptionsExtraction;
MentionsInStatusesExtraction = mentionsInStatusesExtraction;
@ -79,4 +89,4 @@ namespace BirdsiteLive.Domain.Statistics
public int LastMentionsInDescriptionsExtraction { get; }
public int LastMentionsInStatusesExtraction { get; }
}
}
}

View file

@ -1,8 +1,9 @@
using System;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using BirdsiteLive.ActivityPub;
using BirdsiteLive.ActivityPub.Converters;
using BirdsiteLive.ActivityPub.Models;
@ -11,6 +12,7 @@ using BirdsiteLive.Domain.Repository;
using BirdsiteLive.Domain.Statistics;
using BirdsiteLive.Domain.Tools;
using BirdsiteLive.Twitter.Models;
using Tweetinvi.Models;
using Tweetinvi.Models.Entities;
@ -29,7 +31,12 @@ namespace BirdsiteLive.Domain
private readonly IPublicationRepository _publicationRepository;
#region Ctor
public StatusService(InstanceSettings instanceSettings, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler, IPublicationRepository publicationRepository)
public StatusService(
InstanceSettings instanceSettings,
IStatusExtractor statusExtractor,
IExtractionStatisticsHandler statisticsHandler,
IPublicationRepository publicationRepository
)
{
_instanceSettings = instanceSettings;
_statusExtractor = statusExtractor;
@ -41,15 +48,19 @@ namespace BirdsiteLive.Domain
public Note GetStatus(string username, ExtractedTweet tweet)
{
var actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, username);
var noteUrl = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, tweet.Id.ToString());
var noteUrl = UrlFactory.GetNoteUrl(
_instanceSettings.Domain,
username,
tweet.Id.ToString()
);
var to = $"{actorUrl}/followers";
var isUnlisted = _publicationRepository.IsUnlisted(username);
var cc = new string[0];
if (isUnlisted)
cc = new[] {"https://www.w3.org/ns/activitystreams#Public"};
cc = new[] { "https://www.w3.org/ns/activitystreams#Public" };
string summary = null;
var sensitive = _publicationRepository.IsSensitive(username);
if (sensitive || tweet.IsSensitive)
@ -63,38 +74,37 @@ namespace BirdsiteLive.Domain
if (content.Contains("{RT}") && tweet.IsRetweet)
{
if (!string.IsNullOrWhiteSpace(tweet.RetweetUrl))
content = content.Replace("{RT}",
$@"<a href=""{tweet.RetweetUrl}"" rel=""nofollow noopener noreferrer"" target=""_blank"">RT</a>");
content = content.Replace(
"{RT}",
$@"<a href=""{tweet.RetweetUrl}"" rel=""nofollow noopener noreferrer"" target=""_blank"">RT</a>"
);
else
content = content.Replace("{RT}", "RT");
}
string inReplyTo = null;
if (tweet.InReplyToStatusId != default)
inReplyTo = $"https://{_instanceSettings.Domain}/users/{tweet.InReplyToAccount.ToLowerInvariant()}/statuses/{tweet.InReplyToStatusId}";
inReplyTo =
$"https://{_instanceSettings.Domain}/users/{tweet.InReplyToAccount.ToLowerInvariant()}/statuses/{tweet.InReplyToStatusId}";
if( tweet.QuoteTweetUrl != null )
content += $@"<span class=""quote-inline""><br><br>RT: <a href=""{tweet.QuoteTweetUrl}"">{tweet.QuoteTweetUrl}</a></span>";
if (tweet.QuoteTweetUrl != null)
content +=
$@"<span class=""quote-inline""><br><br>RT: <a href=""{tweet.QuoteTweetUrl}"">{tweet.QuoteTweetUrl}</a></span>";
var note = new Note
{
id = noteUrl,
published = tweet.CreatedAt.ToString("s") + "Z",
url = noteUrl,
attributedTo = actorUrl,
inReplyTo = inReplyTo,
to = new[] { to },
cc = cc,
sensitive = tweet.IsSensitive || sensitive,
summary = summary,
content = $"<p>{content}</p>",
attachment = Convert(tweet.Media),
tag = extractedTags.tags,
quoteUrl = tweet.QuoteTweetUrl
};
@ -103,16 +113,19 @@ namespace BirdsiteLive.Domain
private Attachment[] Convert(ExtractedMedia[] media)
{
if(media == null) return new Attachment[0];
return media.Select(x =>
{
return new Attachment
if (media == null)
return new Attachment[0];
return media
.Select(x =>
{
type = "Document",
url = x.Url,
mediaType = x.MediaType
};
}).ToArray();
return new Attachment
{
type = "Document",
url = x.Url,
mediaType = x.MediaType
};
})
.ToArray();
}
}
}

View file

@ -1,162 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Newtonsoft.Json;
namespace BirdsiteLive.Domain
{
public interface ITheFedInfoService
{
Task<List<BslInstanceInfo>> GetBslInstanceListAsync();
}
public class TheFedInfoService : ITheFedInfoService
{
private readonly IHttpClientFactory _httpClientFactory;
#region Ctor
public TheFedInfoService(IHttpClientFactory httpClientFactory)
{
_httpClientFactory = httpClientFactory;
}
#endregion
public async Task<List<BslInstanceInfo>> GetBslInstanceListAsync()
{
var cancellationToken = CancellationToken.None;
var result = await CallGraphQlAsync<MyResponseData>(
new Uri("https://the-federation.info/graphql"),
HttpMethod.Get,
"query ($platform: String!) { nodes(platform: $platform) { host, version } }",
new
{
platform = "birdsitelive",
},
cancellationToken);
var convertedResults = ConvertResults(result);
return convertedResults;
}
private List<BslInstanceInfo> ConvertResults(GraphQLResponse<MyResponseData> qlData)
{
var results = new List<BslInstanceInfo>();
foreach (var instanceInfo in qlData.Data.Nodes)
{
try
{
var rawVersion = instanceInfo.Version.Split('+').First();
if (string.IsNullOrWhiteSpace(rawVersion)) continue;
var version = Version.Parse(rawVersion);
if(version <= new Version(0,1,0)) continue;
var instance = new BslInstanceInfo
{
Host = instanceInfo.Host,
Version = version
};
results.Add(instance);
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
return results;
}
private async Task<GraphQLResponse<TResponse>> CallGraphQlAsync<TResponse>(Uri endpoint, HttpMethod method, string query, object variables, CancellationToken cancellationToken)
{
var content = new StringContent(SerializeGraphQlCall(query, variables), Encoding.UTF8, "application/json");
var httpRequestMessage = new HttpRequestMessage
{
Method = method,
Content = content,
RequestUri = endpoint,
};
//add authorization headers if necessary here
httpRequestMessage.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
var httpClient = _httpClientFactory.CreateClient("BirdsiteLIVE");
using (var response = await httpClient.SendAsync(httpRequestMessage, cancellationToken))
{
//if (response.IsSuccessStatusCode)
if (response?.Content.Headers.ContentType?.MediaType == "application/json")
{
var responseString = await response.Content.ReadAsStringAsync(); //cancellationToken supported for .NET 5/6
return DeserializeGraphQlCall<TResponse>(responseString);
}
else
{
throw new ApplicationException($"Unable to contact '{endpoint}': {response.StatusCode} - {response.ReasonPhrase}");
}
}
}
private string SerializeGraphQlCall(string query, object variables)
{
var sb = new StringBuilder();
var textWriter = new StringWriter(sb);
var serializer = new JsonSerializer();
serializer.Serialize(textWriter, new
{
query = query,
variables = variables,
});
return sb.ToString();
}
private GraphQLResponse<TResponse> DeserializeGraphQlCall<TResponse>(string response)
{
var serializer = new JsonSerializer();
var stringReader = new StringReader(response);
var jsonReader = new JsonTextReader(stringReader);
var result = serializer.Deserialize<GraphQLResponse<TResponse>>(jsonReader);
return result;
}
private class GraphQLResponse<TResponse>
{
public List<GraphQLError> Errors { get; set; }
public TResponse Data { get; set; }
}
private class GraphQLError
{
public string Message { get; set; }
public List<GraphQLErrorLocation> Locations { get; set; }
public List<object> Path { get; set; } //either int or string
}
private class GraphQLErrorLocation
{
public int Line { get; set; }
public int Column { get; set; }
}
private class MyResponseData
{
public Node[] Nodes { get; set; }
}
private class Node
{
public string Host { get; set; }
public string Version { get; set; }
}
}
public class BslInstanceInfo
{
public string Host { get; set; }
public Version Version { get; set; }
}
}

View file

@ -1,6 +1,8 @@
using System;
using System;
using System.Text.RegularExpressions;
using BirdsiteLive.Domain.Repository;
using Org.BouncyCastle.Pkcs;
namespace BirdsiteLive.Domain.Tools
@ -25,4 +27,4 @@ namespace BirdsiteLive.Domain.Tools
return new Regex($@"^{data}$");
}
}
}
}

View file

@ -1,4 +1,4 @@
using System;
using System;
using System.Linq;
namespace BirdsiteLive.Domain.Tools
@ -7,17 +7,20 @@ namespace BirdsiteLive.Domain.Tools
{
public static string[] Parse(string entry)
{
if (string.IsNullOrWhiteSpace(entry)) return new string[0];
if (string.IsNullOrWhiteSpace(entry))
return new string[0];
var separationChar = '|';
if (entry.Contains(";")) separationChar = ';';
else if (entry.Contains(",")) separationChar = ',';
if (entry.Contains(";"))
separationChar = ';';
else if (entry.Contains(","))
separationChar = ',';
var splitEntries = entry
.Split(new[] {separationChar}, StringSplitOptions.RemoveEmptyEntries)
.Split(new[] { separationChar }, StringSplitOptions.RemoveEmptyEntries)
.Where(x => !string.IsNullOrWhiteSpace(x))
.Select(x => x.ToLowerInvariant().Trim());
return splitEntries.ToArray();
}
}
}
}

View file

@ -1,22 +0,0 @@
using System.Linq;
namespace BirdsiteLive.Domain.Tools
{
public class SigValidationResultExtractor
{
public static string GetUserName(SignatureValidationResult result)
{
return result.User.preferredUsername.ToLowerInvariant().Trim();
}
public static string GetHost(SignatureValidationResult result)
{
return result.User.url.Replace("https://", string.Empty).Split('/').First();
}
public static string GetSharedInbox(SignatureValidationResult result)
{
return result.User?.endpoints?.sharedInbox;
}
}
}

View file

@ -1,10 +1,12 @@
using System.Collections.Generic;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using BirdsiteLive.ActivityPub.Models;
using BirdsiteLive.Common.Regexes;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.Twitter;
using Microsoft.Extensions.Logging;
namespace BirdsiteLive.Domain.Tools
@ -27,7 +29,10 @@ namespace BirdsiteLive.Domain.Tools
}
#endregion
public (string content, Tag[] tags) Extract(string messageContent, bool extractMentions = true)
public (string content, Tag[] tags) Extract(
string messageContent,
bool extractMentions = true
)
{
var tags = new List<Tag>();
@ -64,8 +69,11 @@ namespace BirdsiteLive.Domain.Tools
secondPart = truncatedUrl.Substring(30);
}
messageContent = Regex.Replace(messageContent, Regex.Escape(m.ToString()),
$@"{m.Groups[1]}<a href=""{url}"" rel=""nofollow noopener noreferrer"" target=""_blank""><span class=""invisible"">{protocol}</span><span class=""ellipsis"">{firstPart}</span><span class=""invisible"">{secondPart}</span></a>");
messageContent = Regex.Replace(
messageContent,
Regex.Escape(m.ToString()),
$@"{m.Groups[1]}<a href=""{url}"" rel=""nofollow noopener noreferrer"" target=""_blank""><span class=""invisible"">{protocol}</span><span class=""ellipsis"">{firstPart}</span><span class=""invisible"">{secondPart}</span></a>"
);
}
// Extract Hashtags
@ -76,7 +84,11 @@ namespace BirdsiteLive.Domain.Tools
if (!HashtagRegexes.HashtagName.IsMatch(tag))
{
_logger.LogError("Parsing Hashtag failed: {Tag} on {Content}", tag, messageContent);
_logger.LogError(
"Parsing Hashtag failed: {Tag} on {Content}",
tag,
messageContent
);
continue;
}
@ -84,16 +96,21 @@ namespace BirdsiteLive.Domain.Tools
if (tags.All(x => x.href != url))
{
tags.Add(new Tag
{
name = $"#{tag}",
href = url,
type = "Hashtag"
});
tags.Add(
new Tag
{
name = $"#{tag}",
href = url,
type = "Hashtag"
}
);
}
messageContent = Regex.Replace(messageContent, Regex.Escape(m.Groups[0].ToString()),
$@"{m.Groups[1]}<a href=""{url}"" class=""mention hashtag"" rel=""tag"">#<span>{tag}</span></a>{m.Groups[3]}");
messageContent = Regex.Replace(
messageContent,
Regex.Escape(m.Groups[0].ToString()),
$@"{m.Groups[1]}<a href=""{url}"" class=""mention hashtag"" rel=""tag"">#<span>{tag}</span></a>{m.Groups[3]}"
);
}
// Extract Mentions
@ -106,7 +123,11 @@ namespace BirdsiteLive.Domain.Tools
if (!UserRegexes.TwitterAccount.IsMatch(mention))
{
_logger.LogError("Parsing Mention failed: {Mention} on {Content}", mention, messageContent);
_logger.LogError(
"Parsing Mention failed: {Mention} on {Content}",
mention,
messageContent
);
continue;
}
@ -115,26 +136,32 @@ namespace BirdsiteLive.Domain.Tools
if (tags.All(x => x.href != url))
{
tags.Add(new Tag
{
name = name,
href = url,
type = "Mention"
});
tags.Add(
new Tag
{
name = name,
href = url,
type = "Mention"
}
);
}
messageContent = Regex.Replace(messageContent, Regex.Escape(m.Groups[0].ToString()),
$@"{m.Groups[1]}<span class=""h-card""><a href=""https://{_instanceSettings.Domain}/@{mention}"" class=""u-url mention"">@<span>{mention}</span></a></span>{m.Groups[3]}");
messageContent = Regex.Replace(
messageContent,
Regex.Escape(m.Groups[0].ToString()),
$@"{m.Groups[1]}<span class=""h-card""><a href=""https://{_instanceSettings.Domain}/@{mention}"" class=""u-url mention"">@<span>{mention}</span></a></span>{m.Groups[3]}"
);
}
}
return (messageContent.Trim(), tags.ToArray());
}
private IEnumerable<Match> OrderByLength(MatchCollection matches)
{
var result = new List<Match>();
foreach (Match m in matches) result.Add(m);
foreach (Match m in matches)
result.Add(m);
result = result
.OrderBy(x => x.Length)
@ -145,4 +172,4 @@ namespace BirdsiteLive.Domain.Tools
return result;
}
}
}
}

View file

@ -1,10 +1,11 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using BirdsiteLive.ActivityPub;
using BirdsiteLive.ActivityPub.Converters;
using BirdsiteLive.ActivityPub.Models;
@ -12,13 +13,13 @@ using BirdsiteLive.Common.Regexes;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.Cryptography;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Domain.BusinessUseCases;
using BirdsiteLive.Domain.Repository;
using BirdsiteLive.Domain.Statistics;
using BirdsiteLive.Domain.Tools;
using BirdsiteLive.Twitter;
using BirdsiteLive.Twitter.Models;
using Tweetinvi.Core.Exceptions;
using Tweetinvi.Models;
@ -26,17 +27,31 @@ namespace BirdsiteLive.Domain
{
public interface IUserService
{
Actor GetUser(TwitterUser twitterUser, SyncTwitterUser dbTwitterUser);
Task<bool> FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders, ActivityFollow activity, string body);
Task<bool> UndoFollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders, ActivityUndoFollow activity, string body);
Actor GetUser(TwitterUser twitterUser);
Task<bool> FollowRequestedAsync(
string signature,
string method,
string path,
string queryString,
Dictionary<string, string> requestHeaders,
ActivityFollow activity,
string body
);
Task<bool> UndoFollowRequestedAsync(
string signature,
string method,
string path,
string queryString,
Dictionary<string, string> requestHeaders,
ActivityUndoFollow activity,
string body
);
Task<bool> SendRejectFollowAsync(ActivityFollow activity, string followerHost);
Task<bool> DeleteRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders, ActivityDelete activity, string body);
}
public class UserService : IUserService
{
private readonly IProcessDeleteUser _processDeleteUser;
private readonly IProcessFollowUser _processFollowUser;
private readonly IProcessUndoFollowUser _processUndoFollowUser;
@ -53,7 +68,18 @@ namespace BirdsiteLive.Domain
private readonly IFollowersDal _followerDal;
#region Ctor
public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler, ITwitterUserService twitterUserService, IModerationRepository moderationRepository, IFollowersDal followerDal, IProcessDeleteUser processDeleteUser)
public UserService(
InstanceSettings instanceSettings,
ICryptoService cryptoService,
IActivityPubService activityPubService,
IProcessFollowUser processFollowUser,
IProcessUndoFollowUser processUndoFollowUser,
IStatusExtractor statusExtractor,
IExtractionStatisticsHandler statisticsHandler,
ITwitterUserService twitterUserService,
IModerationRepository moderationRepository,
IFollowersDal followerDal
)
{
_instanceSettings = instanceSettings;
_cryptoService = cryptoService;
@ -65,11 +91,10 @@ namespace BirdsiteLive.Domain
_twitterUserService = twitterUserService;
_moderationRepository = moderationRepository;
_followerDal = followerDal;
_processDeleteUser = processDeleteUser;
}
#endregion
public Actor GetUser(TwitterUser twitterUser, SyncTwitterUser dbTwitterUser)
public Actor GetUser(TwitterUser twitterUser)
{
var actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, twitterUser.Acct);
var acct = twitterUser.Acct.ToLowerInvariant();
@ -78,94 +103,80 @@ namespace BirdsiteLive.Domain
var description = twitterUser.Description;
if (!string.IsNullOrWhiteSpace(description))
{
var extracted = _statusExtractor.Extract(description, _instanceSettings.ResolveMentionsInProfiles);
var extracted = _statusExtractor.Extract(
description,
_instanceSettings.ResolveMentionsInProfiles
);
description = extracted.content;
_statisticsHandler.ExtractedDescription(extracted.tags.Count(x => x.type == "Mention"));
_statisticsHandler.ExtractedDescription(
extracted.tags.Count(x => x.type == "Mention")
);
}
var attachments = new List<UserAttachment>();
attachments.Add(new UserAttachment
{
type = "PropertyValue",
name = _instanceSettings.TwitterDomainLabel != "" ? _instanceSettings.TwitterDomainLabel : _instanceSettings.TwitterDomain,
value = $"<a href=\"https://{_instanceSettings.TwitterDomain}/{acct}\" rel=\"me nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">{_instanceSettings.TwitterDomain}/{acct}</span></a>"
});
if(_instanceSettings.TwitterDomain != "twitter.com")
{
attachments.Add(new UserAttachment
attachments.Add(
new UserAttachment
{
type = "PropertyValue",
name = "Twitter",
value = $"twitter.com/{acct}"
});
name =
_instanceSettings.TwitterDomainLabel != ""
? _instanceSettings.TwitterDomainLabel
: _instanceSettings.TwitterDomain,
value =
$"<a href=\"https://{_instanceSettings.TwitterDomain}/{acct}\" rel=\"me nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">{_instanceSettings.TwitterDomain}/{acct}</span></a>"
}
);
if (_instanceSettings.TwitterDomain != "twitter.com")
{
attachments.Add(
new UserAttachment
{
type = "PropertyValue",
name = "Twitter",
value = $"twitter.com/{acct}"
}
);
}
if (_instanceSettings.ShowAboutInstanceOnProfiles)
{
attachments.Add(new UserAttachment
{
type = "PropertyValue",
name = $"About {_instanceSettings.Name}",
value = $"<a href=\"https://{_instanceSettings.Domain}/About\" rel=\"nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">{_instanceSettings.Domain}/About</span></a>"
});
attachments.Add(
new UserAttachment
{
type = "PropertyValue",
name = $"About {_instanceSettings.Name}",
value =
$"<a href=\"https://{_instanceSettings.Domain}/About\" rel=\"nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">{_instanceSettings.Domain}/About</span></a>"
}
);
}
var user = new Actor
{
id = actorUrl,
type = "Service",
type = "Service",
followers = $"{actorUrl}/followers",
preferredUsername = acct,
name = twitterUser.Name,
inbox = $"{actorUrl}/inbox",
summary = "[UNOFFICIAL MIRROR: This is a view of Twitter using ActivityPub]<br/><br/>" + description,
summary = description,
url = actorUrl,
manuallyApprovesFollowers = twitterUser.Protected,
discoverable = false,
publicKey = new PublicKey()
{
id = $"{actorUrl}#main-key",
owner = actorUrl,
publicKeyPem = _cryptoService.GetUserPem(acct)
},
icon = new Image
{
mediaType = "image/jpeg",
url = twitterUser.ProfileImageUrl
},
image = new Image
{
mediaType = "image/jpeg",
url = twitterUser.ProfileBannerURL
},
attachment = new []
{
new UserAttachment
{
type = "PropertyValue",
name = "Official Account",
value = $"<a href=\"https://twitter.com/{acct}\" rel=\"me nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">twitter.com/{acct}</span></a>"
},
new UserAttachment
{
type = "PropertyValue",
name = "Disclaimer",
value = "This is an automatically created and managed mirror profile from Twitter. While it reflects exactly the content of the original account, it doesn't provide support for interactions and replies. It is an equivalent view from other 3rd party Twitter client apps and uses the same technical means to provide it."
},
new UserAttachment
{
type = "PropertyValue",
name = "Take control of this account",
value = $"<a href=\"https://{_instanceSettings.Domain}/migration/move/{acct}\" rel=\"me nofollow noopener noreferrer\" target=\"_blank\">MANAGE</a>"
}
},
icon = new Image { mediaType = "image/jpeg", url = twitterUser.ProfileImageUrl },
image = new Image { mediaType = "image/jpeg", url = twitterUser.ProfileBannerURL },
attachment = attachments.ToArray(),
endpoints = new EndPoints
{
sharedInbox = $"https://{_instanceSettings.Domain}/inbox"
},
movedTo = dbTwitterUser?.MovedTo
}
};
if (twitterUser.Verified)
@ -191,49 +202,96 @@ namespace BirdsiteLive.Domain
return user;
}
public async Task<bool> FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders, ActivityFollow activity, string body)
public async Task<bool> FollowRequestedAsync(
string signature,
string method,
string path,
string queryString,
Dictionary<string, string> requestHeaders,
ActivityFollow activity,
string body
)
{
// Validate
var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body);
if (!sigValidation.SignatureIsValidated) return false;
var sigValidation = await ValidateSignature(
activity.actor,
signature,
method,
path,
queryString,
requestHeaders,
body
);
if (!sigValidation.SignatureIsValidated)
return false;
// Prepare data
var followerUserName = SigValidationResultExtractor.GetUserName(sigValidation);
var followerHost = SigValidationResultExtractor.GetHost(sigValidation);
var followerUserName = sigValidation.User.preferredUsername.ToLowerInvariant().Trim();
var followerHost = sigValidation.User.url
.Replace("https://", string.Empty)
.Split('/')
.First();
var followerInbox = sigValidation.User.inbox;
var followerSharedInbox = SigValidationResultExtractor.GetSharedInbox(sigValidation);
var twitterUser = activity.apObject.Split('/').Last().Replace("@", string.Empty).ToLowerInvariant().Trim();
var followerSharedInbox = sigValidation.User?.endpoints?.sharedInbox;
var twitterUser = activity.apObject
.Split('/')
.Last()
.Replace("@", string.Empty)
.ToLowerInvariant()
.Trim();
// Make sure to only keep routes
followerInbox = OnlyKeepRoute(followerInbox, followerHost);
followerSharedInbox = OnlyKeepRoute(followerSharedInbox, followerHost);
// Validate Moderation status
var followerModPolicy = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.Follower);
var followerModPolicy = _moderationRepository.GetModerationType(
ModerationEntityTypeEnum.Follower
);
if (followerModPolicy != ModerationTypeEnum.None)
{
var followerStatus = _moderationRepository.CheckStatus(ModerationEntityTypeEnum.Follower, $"@{followerUserName}@{followerHost}");
if(followerModPolicy == ModerationTypeEnum.WhiteListing && followerStatus != ModeratedTypeEnum.WhiteListed ||
followerModPolicy == ModerationTypeEnum.BlackListing && followerStatus == ModeratedTypeEnum.BlackListed)
var followerStatus = _moderationRepository.CheckStatus(
ModerationEntityTypeEnum.Follower,
$"@{followerUserName}@{followerHost}"
);
if (
followerModPolicy == ModerationTypeEnum.WhiteListing
&& followerStatus != ModeratedTypeEnum.WhiteListed
|| followerModPolicy == ModerationTypeEnum.BlackListing
&& followerStatus == ModeratedTypeEnum.BlackListed
)
return await SendRejectFollowAsync(activity, followerHost);
}
// Validate TwitterAccount status
var twitterAccountModPolicy = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.TwitterAccount);
var twitterAccountModPolicy = _moderationRepository.GetModerationType(
ModerationEntityTypeEnum.TwitterAccount
);
if (twitterAccountModPolicy != ModerationTypeEnum.None)
{
var twitterUserStatus = _moderationRepository.CheckStatus(ModerationEntityTypeEnum.TwitterAccount, twitterUser);
if (twitterAccountModPolicy == ModerationTypeEnum.WhiteListing && twitterUserStatus != ModeratedTypeEnum.WhiteListed ||
twitterAccountModPolicy == ModerationTypeEnum.BlackListing && twitterUserStatus == ModeratedTypeEnum.BlackListed)
var twitterUserStatus = _moderationRepository.CheckStatus(
ModerationEntityTypeEnum.TwitterAccount,
twitterUser
);
if (
twitterAccountModPolicy == ModerationTypeEnum.WhiteListing
&& twitterUserStatus != ModeratedTypeEnum.WhiteListed
|| twitterAccountModPolicy == ModerationTypeEnum.BlackListing
&& twitterUserStatus == ModeratedTypeEnum.BlackListed
)
return await SendRejectFollowAsync(activity, followerHost);
}
// Validate follower count < MaxFollowsPerUser
if (_instanceSettings.MaxFollowsPerUser > 0) {
if (_instanceSettings.MaxFollowsPerUser > 0)
{
var follower = await _followerDal.GetFollowerAsync(followerUserName, followerHost);
if (follower != null && follower.Followings.Count + 1 > _instanceSettings.MaxFollowsPerUser)
if (
follower != null
&& follower.Followings.Count + 1 > _instanceSettings.MaxFollowsPerUser
)
{
return await SendRejectFollowAsync(activity, followerHost);
}
@ -244,7 +302,14 @@ namespace BirdsiteLive.Domain
if (!user.Protected)
{
// Execute
await _processFollowUser.ExecuteAsync(followerUserName, followerHost, twitterUser, followerInbox, followerSharedInbox, activity.actor);
await _processFollowUser.ExecuteAsync(
followerUserName,
followerHost,
twitterUser,
followerInbox,
followerSharedInbox,
activity.actor
);
return await SendAcceptFollowAsync(activity, followerHost);
}
@ -253,7 +318,7 @@ namespace BirdsiteLive.Domain
return await SendRejectFollowAsync(activity, followerHost);
}
}
private async Task<bool> SendAcceptFollowAsync(ActivityFollow activity, string followerHost)
{
var acceptFollow = new ActivityAcceptFollow()
@ -270,9 +335,12 @@ namespace BirdsiteLive.Domain
apObject = activity.apObject
}
};
var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject);
return result == HttpStatusCode.Accepted ||
result == HttpStatusCode.OK; //TODO: revamp this for better error handling
var result = await _activityPubService.PostDataAsync(
acceptFollow,
followerHost,
activity.apObject
);
return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; //TODO: revamp this for better error handling
}
public async Task<bool> SendRejectFollowAsync(ActivityFollow activity, string followerHost)
@ -291,14 +359,17 @@ namespace BirdsiteLive.Domain
apObject = activity.apObject
}
};
var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject);
return result == HttpStatusCode.Accepted ||
result == HttpStatusCode.OK; //TODO: revamp this for better error handling
var result = await _activityPubService.PostDataAsync(
acceptFollow,
followerHost,
activity.apObject
);
return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; //TODO: revamp this for better error handling
}
private string OnlyKeepRoute(string inbox, string host)
{
if (string.IsNullOrWhiteSpace(inbox))
if (string.IsNullOrWhiteSpace(inbox))
return null;
if (inbox.Contains(host))
@ -307,18 +378,40 @@ namespace BirdsiteLive.Domain
return inbox;
}
public async Task<bool> UndoFollowRequestedAsync(string signature, string method, string path, string queryString,
Dictionary<string, string> requestHeaders, ActivityUndoFollow activity, string body)
public async Task<bool> UndoFollowRequestedAsync(
string signature,
string method,
string path,
string queryString,
Dictionary<string, string> requestHeaders,
ActivityUndoFollow activity,
string body
)
{
// Validate
var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body);
if (!sigValidation.SignatureIsValidated) return false;
var sigValidation = await ValidateSignature(
activity.actor,
signature,
method,
path,
queryString,
requestHeaders,
body
);
if (!sigValidation.SignatureIsValidated)
return false;
// Save Follow in DB
var followerUserName = sigValidation.User.preferredUsername.ToLowerInvariant();
var followerHost = sigValidation.User.url.Replace("https://", string.Empty).Split('/').First();
var followerHost = sigValidation.User.url
.Replace("https://", string.Empty)
.Split('/')
.First();
//var followerInbox = sigValidation.User.inbox;
var twitterUser = activity.apObject.apObject.Split('/').Last().Replace("@", string.Empty);
var twitterUser = activity.apObject.apObject
.Split('/')
.Last()
.Replace("@", string.Empty);
await _processUndoFollowUser.ExecuteAsync(followerUserName, followerHost, twitterUser);
// Send Accept Activity
@ -336,40 +429,40 @@ namespace BirdsiteLive.Domain
apObject = activity.apObject
}
};
var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject.apObject);
var result = await _activityPubService.PostDataAsync(
acceptFollow,
followerHost,
activity.apObject.apObject
);
return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; //TODO: revamp this for better error handling
}
public async Task<bool> DeleteRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders,
ActivityDelete activity, string body)
{
// Validate
var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body);
if (!sigValidation.SignatureIsValidated) return false;
// Remove user and followings
var followerUserName = SigValidationResultExtractor.GetUserName(sigValidation);
var followerHost = SigValidationResultExtractor.GetHost(sigValidation);
await _processDeleteUser.ExecuteAsync(followerUserName, followerHost);
return true;
}
private async Task<SignatureValidationResult> ValidateSignature(string actor, string rawSig, string method, string path, string queryString, Dictionary<string, string> requestHeaders, string body)
private async Task<SignatureValidationResult> ValidateSignature(
string actor,
string rawSig,
string method,
string path,
string queryString,
Dictionary<string, string> requestHeaders,
string body
)
{
//Check Date Validity
var date = requestHeaders["date"];
var d = DateTime.Parse(date).ToUniversalTime();
var now = DateTime.UtcNow;
var delta = Math.Abs((d - now).TotalSeconds);
if (delta > 30) return new SignatureValidationResult { SignatureIsValidated = false };
if (delta > 30)
return new SignatureValidationResult { SignatureIsValidated = false };
//Check Digest
var digest = requestHeaders["digest"];
var digestHash = digest.Split(new [] {"SHA-256="},StringSplitOptions.RemoveEmptyEntries).LastOrDefault();
var digestHash = digest
.Split(new[] { "SHA-256=" }, StringSplitOptions.RemoveEmptyEntries)
.LastOrDefault();
var calculatedDigestHash = _cryptoService.ComputeSha256Hash(body);
if (digestHash != calculatedDigestHash) return new SignatureValidationResult { SignatureIsValidated = false };
if (digestHash != calculatedDigestHash)
return new SignatureValidationResult { SignatureIsValidated = false };
//Check Signature
var signatures = rawSig.Split(',');
@ -389,15 +482,19 @@ namespace BirdsiteLive.Domain
var remoteUser = await _activityPubService.GetUser(actor);
// Prepare Key data
var toDecode = remoteUser.publicKey.publicKeyPem.Trim().Remove(0, remoteUser.publicKey.publicKeyPem.IndexOf('\n'));
var toDecode = remoteUser.publicKey.publicKeyPem
.Trim()
.Remove(0, remoteUser.publicKey.publicKeyPem.IndexOf('\n'));
toDecode = toDecode.Remove(toDecode.LastIndexOf('\n')).Replace("\n", "");
var signKey = ASN1.ToRSA(Convert.FromBase64String(toDecode));
var toSign = new StringBuilder();
foreach (var headerKey in headers.Split(' '))
{
if (headerKey == "(request-target)") toSign.Append($"(request-target): {method.ToLower()} {path}{queryString}\n");
else toSign.Append($"{headerKey}: {string.Join(", ", requestHeaders[headerKey])}\n");
if (headerKey == "(request-target)")
toSign.Append($"(request-target): {method.ToLower()} {path}{queryString}\n");
else
toSign.Append($"{headerKey}: {string.Join(", ", requestHeaders[headerKey])}\n");
}
toSign.Remove(toSign.Length - 1, 1);
@ -408,7 +505,12 @@ namespace BirdsiteLive.Domain
key.ImportParameters(rsaKeyInfo);
// Trust and Verify
var result = signKey.VerifyData(Encoding.UTF8.GetBytes(toSign.ToString()), sig, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
var result = signKey.VerifyData(
Encoding.UTF8.GetBytes(toSign.ToString()),
sig,
HashAlgorithmName.SHA256,
RSASignaturePadding.Pkcs1
);
return new SignatureValidationResult()
{
@ -418,7 +520,7 @@ namespace BirdsiteLive.Domain
}
}
public class SignatureValidationResult
public class SignatureValidationResult
{
public bool SignatureIsValidated { get; set; }
public Actor User { get; set; }

View file

@ -1,5 +1,6 @@
using System;
using System;
using System.Threading.Tasks;
using BirdsiteLive.ActivityPub;
using BirdsiteLive.ActivityPub.Converters;
using BirdsiteLive.Common.Settings;
@ -21,7 +22,11 @@ namespace BirdsiteLive.Moderation.Actions
private readonly InstanceSettings _instanceSettings;
#region Ctor
public RejectAllFollowingsAction(ITwitterUserDal twitterUserDal, IUserService userService, InstanceSettings instanceSettings)
public RejectAllFollowingsAction(
ITwitterUserDal twitterUserDal,
IUserService userService,
InstanceSettings instanceSettings
)
{
_twitterUserDal = twitterUserDal;
_userService = userService;
@ -49,4 +54,4 @@ namespace BirdsiteLive.Moderation.Actions
}
}
}
}
}

View file

@ -1,5 +1,6 @@
using System;
using System;
using System.Threading.Tasks;
using BirdsiteLive.ActivityPub;
using BirdsiteLive.ActivityPub.Converters;
using BirdsiteLive.Common.Settings;
@ -17,7 +18,7 @@ namespace BirdsiteLive.Moderation.Actions
{
private readonly IUserService _userService;
private readonly InstanceSettings _instanceSettings;
#region Ctor
public RejectFollowingAction(IUserService userService, InstanceSettings instanceSettings)
{
@ -41,4 +42,4 @@ namespace BirdsiteLive.Moderation.Actions
catch (Exception) { }
}
}
}
}

View file

@ -1,6 +1,13 @@
using System.Threading.Tasks;
using System;
using System.Linq;
using System.Threading.Tasks;
using BirdsiteLive.ActivityPub;
using BirdsiteLive.ActivityPub.Converters;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Domain.BusinessUseCases;
using BirdsiteLive.Domain;
namespace BirdsiteLive.Moderation.Actions
{
@ -11,14 +18,20 @@ namespace BirdsiteLive.Moderation.Actions
public class RemoveFollowerAction : IRemoveFollowerAction
{
private readonly IFollowersDal _followersDal;
private readonly ITwitterUserDal _twitterUserDal;
private readonly IRejectAllFollowingsAction _rejectAllFollowingsAction;
private readonly IProcessDeleteUser _processDeleteUser;
#region Ctor
public RemoveFollowerAction(IRejectAllFollowingsAction rejectAllFollowingsAction, IProcessDeleteUser processDeleteUser)
public RemoveFollowerAction(
IFollowersDal followersDal,
ITwitterUserDal twitterUserDal,
IRejectAllFollowingsAction rejectAllFollowingsAction
)
{
_followersDal = followersDal;
_twitterUserDal = twitterUserDal;
_rejectAllFollowingsAction = rejectAllFollowingsAction;
_processDeleteUser = processDeleteUser;
}
#endregion
@ -28,7 +41,16 @@ namespace BirdsiteLive.Moderation.Actions
await _rejectAllFollowingsAction.ProcessAsync(follower);
// Remove twitter users if no more followers
await _processDeleteUser.ExecuteAsync(follower);
var followings = follower.Followings;
foreach (var following in followings)
{
var followers = await _followersDal.GetFollowersAsync(following);
if (followers.Length == 1 && followers.First().Id == follower.Id)
await _twitterUserDal.DeleteTwitterUserAsync(following);
}
// Remove follower from DB
await _followersDal.DeleteFollowerAsync(follower.Id);
}
}
}
}

View file

@ -1,5 +1,6 @@
using System.Linq;
using System.Linq;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
@ -17,7 +18,11 @@ namespace BirdsiteLive.Moderation.Actions
private readonly IRejectFollowingAction _rejectFollowingAction;
#region Ctor
public RemoveTwitterAccountAction(IFollowersDal followersDal, ITwitterUserDal twitterUserDal, IRejectFollowingAction rejectFollowingAction)
public RemoveTwitterAccountAction(
IFollowersDal followersDal,
ITwitterUserDal twitterUserDal,
IRejectFollowingAction rejectFollowingAction
)
{
_followersDal = followersDal;
_twitterUserDal = twitterUserDal;
@ -27,12 +32,12 @@ namespace BirdsiteLive.Moderation.Actions
public async Task ProcessAsync(SyncTwitterUser twitterUser)
{
// Check Followers
// Check Followers
var twitterUserId = twitterUser.Id;
var followers = await _followersDal.GetFollowersAsync(twitterUserId);
// Remove all Followers
foreach (var follower in followers)
foreach (var follower in followers)
{
// Perform undo following to user instance
await _rejectFollowingAction.ProcessAsync(follower, twitterUser);
@ -54,4 +59,4 @@ namespace BirdsiteLive.Moderation.Actions
await _twitterUserDal.DeleteTwitterUserAsync(twitterUserId);
}
}
}
}

View file

@ -1,7 +1,9 @@
using System;
using System;
using System.Threading.Tasks;
using BirdsiteLive.Domain.Repository;
using BirdsiteLive.Moderation.Processors;
using Microsoft.Extensions.Logging;
namespace BirdsiteLive.Moderation
@ -20,7 +22,12 @@ namespace BirdsiteLive.Moderation
private readonly ILogger<ModerationPipeline> _logger;
#region Ctor
public ModerationPipeline(IModerationRepository moderationRepository, IFollowerModerationProcessor followerModerationProcessor, ITwitterAccountModerationProcessor twitterAccountModerationProcessor, ILogger<ModerationPipeline> logger)
public ModerationPipeline(
IModerationRepository moderationRepository,
IFollowerModerationProcessor followerModerationProcessor,
ITwitterAccountModerationProcessor twitterAccountModerationProcessor,
ILogger<ModerationPipeline> logger
)
{
_moderationRepository = moderationRepository;
_followerModerationProcessor = followerModerationProcessor;
@ -44,16 +51,22 @@ namespace BirdsiteLive.Moderation
private async Task CheckFollowerModerationPolicyAsync()
{
var followerPolicy = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.Follower);
if (followerPolicy == ModerationTypeEnum.None) return;
var followerPolicy = _moderationRepository.GetModerationType(
ModerationEntityTypeEnum.Follower
);
if (followerPolicy == ModerationTypeEnum.None)
return;
await _followerModerationProcessor.ProcessAsync(followerPolicy);
}
private async Task CheckTwitterAccountModerationPolicyAsync()
{
var twitterAccountPolicy = _moderationRepository.GetModerationType(ModerationEntityTypeEnum.TwitterAccount);
if (twitterAccountPolicy == ModerationTypeEnum.None) return;
var twitterAccountPolicy = _moderationRepository.GetModerationType(
ModerationEntityTypeEnum.TwitterAccount
);
if (twitterAccountPolicy == ModerationTypeEnum.None)
return;
await _twitterAccountModerationProcessor.ProcessAsync(twitterAccountPolicy);
}

View file

@ -1,5 +1,6 @@
using System;
using System;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.Domain.Repository;
using BirdsiteLive.Moderation.Actions;
@ -18,7 +19,11 @@ namespace BirdsiteLive.Moderation.Processors
private readonly IRemoveFollowerAction _removeFollowerAction;
#region Ctor
public FollowerModerationProcessor(IFollowersDal followersDal, IModerationRepository moderationRepository, IRemoveFollowerAction removeFollowerAction)
public FollowerModerationProcessor(
IFollowersDal followersDal,
IModerationRepository moderationRepository,
IRemoveFollowerAction removeFollowerAction
)
{
_followersDal = followersDal;
_moderationRepository = moderationRepository;
@ -28,17 +33,26 @@ namespace BirdsiteLive.Moderation.Processors
public async Task ProcessAsync(ModerationTypeEnum type)
{
if (type == ModerationTypeEnum.None) return;
if (type == ModerationTypeEnum.None)
return;
var followers = await _followersDal.GetAllFollowersAsync();
foreach (var follower in followers)
{
var followerHandle = $"@{follower.Acct.Trim()}@{follower.Host.Trim()}".ToLowerInvariant();
var status = _moderationRepository.CheckStatus(ModerationEntityTypeEnum.Follower, followerHandle);
var followerHandle =
$"@{follower.Acct.Trim()}@{follower.Host.Trim()}".ToLowerInvariant();
var status = _moderationRepository.CheckStatus(
ModerationEntityTypeEnum.Follower,
followerHandle
);
if (type == ModerationTypeEnum.WhiteListing && status != ModeratedTypeEnum.WhiteListed ||
type == ModerationTypeEnum.BlackListing && status == ModeratedTypeEnum.BlackListed)
if (
type == ModerationTypeEnum.WhiteListing
&& status != ModeratedTypeEnum.WhiteListed
|| type == ModerationTypeEnum.BlackListing
&& status == ModeratedTypeEnum.BlackListed
)
{
Console.WriteLine($"Remove {followerHandle}");
await _removeFollowerAction.ProcessAsync(follower);
@ -46,4 +60,4 @@ namespace BirdsiteLive.Moderation.Processors
}
}
}
}
}

View file

@ -1,4 +1,5 @@
using System.Threading.Tasks;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.Domain.Repository;
using BirdsiteLive.Moderation.Actions;
@ -15,9 +16,13 @@ namespace BirdsiteLive.Moderation.Processors
private readonly ITwitterUserDal _twitterUserDal;
private readonly IModerationRepository _moderationRepository;
private readonly IRemoveTwitterAccountAction _removeTwitterAccountAction;
#region Ctor
public TwitterAccountModerationProcessor(ITwitterUserDal twitterUserDal, IModerationRepository moderationRepository, IRemoveTwitterAccountAction removeTwitterAccountAction)
public TwitterAccountModerationProcessor(
ITwitterUserDal twitterUserDal,
IModerationRepository moderationRepository,
IRemoveTwitterAccountAction removeTwitterAccountAction
)
{
_twitterUserDal = twitterUserDal;
_moderationRepository = moderationRepository;
@ -27,19 +32,27 @@ namespace BirdsiteLive.Moderation.Processors
public async Task ProcessAsync(ModerationTypeEnum type)
{
if (type == ModerationTypeEnum.None) return;
var twitterUsers = await _twitterUserDal.GetAllTwitterUsersAsync(false);
if (type == ModerationTypeEnum.None)
return;
var twitterUsers = await _twitterUserDal.GetAllTwitterUsersAsync();
foreach (var user in twitterUsers)
{
var userHandle = user.Acct.ToLowerInvariant().Trim();
var status = _moderationRepository.CheckStatus(ModerationEntityTypeEnum.TwitterAccount, userHandle);
var status = _moderationRepository.CheckStatus(
ModerationEntityTypeEnum.TwitterAccount,
userHandle
);
if (type == ModerationTypeEnum.WhiteListing && status != ModeratedTypeEnum.WhiteListed ||
type == ModerationTypeEnum.BlackListing && status == ModeratedTypeEnum.BlackListed)
if (
type == ModerationTypeEnum.WhiteListing
&& status != ModeratedTypeEnum.WhiteListed
|| type == ModerationTypeEnum.BlackListing
&& status == ModeratedTypeEnum.BlackListed
)
await _removeTwitterAccountAction.ProcessAsync(user);
}
}
}
}
}

View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>

View file

@ -1,5 +1,6 @@
using System.Threading;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Pipeline.Models;
@ -7,6 +8,9 @@ namespace BirdsiteLive.Pipeline.Contracts
{
public interface IRefreshTwitterUserStatusProcessor
{
Task<UserWithDataToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct);
Task<UserWithDataToSync[]> ProcessAsync(
SyncTwitterUser[] syncTwitterUsers,
CancellationToken ct
);
}
}
}

View file

@ -1,13 +1,17 @@
using System.Collections.Generic;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.Pipeline.Models;
namespace BirdsiteLive.Pipeline.Contracts
{
public interface IRetrieveFollowersProcessor
{
Task<IEnumerable<UserWithDataToSync>> ProcessAsync(UserWithDataToSync[] userWithTweetsToSyncs, CancellationToken ct);
Task<IEnumerable<UserWithDataToSync>> ProcessAsync(
UserWithDataToSync[] userWithTweetsToSyncs,
CancellationToken ct
);
//IAsyncEnumerable<UserWithTweetsToSync> ProcessAsync(UserWithTweetsToSync[] userWithTweetsToSyncs, CancellationToken ct);
}
}
}

View file

@ -1,5 +1,6 @@
using System.Threading;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Pipeline.Models;
@ -7,6 +8,9 @@ namespace BirdsiteLive.Pipeline.Contracts
{
public interface IRetrieveTweetsProcessor
{
Task<UserWithDataToSync[]> ProcessAsync(UserWithDataToSync[] syncTwitterUsers, CancellationToken ct);
Task<UserWithDataToSync[]> ProcessAsync(
UserWithDataToSync[] syncTwitterUsers,
CancellationToken ct
);
}
}
}

View file

@ -1,12 +1,16 @@
using System.Threading;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using BirdsiteLive.DAL.Models;
namespace BirdsiteLive.Pipeline.Contracts
{
public interface IRetrieveTwitterUsersProcessor
{
Task GetTwitterUsersAsync(BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock, CancellationToken ct);
Task GetTwitterUsersAsync(
BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock,
CancellationToken ct
);
}
}
}

View file

@ -1,5 +1,6 @@
using System.Threading;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.Pipeline.Models;
namespace BirdsiteLive.Pipeline.Contracts
@ -8,4 +9,4 @@ namespace BirdsiteLive.Pipeline.Contracts
{
Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct);
}
}
}

View file

@ -1,11 +1,15 @@
using System.Threading;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.Pipeline.Models;
namespace BirdsiteLive.Pipeline.Contracts
{
public interface ISendTweetsToFollowersProcessor
{
Task<UserWithDataToSync> ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct);
Task<UserWithDataToSync> ProcessAsync(
UserWithDataToSync userWithTweetsToSync,
CancellationToken ct
);
}
}
}

View file

@ -1,5 +1,6 @@
using BirdsiteLive.DAL.Models;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Twitter.Models;
using Tweetinvi.Models;
namespace BirdsiteLive.Pipeline.Models
@ -10,4 +11,4 @@ namespace BirdsiteLive.Pipeline.Models
public ExtractedTweet[] Tweets { get; set; }
public Follower[] Followers { get; set; }
}
}
}

View file

@ -1,7 +1,8 @@
using System;
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
@ -9,7 +10,6 @@ using BirdsiteLive.Moderation.Actions;
using BirdsiteLive.Pipeline.Contracts;
using BirdsiteLive.Pipeline.Models;
using BirdsiteLive.Twitter;
using BirdsiteLive.Twitter.Models;
namespace BirdsiteLive.Pipeline.Processors
{
@ -21,7 +21,12 @@ namespace BirdsiteLive.Pipeline.Processors
private readonly InstanceSettings _instanceSettings;
#region Ctor
public RefreshTwitterUserStatusProcessor(ICachedTwitterUserService twitterUserService, ITwitterUserDal twitterUserDal, IRemoveTwitterAccountAction removeTwitterAccountAction, InstanceSettings instanceSettings)
public RefreshTwitterUserStatusProcessor(
ICachedTwitterUserService twitterUserService,
ITwitterUserDal twitterUserDal,
IRemoveTwitterAccountAction removeTwitterAccountAction,
InstanceSettings instanceSettings
)
{
_twitterUserService = twitterUserService;
_twitterUserDal = twitterUserDal;
@ -30,67 +35,32 @@ namespace BirdsiteLive.Pipeline.Processors
}
#endregion
public async Task<UserWithDataToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct)
public async Task<UserWithDataToSync[]> ProcessAsync(
SyncTwitterUser[] syncTwitterUsers,
CancellationToken ct
)
{
var usersWtData = new List<UserWithDataToSync>();
foreach (var user in syncTwitterUsers)
{
TwitterUser userView = null;
try
var userView = _twitterUserService.GetUser(user.Acct);
if (userView == null)
{
userView = _twitterUserService.GetUser(user.Acct);
await AnalyseFailingUserAsync(user);
}
catch (UserNotFoundException)
else if (!userView.Protected)
{
await ProcessNotFoundUserAsync(user);
continue;
user.FetchingErrorCount = 0;
var userWtData = new UserWithDataToSync { User = user };
usersWtData.Add(userWtData);
}
catch (UserHasBeenSuspendedException)
{
await ProcessNotFoundUserAsync(user);
continue;
}
catch (RateLimitExceededException)
{
await ProcessRateLimitExceededAsync(user);
continue;
}
catch (Exception)
{
// ignored
}
if (userView == null || userView.Protected)
{
await ProcessFailingUserAsync(user);
continue;
}
user.FetchingErrorCount = 0;
var userWtData = new UserWithDataToSync
{
User = user
};
usersWtData.Add(userWtData);
}
return usersWtData.ToArray();
}
private async Task ProcessRateLimitExceededAsync(SyncTwitterUser user)
{
var dbUser = await _twitterUserDal.GetTwitterUserAsync(user.Acct);
dbUser.LastSync = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(dbUser);
}
private async Task ProcessNotFoundUserAsync(SyncTwitterUser user)
{
await _removeTwitterAccountAction.ProcessAsync(user);
}
private async Task ProcessFailingUserAsync(SyncTwitterUser user)
private async Task AnalyseFailingUserAsync(SyncTwitterUser user)
{
var dbUser = await _twitterUserDal.GetTwitterUserAsync(user.Acct);
dbUser.FetchingErrorCount++;
@ -104,6 +74,9 @@ namespace BirdsiteLive.Pipeline.Processors
{
await _twitterUserDal.UpdateTwitterUserAsync(dbUser);
}
// Purge
_twitterUserService.PurgeUser(user.Acct);
}
}
}
}

View file

@ -1,6 +1,7 @@
using System.Collections.Generic;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.Pipeline.Contracts;
using BirdsiteLive.Pipeline.Models;
@ -18,7 +19,10 @@ namespace BirdsiteLive.Pipeline.Processors
}
#endregion
public async Task<IEnumerable<UserWithDataToSync>> ProcessAsync(UserWithDataToSync[] userWithTweetsToSyncs, CancellationToken ct)
public async Task<IEnumerable<UserWithDataToSync>> ProcessAsync(
UserWithDataToSync[] userWithTweetsToSyncs,
CancellationToken ct
)
{
//TODO multithread this
foreach (var user in userWithTweetsToSyncs)
@ -30,4 +34,4 @@ namespace BirdsiteLive.Pipeline.Processors
return userWithTweetsToSyncs;
}
}
}
}

View file

@ -1,15 +1,18 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Pipeline.Contracts;
using BirdsiteLive.Pipeline.Models;
using BirdsiteLive.Twitter;
using BirdsiteLive.Twitter.Models;
using Microsoft.Extensions.Logging;
using Tweetinvi.Models;
namespace BirdsiteLive.Pipeline.Processors
@ -22,7 +25,12 @@ namespace BirdsiteLive.Pipeline.Processors
private readonly ILogger<RetrieveTweetsProcessor> _logger;
#region Ctor
public RetrieveTweetsProcessor(ITwitterTweetsService twitterTweetsService, ITwitterUserDal twitterUserDal, ICachedTwitterUserService twitterUserService, ILogger<RetrieveTweetsProcessor> logger)
public RetrieveTweetsProcessor(
ITwitterTweetsService twitterTweetsService,
ITwitterUserDal twitterUserDal,
ICachedTwitterUserService twitterUserService,
ILogger<RetrieveTweetsProcessor> logger
)
{
_twitterTweetsService = twitterTweetsService;
_twitterUserDal = twitterUserDal;
@ -31,7 +39,10 @@ namespace BirdsiteLive.Pipeline.Processors
}
#endregion
public async Task<UserWithDataToSync[]> ProcessAsync(UserWithDataToSync[] syncTwitterUsers, CancellationToken ct)
public async Task<UserWithDataToSync[]> ProcessAsync(
UserWithDataToSync[] syncTwitterUsers,
CancellationToken ct
)
{
var usersWtTweets = new List<UserWithDataToSync>();
@ -49,12 +60,24 @@ namespace BirdsiteLive.Pipeline.Processors
{
var tweetId = tweets.Last().Id;
var now = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now, user.MovedTo, user.MovedToAcct, user.Deleted);
await _twitterUserDal.UpdateTwitterUserAsync(
user.Id,
tweetId,
tweetId,
user.FetchingErrorCount,
now
);
}
else
{
var now = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now, user.MovedTo, user.MovedToAcct, user.Deleted);
await _twitterUserDal.UpdateTwitterUserAsync(
user.Id,
user.LastTweetPostedId,
user.LastTweetSynchronizedForAllFollowersId,
user.FetchingErrorCount,
now
);
}
}
@ -63,22 +86,31 @@ namespace BirdsiteLive.Pipeline.Processors
private ExtractedTweet[] RetrieveNewTweets(SyncTwitterUser user)
{
var tweets = new ExtractedTweet[0];
var tweets = Array.Empty<ExtractedTweet>();
try
{
if (user.LastTweetPostedId == -1)
tweets = _twitterTweetsService.GetTimeline(user.Acct, 1);
else
tweets = _twitterTweetsService.GetTimeline(user.Acct, 200, user.LastTweetSynchronizedForAllFollowersId);
tweets = _twitterTweetsService.GetTimeline(
user.Acct,
200,
user.LastTweetSynchronizedForAllFollowersId
);
}
catch (Exception e)
{
_logger.LogError(e, "Error retrieving TL of {Username} from {LastTweetPostedId}, purging user from cache", user.Acct, user.LastTweetPostedId);
_logger.LogError(
e,
"Error retrieving TL of {Username} from {LastTweetPostedId}, purging user from cache",
user.Acct,
user.LastTweetPostedId
);
_twitterUserService.PurgeUser(user.Acct);
}
return tweets;
}
}
}
}

View file

@ -1,14 +1,16 @@
using System;
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using BirdsiteLive.Common.Extensions;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Pipeline.Contracts;
using BirdsiteLive.Pipeline.Tools;
using Microsoft.Extensions.Logging;
namespace BirdsiteLive.Pipeline.Processors
@ -18,11 +20,14 @@ namespace BirdsiteLive.Pipeline.Processors
private readonly ITwitterUserDal _twitterUserDal;
private readonly IMaxUsersNumberProvider _maxUsersNumberProvider;
private readonly ILogger<RetrieveTwitterUsersProcessor> _logger;
public int WaitFactor = 1000 * 60; //1 min
public int WaitFactor = 1000 * 60; //1 min
#region Ctor
public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IMaxUsersNumberProvider maxUsersNumberProvider, ILogger<RetrieveTwitterUsersProcessor> logger)
public RetrieveTwitterUsersProcessor(
ITwitterUserDal twitterUserDal,
IMaxUsersNumberProvider maxUsersNumberProvider,
ILogger<RetrieveTwitterUsersProcessor> logger
)
{
_twitterUserDal = twitterUserDal;
_maxUsersNumberProvider = maxUsersNumberProvider;
@ -30,7 +35,10 @@ namespace BirdsiteLive.Pipeline.Processors
}
#endregion
public async Task GetTwitterUsersAsync(BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock, CancellationToken ct)
public async Task GetTwitterUsersAsync(
BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock,
CancellationToken ct
)
{
for (; ; )
{
@ -39,10 +47,10 @@ namespace BirdsiteLive.Pipeline.Processors
try
{
var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync();
var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber, false);
var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber);
var userCount = users.Any() ? users.Length : 1;
var splitNumber = (int) Math.Ceiling(userCount / 15d);
var splitNumber = (int)Math.Ceiling(userCount / 15d);
var splitUsers = users.Split(splitNumber).ToList();
foreach (var u in splitUsers)
@ -55,7 +63,8 @@ namespace BirdsiteLive.Pipeline.Processors
}
var splitCount = splitUsers.Count();
if (splitCount < 15) await Task.Delay((15 - splitCount) * WaitFactor, ct); //Always wait 15min
if (splitCount < 15)
await Task.Delay((15 - splitCount) * WaitFactor, ct); //Always wait 15min
//// Extra wait time to fit 100.000/day limit
//var extraWaitTime = (int)Math.Ceiling((60 / ((100000d / 24) / userCount)) - 15);
@ -69,4 +78,4 @@ namespace BirdsiteLive.Pipeline.Processors
}
}
}
}
}

View file

@ -1,66 +1,87 @@
using System;
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Moderation.Actions;
using BirdsiteLive.Pipeline.Contracts;
using BirdsiteLive.Pipeline.Models;
using Microsoft.Extensions.Logging;
namespace BirdsiteLive.Pipeline.Processors
namespace BirdsiteLive.Pipeline.Processors;
public class SaveProgressionProcessor : ISaveProgressionProcessor
{
public class SaveProgressionProcessor : ISaveProgressionProcessor
private readonly ITwitterUserDal _twitterUserDal;
private readonly ILogger<SaveProgressionProcessor> _logger;
private readonly IRemoveTwitterAccountAction _removeTwitterAccountAction;
#region Ctor
public SaveProgressionProcessor(
ITwitterUserDal twitterUserDal,
ILogger<SaveProgressionProcessor> logger,
IRemoveTwitterAccountAction removeTwitterAccountAction
)
{
private readonly ITwitterUserDal _twitterUserDal;
private readonly ILogger<SaveProgressionProcessor> _logger;
private readonly IRemoveTwitterAccountAction _removeTwitterAccountAction;
_twitterUserDal = twitterUserDal;
_logger = logger;
_removeTwitterAccountAction = removeTwitterAccountAction;
}
#endregion
#region Ctor
public SaveProgressionProcessor(ITwitterUserDal twitterUserDal, ILogger<SaveProgressionProcessor> logger, IRemoveTwitterAccountAction removeTwitterAccountAction)
public async Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
{
try
{
_twitterUserDal = twitterUserDal;
_logger = logger;
_removeTwitterAccountAction = removeTwitterAccountAction;
}
#endregion
public async Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
{
try
if (userWithTweetsToSync.Tweets.Length == 0)
{
if (userWithTweetsToSync.Tweets.Length == 0)
{
_logger.LogInformation("No tweets synchronized");
await UpdateUserSyncDateAsync(userWithTweetsToSync.User);
return;
}
if(userWithTweetsToSync.Followers.Length == 0)
{
_logger.LogInformation("No Followers found for {User}", userWithTweetsToSync.User.Acct);
await _removeTwitterAccountAction.ProcessAsync(userWithTweetsToSync.User);
return;
}
var userId = userWithTweetsToSync.User.Id;
var followingSyncStatuses = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).ToList();
var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();
var minimumSync = followingSyncStatuses.Min();
var now = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now, userWithTweetsToSync.User.MovedTo, userWithTweetsToSync.User.MovedToAcct, userWithTweetsToSync.User.Deleted);
_logger.LogWarning("No tweets synchronized");
return;
}
catch (Exception e)
if (userWithTweetsToSync.Followers.Length == 0)
{
_logger.LogError(e, "SaveProgressionProcessor.ProcessAsync() Exception");
throw;
}
}
_logger.LogWarning(
"No Followers found for {User}, purging them",
userWithTweetsToSync.User.Acct
);
await _removeTwitterAccountAction.ProcessAsync(userWithTweetsToSync.User);
_logger.LogInformation("Account {User} purged", userWithTweetsToSync.User.Acct);
private async Task UpdateUserSyncDateAsync(SyncTwitterUser user)
return;
}
var userId = userWithTweetsToSync.User.Id;
var followingSyncStatuses = userWithTweetsToSync.Followers
.Select(x => x.FollowingsSyncStatus[userId])
.ToList();
if (followingSyncStatuses.Count == 0)
{
_logger.LogWarning(
"No Followers sync found for {User}, Id: {UserId}",
userWithTweetsToSync.User.Acct,
userId
);
return;
}
var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();
var minimumSync = followingSyncStatuses.Min();
var now = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(
userId,
lastPostedTweet,
minimumSync,
userWithTweetsToSync.User.FetchingErrorCount,
now
);
}
catch (Exception e)
{
user.LastSync = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(user);
_logger.LogError(e, "SaveProgressionProcessor.ProcessAsync() Exception");
throw;
}
}
}
}

View file

@ -1,21 +1,22 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Domain;
using BirdsiteLive.Moderation.Actions;
using BirdsiteLive.Pipeline.Contracts;
using BirdsiteLive.Pipeline.Models;
using BirdsiteLive.Pipeline.Processors.SubTasks;
using BirdsiteLive.Twitter;
using BirdsiteLive.Twitter.Models;
using Microsoft.Extensions.Logging;
using Tweetinvi.Models;
namespace BirdsiteLive.Pipeline.Processors
@ -25,23 +26,27 @@ namespace BirdsiteLive.Pipeline.Processors
private readonly ISendTweetsToInboxTask _sendTweetsToInboxTask;
private readonly ISendTweetsToSharedInboxTask _sendTweetsToSharedInbox;
private readonly IFollowersDal _followersDal;
private readonly InstanceSettings _instanceSettings;
private readonly ILogger<SendTweetsToFollowersProcessor> _logger;
private readonly IRemoveFollowerAction _removeFollowerAction;
#region Ctor
public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox, IFollowersDal followersDal, ILogger<SendTweetsToFollowersProcessor> logger, InstanceSettings instanceSettings, IRemoveFollowerAction removeFollowerAction)
public SendTweetsToFollowersProcessor(
ISendTweetsToInboxTask sendTweetsToInboxTask,
ISendTweetsToSharedInboxTask sendTweetsToSharedInbox,
IFollowersDal followersDal,
ILogger<SendTweetsToFollowersProcessor> logger
)
{
_sendTweetsToInboxTask = sendTweetsToInboxTask;
_sendTweetsToSharedInbox = sendTweetsToSharedInbox;
_logger = logger;
_instanceSettings = instanceSettings;
_removeFollowerAction = removeFollowerAction;
_followersDal = followersDal;
}
#endregion
public async Task<UserWithDataToSync> ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
public async Task<UserWithDataToSync> ProcessAsync(
UserWithDataToSync userWithTweetsToSync,
CancellationToken ct
)
{
var user = userWithTweetsToSync.User;
@ -49,18 +54,30 @@ namespace BirdsiteLive.Pipeline.Processors
var followersWtSharedInbox = userWithTweetsToSync.Followers
.Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
.ToList();
await ProcessFollowersWithSharedInboxAsync(userWithTweetsToSync.Tweets, followersWtSharedInbox, user);
await ProcessFollowersWithSharedInboxAsync(
userWithTweetsToSync.Tweets,
followersWtSharedInbox,
user
);
// Process Inbox
var followerWtInbox = userWithTweetsToSync.Followers
.Where(x => string.IsNullOrWhiteSpace(x.SharedInboxRoute))
.ToList();
await ProcessFollowersWithInboxAsync(userWithTweetsToSync.Tweets, followerWtInbox, user);
await ProcessFollowersWithInboxAsync(
userWithTweetsToSync.Tweets,
followerWtInbox,
user
);
return userWithTweetsToSync;
}
private async Task ProcessFollowersWithSharedInboxAsync(ExtractedTweet[] tweets, List<Follower> followers, SyncTwitterUser user)
private async Task ProcessFollowersWithSharedInboxAsync(
ExtractedTweet[] tweets,
List<Follower> followers,
SyncTwitterUser user
)
{
var followersPerInstances = followers.GroupBy(x => x.Host);
@ -68,7 +85,12 @@ namespace BirdsiteLive.Pipeline.Processors
{
try
{
await _sendTweetsToSharedInbox.ExecuteAsync(tweets, user, followersPerInstance.Key, followersPerInstance.ToArray());
await _sendTweetsToSharedInbox.ExecuteAsync(
tweets,
user,
followersPerInstance.Key,
followersPerInstance.ToArray()
);
foreach (var f in followersPerInstance)
await ProcessWorkingUserAsync(f);
@ -76,15 +98,24 @@ namespace BirdsiteLive.Pipeline.Processors
catch (Exception e)
{
var follower = followersPerInstance.First();
_logger.LogError(e, "Posting to {Host}{Route} failed", follower.Host, follower.SharedInboxRoute);
_logger.LogError(
e,
"Posting to {Host}{Route} failed",
follower.Host,
follower.SharedInboxRoute
);
foreach (var f in followersPerInstance)
await ProcessFailingUserAsync(f);
}
}
}
private async Task ProcessFollowersWithInboxAsync(ExtractedTweet[] tweets, List<Follower> followerWtInbox, SyncTwitterUser user)
private async Task ProcessFollowersWithInboxAsync(
ExtractedTweet[] tweets,
List<Follower> followerWtInbox,
SyncTwitterUser user
)
{
foreach (var follower in followerWtInbox)
{
@ -95,7 +126,12 @@ namespace BirdsiteLive.Pipeline.Processors
}
catch (Exception e)
{
_logger.LogError(e, "Posting to {Host}{Route} failed", follower.Host, follower.InboxRoute);
_logger.LogError(
e,
"Posting to {Host}{Route} failed",
follower.Host,
follower.InboxRoute
);
await ProcessFailingUserAsync(follower);
}
}
@ -113,17 +149,7 @@ namespace BirdsiteLive.Pipeline.Processors
private async Task ProcessFailingUserAsync(Follower follower)
{
follower.PostingErrorCount++;
if (follower.PostingErrorCount > _instanceSettings.FailingFollowerCleanUpThreshold
&& _instanceSettings.FailingFollowerCleanUpThreshold > 0
|| follower.PostingErrorCount > 2147483600)
{
await _removeFollowerAction.ProcessAsync(follower);
}
else
{
await _followersDal.UpdateFollowerAsync(follower);
}
await _followersDal.UpdateFollowerAsync(follower);
}
}
}
}

View file

@ -1,20 +1,26 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Domain;
using BirdsiteLive.Twitter.Models;
using Microsoft.Extensions.Logging;
namespace BirdsiteLive.Pipeline.Processors.SubTasks
{
public interface ISendTweetsToInboxTask
{
Task ExecuteAsync(IEnumerable<ExtractedTweet> tweets, Follower follower, SyncTwitterUser user);
Task ExecuteAsync(
IEnumerable<ExtractedTweet> tweets,
Follower follower,
SyncTwitterUser user
);
}
public class SendTweetsToInboxTask : ISendTweetsToInboxTask
@ -25,9 +31,14 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
private readonly InstanceSettings _settings;
private readonly ILogger<SendTweetsToInboxTask> _logger;
#region Ctor
public SendTweetsToInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger<SendTweetsToInboxTask> logger)
public SendTweetsToInboxTask(
IActivityPubService activityPubService,
IStatusService statusService,
IFollowersDal followersDal,
InstanceSettings settings,
ILogger<SendTweetsToInboxTask> logger
)
{
_activityPubService = activityPubService;
_statusService = statusService;
@ -37,14 +48,15 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
}
#endregion
public async Task ExecuteAsync(IEnumerable<ExtractedTweet> tweets, Follower follower, SyncTwitterUser user)
public async Task ExecuteAsync(
IEnumerable<ExtractedTweet> tweets,
Follower follower,
SyncTwitterUser user
)
{
var userId = user.Id;
var fromStatusId = follower.FollowingsSyncStatus[userId];
var tweetsToSend = tweets
.Where(x => x.Id > fromStatusId)
.OrderBy(x => x.Id)
.ToList();
var tweetsToSend = tweets.Where(x => x.Id > fromStatusId).OrderBy(x => x.Id).ToList();
var inbox = follower.InboxRoute;
@ -55,19 +67,34 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
{
try
{
if (!tweet.IsReply ||
tweet.IsReply && tweet.IsThread ||
_settings.PublishReplies)
if (
!tweet.IsReply
|| tweet.IsReply && tweet.IsThread
|| _settings.PublishReplies
)
{
var note = _statusService.GetStatus(user.Acct, tweet);
await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox);
await _activityPubService.PostNewNoteActivity(
note,
user.Acct,
tweet.Id.ToString(),
follower.Host,
inbox
);
}
}
catch (ArgumentException e)
{
if (e.Message.Contains("Invalid pattern") && e.Message.Contains("at offset")) //Regex exception
if (
e.Message.Contains("Invalid pattern") && e.Message.Contains("at offset")
) //Regex exception
{
_logger.LogError(e, "Can't parse {MessageContent} from Tweet {Id}", tweet.MessageContent, tweet.Id);
_logger.LogError(
e,
"Can't parse {MessageContent} from Tweet {Id}",
tweet.MessageContent,
tweet.Id
);
}
else
{
@ -88,4 +115,4 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
}
}
}
}
}

View file

@ -1,19 +1,26 @@
using System;
using System;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Domain;
using BirdsiteLive.Twitter.Models;
using Microsoft.Extensions.Logging;
namespace BirdsiteLive.Pipeline.Processors.SubTasks
{
public interface ISendTweetsToSharedInboxTask
{
Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, string host, Follower[] followersPerInstance);
Task ExecuteAsync(
ExtractedTweet[] tweets,
SyncTwitterUser user,
string host,
Follower[] followersPerInstance
);
}
public class SendTweetsToSharedInboxTask : ISendTweetsToSharedInboxTask
@ -25,7 +32,13 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
private readonly ILogger<SendTweetsToSharedInboxTask> _logger;
#region Ctor
public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger<SendTweetsToSharedInboxTask> logger)
public SendTweetsToSharedInboxTask(
IActivityPubService activityPubService,
IStatusService statusService,
IFollowersDal followersDal,
InstanceSettings settings,
ILogger<SendTweetsToSharedInboxTask> logger
)
{
_activityPubService = activityPubService;
_statusService = statusService;
@ -35,18 +48,19 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
}
#endregion
public async Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, string host, Follower[] followersPerInstance)
public async Task ExecuteAsync(
ExtractedTweet[] tweets,
SyncTwitterUser user,
string host,
Follower[] followersPerInstance
)
{
var userId = user.Id;
var inbox = followersPerInstance.First().SharedInboxRoute;
var fromStatusId = followersPerInstance
.Max(x => x.FollowingsSyncStatus[userId]);
var fromStatusId = followersPerInstance.Max(x => x.FollowingsSyncStatus[userId]);
var tweetsToSend = tweets
.Where(x => x.Id > fromStatusId)
.OrderBy(x => x.Id)
.ToList();
var tweetsToSend = tweets.Where(x => x.Id > fromStatusId).OrderBy(x => x.Id).ToList();
var syncStatus = fromStatusId;
try
@ -55,19 +69,34 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
{
try
{
if (!tweet.IsReply ||
tweet.IsReply && tweet.IsThread ||
_settings.PublishReplies)
if (
!tweet.IsReply
|| tweet.IsReply && tweet.IsThread
|| _settings.PublishReplies
)
{
var note = _statusService.GetStatus(user.Acct, tweet);
await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox);
await _activityPubService.PostNewNoteActivity(
note,
user.Acct,
tweet.Id.ToString(),
host,
inbox
);
}
}
catch (ArgumentException e)
{
if (e.Message.Contains("Invalid pattern") && e.Message.Contains("at offset")) //Regex exception
if (
e.Message.Contains("Invalid pattern") && e.Message.Contains("at offset")
) //Regex exception
{
_logger.LogError(e, "Can't parse {MessageContent} from Tweet {Id}", tweet.MessageContent, tweet.Id);
_logger.LogError(
e,
"Can't parse {MessageContent} from Tweet {Id}",
tweet.MessageContent,
tweet.Id
);
}
else
{
@ -91,4 +120,4 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
}
}
}
}
}

View file

@ -1,11 +1,13 @@
using System;
using System;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using BirdsiteLive.DAL.Models;
using BirdsiteLive.Pipeline.Contracts;
using BirdsiteLive.Pipeline.Models;
using Microsoft.Extensions.Logging;
namespace BirdsiteLive.Pipeline
@ -26,7 +28,15 @@ namespace BirdsiteLive.Pipeline
private readonly ILogger<StatusPublicationPipeline> _logger;
#region Ctor
public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ISaveProgressionProcessor saveProgressionProcessor, IRefreshTwitterUserStatusProcessor refreshTwitterUserStatusProcessor, ILogger<StatusPublicationPipeline> logger)
public StatusPublicationPipeline(
IRetrieveTweetsProcessor retrieveTweetsProcessor,
IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor,
IRetrieveFollowersProcessor retrieveFollowersProcessor,
ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor,
ISaveProgressionProcessor saveProgressionProcessor,
IRefreshTwitterUserStatusProcessor refreshTwitterUserStatusProcessor,
ILogger<StatusPublicationPipeline> logger
)
{
_retrieveTweetsProcessor = retrieveTweetsProcessor;
_retrieveTwitterAccountsProcessor = retrieveTwitterAccountsProcessor;
@ -41,37 +51,107 @@ namespace BirdsiteLive.Pipeline
public async Task ExecuteAsync(CancellationToken ct)
{
// Create blocks
var twitterUserToRefreshBufferBlock = new BufferBlock<SyncTwitterUser[]>(new DataflowBlockOptions
{ BoundedCapacity = 1, CancellationToken = ct });
var twitterUserToRefreshBlock = new TransformBlock<SyncTwitterUser[], UserWithDataToSync[]>(async x => await _refreshTwitterUserStatusProcessor.ProcessAsync(x, ct));
var twitterUsersBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
var retrieveTweetsBlock = new TransformBlock<UserWithDataToSync[], UserWithDataToSync[]>(async x => await _retrieveTweetsProcessor.ProcessAsync(x, ct));
var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
var retrieveFollowersBlock = new TransformManyBlock<UserWithDataToSync[], UserWithDataToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct));
var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
var sendTweetsToFollowersBlock = new TransformBlock<UserWithDataToSync, UserWithDataToSync>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
var sendTweetsToFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
var saveProgressionBlock = new ActionBlock<UserWithDataToSync>(async x => await _saveProgressionProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
// Create blocks
var twitterUserToRefreshBufferBlock = new BufferBlock<SyncTwitterUser[]>(
new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct }
);
var twitterUserToRefreshBlock = new TransformBlock<
SyncTwitterUser[],
UserWithDataToSync[]
>(async x => await _refreshTwitterUserStatusProcessor.ProcessAsync(x, ct));
var twitterUsersBufferBlock = new BufferBlock<UserWithDataToSync[]>(
new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct }
);
var retrieveTweetsBlock = new TransformBlock<
UserWithDataToSync[],
UserWithDataToSync[]
>(async x => await _retrieveTweetsProcessor.ProcessAsync(x, ct));
var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(
new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct }
);
var retrieveFollowersBlock = new TransformManyBlock<
UserWithDataToSync[],
UserWithDataToSync
>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct));
var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(
new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct }
);
var sendTweetsToFollowersBlock = new TransformBlock<
UserWithDataToSync,
UserWithDataToSync
>(
async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 5,
CancellationToken = ct
}
);
var sendTweetsToFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(
new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct }
);
var saveProgressionBlock = new ActionBlock<UserWithDataToSync>(
async x => await _saveProgressionProcessor.ProcessAsync(x, ct),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 5,
CancellationToken = ct
}
);
// Link pipeline
twitterUserToRefreshBufferBlock.LinkTo(twitterUserToRefreshBlock, new DataflowLinkOptions { PropagateCompletion = true });
twitterUserToRefreshBlock.LinkTo(twitterUsersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
twitterUsersBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions { PropagateCompletion = true });
retrieveTweetsBlock.LinkTo(retrieveTweetsBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
retrieveTweetsBufferBlock.LinkTo(retrieveFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
retrieveFollowersBlock.LinkTo(retrieveFollowersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
retrieveFollowersBufferBlock.LinkTo(sendTweetsToFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
sendTweetsToFollowersBlock.LinkTo(sendTweetsToFollowersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
sendTweetsToFollowersBufferBlock.LinkTo(saveProgressionBlock, new DataflowLinkOptions { PropagateCompletion = true });
twitterUserToRefreshBufferBlock.LinkTo(
twitterUserToRefreshBlock,
new DataflowLinkOptions { PropagateCompletion = true }
);
twitterUserToRefreshBlock.LinkTo(
twitterUsersBufferBlock,
new DataflowLinkOptions { PropagateCompletion = true }
);
twitterUsersBufferBlock.LinkTo(
retrieveTweetsBlock,
new DataflowLinkOptions { PropagateCompletion = true }
);
retrieveTweetsBlock.LinkTo(
retrieveTweetsBufferBlock,
new DataflowLinkOptions { PropagateCompletion = true }
);
retrieveTweetsBufferBlock.LinkTo(
retrieveFollowersBlock,
new DataflowLinkOptions { PropagateCompletion = true }
);
retrieveFollowersBlock.LinkTo(
retrieveFollowersBufferBlock,
new DataflowLinkOptions { PropagateCompletion = true }
);
retrieveFollowersBufferBlock.LinkTo(
sendTweetsToFollowersBlock,
new DataflowLinkOptions { PropagateCompletion = true }
);
sendTweetsToFollowersBlock.LinkTo(
sendTweetsToFollowersBufferBlock,
new DataflowLinkOptions { PropagateCompletion = true }
);
sendTweetsToFollowersBufferBlock.LinkTo(
saveProgressionBlock,
new DataflowLinkOptions { PropagateCompletion = true }
);
// Launch twitter user retriever
var retrieveTwitterAccountsTask = _retrieveTwitterAccountsProcessor.GetTwitterUsersAsync(twitterUserToRefreshBufferBlock, ct);
var retrieveTwitterAccountsTask =
_retrieveTwitterAccountsProcessor.GetTwitterUsersAsync(
twitterUserToRefreshBufferBlock,
ct
);
// Wait
await Task.WhenAny(new[] { retrieveTwitterAccountsTask, saveProgressionBlock.Completion });
await Task.WhenAny(
new[] { retrieveTwitterAccountsTask, saveProgressionBlock.Completion }
);
var ex = retrieveTwitterAccountsTask.IsFaulted ? retrieveTwitterAccountsTask.Exception : saveProgressionBlock.Completion.Exception;
var ex = retrieveTwitterAccountsTask.IsFaulted
? retrieveTwitterAccountsTask.Exception
: saveProgressionBlock.Completion.Exception;
_logger.LogCritical(ex, "An error occurred, pipeline stopped");
}
}

View file

@ -1,4 +1,5 @@
using System.Threading.Tasks;
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
@ -13,13 +14,16 @@ namespace BirdsiteLive.Pipeline.Tools
{
private readonly InstanceSettings _instanceSettings;
private readonly ITwitterUserDal _twitterUserDal;
private int _totalUsersCount = -1;
private int _warmUpIterations;
private const int WarmUpMaxCapacity = 200;
#region Ctor
public MaxUsersNumberProvider(InstanceSettings instanceSettings, ITwitterUserDal twitterUserDal)
public MaxUsersNumberProvider(
InstanceSettings instanceSettings,
ITwitterUserDal twitterUserDal
)
{
_instanceSettings = instanceSettings;
_twitterUserDal = twitterUserDal;
@ -29,21 +33,21 @@ namespace BirdsiteLive.Pipeline.Tools
public async Task<int> GetMaxUsersNumberAsync()
{
// Init data
if (_totalUsersCount == -1)
if (_totalUsersCount == -1)
{
_totalUsersCount = await _twitterUserDal.GetTwitterUsersCountAsync();
_warmUpIterations = (int)(_totalUsersCount / (float)WarmUpMaxCapacity);
}
// Return if warm up ended
if (_warmUpIterations <= 0) return _instanceSettings.MaxUsersCapacity;
if (_warmUpIterations <= 0)
return _instanceSettings.MaxUsersCapacity;
// Calculate warm up value
var maxUsers = _warmUpIterations > 0
? WarmUpMaxCapacity
: _instanceSettings.MaxUsersCapacity;
var maxUsers =
_warmUpIterations > 0 ? WarmUpMaxCapacity : _instanceSettings.MaxUsersCapacity;
_warmUpIterations--;
return maxUsers;
}
}
}
}

Some files were not shown because too many files have changed in this diff Show more