From 55e04331a0adf144775d017ae1053a1336789754 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Tue, 7 Jul 2020 01:33:52 -0400 Subject: [PATCH 01/67] added worker service --- src/BirdsiteLive/Program.cs | 6 ++++ .../Services/FederationService.cs | 29 +++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 src/BirdsiteLive/Services/FederationService.cs diff --git a/src/BirdsiteLive/Program.cs b/src/BirdsiteLive/Program.cs index c109ad2..d238b02 100644 --- a/src/BirdsiteLive/Program.cs +++ b/src/BirdsiteLive/Program.cs @@ -2,9 +2,11 @@ using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using BirdsiteLive.Services; using Lamar.Microsoft.DependencyInjection; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -23,6 +25,10 @@ namespace BirdsiteLive .ConfigureWebHostDefaults(webBuilder => { webBuilder.UseStartup(); + }) + .ConfigureServices(services => + { + services.AddHostedService(); }); } } diff --git a/src/BirdsiteLive/Services/FederationService.cs b/src/BirdsiteLive/Services/FederationService.cs new file mode 100644 index 0000000..ee07161 --- /dev/null +++ b/src/BirdsiteLive/Services/FederationService.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.Domain; +using Microsoft.Extensions.Hosting; + +namespace BirdsiteLive.Services +{ + public class FederationService : BackgroundService + { + private readonly IUserService _userService; + + #region Ctor + public FederationService(IUserService userService) + { + _userService = userService; + } + #endregion + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + for (;;) + { + Console.WriteLine("RUNNING SERVICE"); + await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken); + } + } + } +} \ No newline at end of file From 6a0c2884cd6dec6af52ab3923062d438ab80f9bd Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Tue, 7 Jul 2020 02:16:22 -0400 Subject: [PATCH 02/67] trigger on all branches --- .github/workflows/dotnet-core.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/dotnet-core.yml b/.github/workflows/dotnet-core.yml index 9e68110..b1fc70d 100644 --- a/.github/workflows/dotnet-core.yml +++ b/.github/workflows/dotnet-core.yml @@ -1,10 +1,6 @@ -name: .NET Core +name: ASP.NET Core Build & Tests -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] +on: [push, pull_request] jobs: build: From 24b6061b16deae912d11c2a8735f211048a6571b Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Tue, 7 Jul 2020 02:57:25 -0400 Subject: [PATCH 03/67] fix nodeinfo 2.0 --- src/BirdsiteLive/Controllers/WellKnownController.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/BirdsiteLive/Controllers/WellKnownController.cs b/src/BirdsiteLive/Controllers/WellKnownController.cs index 613b948..9c889a0 100644 --- a/src/BirdsiteLive/Controllers/WellKnownController.cs +++ b/src/BirdsiteLive/Controllers/WellKnownController.cs @@ -57,14 +57,19 @@ namespace BirdsiteLive.Controllers }, software = new Software() { - name = "BirdsiteLive", + name = "birdsitelive", version = "0.1.0" }, protocols = new [] { "activitypub" }, - openRegistrations = false + openRegistrations = false, + services = new Services() + { + inbound = new object[0], + outbound = new object[0] + } }; return new JsonResult(nodeInfo); @@ -164,7 +169,7 @@ namespace BirdsiteLive.Controllers public Usage usage { get; set; } public bool openRegistrations { get; set; } public Services services { get; set; } - public object metadata { get; set; } + //public object metadata { get; set; } } public class Services From 5cd6279da82e85df4b9f461a3f9f200b311f82ea Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Tue, 7 Jul 2020 18:30:52 -0400 Subject: [PATCH 04/67] added nodeinfo 2.1 --- .../Controllers/WellKnownController.cs | 151 ++++++++---------- .../Models/WellKnownModels/Link.cs | 8 + .../Models/WellKnownModels/NodeInfoV20.cs | 13 ++ .../Models/WellKnownModels/NodeInfoV21.cs | 13 ++ .../Models/WellKnownModels/Services.cs | 8 + .../Models/WellKnownModels/Software.cs | 8 + .../Models/WellKnownModels/SoftwareV21.cs | 9 ++ .../Models/WellKnownModels/Usage.cs | 8 + .../Models/WellKnownModels/Users.cs | 7 + .../Models/WellKnownModels/WebFingerLink.cs | 9 ++ .../Models/WellKnownModels/WebFingerResult.cs | 11 ++ .../WellKnownModels/WellKnownNodeInfo.cs | 7 + 12 files changed, 168 insertions(+), 84 deletions(-) create mode 100644 src/BirdsiteLive/Models/WellKnownModels/Link.cs create mode 100644 src/BirdsiteLive/Models/WellKnownModels/NodeInfoV20.cs create mode 100644 src/BirdsiteLive/Models/WellKnownModels/NodeInfoV21.cs create mode 100644 src/BirdsiteLive/Models/WellKnownModels/Services.cs create mode 100644 src/BirdsiteLive/Models/WellKnownModels/Software.cs create mode 100644 src/BirdsiteLive/Models/WellKnownModels/SoftwareV21.cs create mode 100644 src/BirdsiteLive/Models/WellKnownModels/Usage.cs create mode 100644 src/BirdsiteLive/Models/WellKnownModels/Users.cs create mode 100644 src/BirdsiteLive/Models/WellKnownModels/WebFingerLink.cs create mode 100644 src/BirdsiteLive/Models/WellKnownModels/WebFingerResult.cs create mode 100644 src/BirdsiteLive/Models/WellKnownModels/WellKnownNodeInfo.cs diff --git a/src/BirdsiteLive/Controllers/WellKnownController.cs b/src/BirdsiteLive/Controllers/WellKnownController.cs index 9c889a0..3fce212 100644 --- a/src/BirdsiteLive/Controllers/WellKnownController.cs +++ b/src/BirdsiteLive/Controllers/WellKnownController.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Threading.Tasks; using BirdsiteLive.Common.Settings; using BirdsiteLive.Models; +using BirdsiteLive.Models.WellKnownModels; using BirdsiteLive.Twitter; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; @@ -35,44 +36,85 @@ namespace BirdsiteLive.Controllers { rel = "http://nodeinfo.diaspora.software/ns/schema/2.0", href = $"https://{_settings.Domain}/nodeinfo/2.0.json" + }, + new Link() + { + rel = "http://nodeinfo.diaspora.software/ns/schema/2.1", + href = $"https://{_settings.Domain}/nodeinfo/2.1.json" } } }; return new JsonResult(nodeInfo); } - [Route("/nodeinfo/2.0.json")] - public IActionResult NodeInfo() + [Route("/nodeinfo/{id}.json")] + public IActionResult NodeInfo(string id) { - var nodeInfo = new NodeInfo + if (id == "2.0") { - version = "2.0", - usage = new Usage() + var nodeInfo = new NodeInfoV20 { - localPosts = 0, - users = new Users() + version = "2.0", + usage = new Usage() { - total = 0 + localPosts = 0, + users = new Users() + { + total = 0 + } + }, + software = new Software() + { + name = "birdsitelive", + version = "0.1.0" + }, + protocols = new[] + { + "activitypub" + }, + openRegistrations = false, + services = new Models.WellKnownModels.Services() + { + inbound = new object[0], + outbound = new object[0] } - }, - software = new Software() + }; + return new JsonResult(nodeInfo); + } + if (id == "2.1") + { + var nodeInfo = new NodeInfoV21 { - name = "birdsitelive", - version = "0.1.0" - }, - protocols = new [] - { - "activitypub" - }, - openRegistrations = false, - services = new Services() - { - inbound = new object[0], - outbound = new object[0] - } - }; + version = "2.1", + usage = new Usage() + { + localPosts = 0, + users = new Users() + { + total = 0 + } + }, + software = new SoftwareV21() + { + name = "birdsitelive", + version = "0.1.0", + repository = "https://github.com/NicolasConstant/BirdsiteLive" + }, + protocols = new[] + { + "activitypub" + }, + openRegistrations = false, + services = new Models.WellKnownModels.Services() + { + inbound = new object[0], + outbound = new object[0] + } + }; + return new JsonResult(nodeInfo); + } - return new JsonResult(nodeInfo); + return NotFound(); } [Route("/.well-known/webfinger")] @@ -135,63 +177,4 @@ namespace BirdsiteLive.Controllers return new JsonResult(result); } } - - public class WebFingerResult - { - public string subject { get; set; } - public string[] aliases { get; set; } - public List links { get; set; } = new List(); - } - - public class WebFingerLink - { - public string rel { get; set; } - public string type { get; set; } - public string href { get; set; } - } - - public class WellKnownNodeInfo - { - public Link[] links { get; set; } - } - - public class Link - { - public string href { get; set; } - public string rel { get; set; } - } - - public class NodeInfo - { - public string version { get; set; } - public string[] protocols { get; set; } - public Software software { get; set; } - public Usage usage { get; set; } - public bool openRegistrations { get; set; } - public Services services { get; set; } - //public object metadata { get; set; } - } - - public class Services - { - public object[] inbound { get; set; } - public object[] outbound { get; set; } - } - - public class Software - { - public string name { get; set; } - public string version { get; set; } - } - - public class Usage - { - public int localPosts { get; set; } - public Users users { get; set; } - } - - public class Users - { - public int total { get; set; } - } } \ No newline at end of file diff --git a/src/BirdsiteLive/Models/WellKnownModels/Link.cs b/src/BirdsiteLive/Models/WellKnownModels/Link.cs new file mode 100644 index 0000000..e4bedfe --- /dev/null +++ b/src/BirdsiteLive/Models/WellKnownModels/Link.cs @@ -0,0 +1,8 @@ +namespace BirdsiteLive.Models.WellKnownModels +{ + public class Link + { + public string href { get; set; } + public string rel { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Models/WellKnownModels/NodeInfoV20.cs b/src/BirdsiteLive/Models/WellKnownModels/NodeInfoV20.cs new file mode 100644 index 0000000..f2c1f92 --- /dev/null +++ b/src/BirdsiteLive/Models/WellKnownModels/NodeInfoV20.cs @@ -0,0 +1,13 @@ +namespace BirdsiteLive.Models.WellKnownModels +{ + public class NodeInfoV20 + { + public string version { get; set; } + public string[] protocols { get; set; } + public Software software { get; set; } + public Usage usage { get; set; } + public bool openRegistrations { get; set; } + public Services services { get; set; } + //public object metadata { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Models/WellKnownModels/NodeInfoV21.cs b/src/BirdsiteLive/Models/WellKnownModels/NodeInfoV21.cs new file mode 100644 index 0000000..ba50f5b --- /dev/null +++ b/src/BirdsiteLive/Models/WellKnownModels/NodeInfoV21.cs @@ -0,0 +1,13 @@ +namespace BirdsiteLive.Models.WellKnownModels +{ + public class NodeInfoV21 + { + public string version { get; set; } + public string[] protocols { get; set; } + public Usage usage { get; set; } + public bool openRegistrations { get; set; } + public SoftwareV21 software { get; set; } + public Services services { get; set; } + //public object metadata { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Models/WellKnownModels/Services.cs b/src/BirdsiteLive/Models/WellKnownModels/Services.cs new file mode 100644 index 0000000..fa25074 --- /dev/null +++ b/src/BirdsiteLive/Models/WellKnownModels/Services.cs @@ -0,0 +1,8 @@ +namespace BirdsiteLive.Models.WellKnownModels +{ + public class Services + { + public object[] inbound { get; set; } + public object[] outbound { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Models/WellKnownModels/Software.cs b/src/BirdsiteLive/Models/WellKnownModels/Software.cs new file mode 100644 index 0000000..9cbefa6 --- /dev/null +++ b/src/BirdsiteLive/Models/WellKnownModels/Software.cs @@ -0,0 +1,8 @@ +namespace BirdsiteLive.Models.WellKnownModels +{ + public class Software + { + public string name { get; set; } + public string version { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Models/WellKnownModels/SoftwareV21.cs b/src/BirdsiteLive/Models/WellKnownModels/SoftwareV21.cs new file mode 100644 index 0000000..c6fa851 --- /dev/null +++ b/src/BirdsiteLive/Models/WellKnownModels/SoftwareV21.cs @@ -0,0 +1,9 @@ +namespace BirdsiteLive.Models.WellKnownModels +{ + public class SoftwareV21 + { + public string name { get; set; } + public string repository { get; set; } + public string version { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Models/WellKnownModels/Usage.cs b/src/BirdsiteLive/Models/WellKnownModels/Usage.cs new file mode 100644 index 0000000..693875f --- /dev/null +++ b/src/BirdsiteLive/Models/WellKnownModels/Usage.cs @@ -0,0 +1,8 @@ +namespace BirdsiteLive.Models.WellKnownModels +{ + public class Usage + { + public int localPosts { get; set; } + public Users users { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Models/WellKnownModels/Users.cs b/src/BirdsiteLive/Models/WellKnownModels/Users.cs new file mode 100644 index 0000000..3abdb70 --- /dev/null +++ b/src/BirdsiteLive/Models/WellKnownModels/Users.cs @@ -0,0 +1,7 @@ +namespace BirdsiteLive.Models.WellKnownModels +{ + public class Users + { + public int total { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Models/WellKnownModels/WebFingerLink.cs b/src/BirdsiteLive/Models/WellKnownModels/WebFingerLink.cs new file mode 100644 index 0000000..9945336 --- /dev/null +++ b/src/BirdsiteLive/Models/WellKnownModels/WebFingerLink.cs @@ -0,0 +1,9 @@ +namespace BirdsiteLive.Models.WellKnownModels +{ + public class WebFingerLink + { + public string rel { get; set; } + public string type { get; set; } + public string href { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Models/WellKnownModels/WebFingerResult.cs b/src/BirdsiteLive/Models/WellKnownModels/WebFingerResult.cs new file mode 100644 index 0000000..96c2e84 --- /dev/null +++ b/src/BirdsiteLive/Models/WellKnownModels/WebFingerResult.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; + +namespace BirdsiteLive.Models.WellKnownModels +{ + public class WebFingerResult + { + public string subject { get; set; } + public string[] aliases { get; set; } + public List links { get; set; } = new List(); + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Models/WellKnownModels/WellKnownNodeInfo.cs b/src/BirdsiteLive/Models/WellKnownModels/WellKnownNodeInfo.cs new file mode 100644 index 0000000..d34abe6 --- /dev/null +++ b/src/BirdsiteLive/Models/WellKnownModels/WellKnownNodeInfo.cs @@ -0,0 +1,7 @@ +namespace BirdsiteLive.Models.WellKnownModels +{ + public class WellKnownNodeInfo + { + public Link[] links { get; set; } + } +} \ No newline at end of file From 34bf9ff140969e4b7b994793d00ad9fe555a0640 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Tue, 7 Jul 2020 18:39:35 -0400 Subject: [PATCH 05/67] added inbox property for followers --- .../DbInitializerPostgresDal.cs | 5 ++-- .../DataAccessLayers/FollowersPostgresDal.cs | 8 +++--- .../Contracts/IFollowersDal.cs | 2 +- .../BirdsiteLive.DAL/Models/Follower.cs | 1 + .../FollowersPostgresDalTests.cs | 25 +++++++++++++------ 5 files changed, 27 insertions(+), 14 deletions(-) diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs index 7b36644..832eaf4 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs @@ -106,8 +106,9 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers followings INTEGER[], followingsSyncStatus JSONB, - acct VARCHAR(50), - host VARCHAR(253), + acct VARCHAR(50) NOT NULL, + host VARCHAR(253) NOT NULL, + inboxUrl VARCHAR(2048) NOT NULL, UNIQUE (acct, host) );"; await _tools.ExecuteRequestAsync(createFollowers); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs index 7e6c2ac..fdfd7e7 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs @@ -20,7 +20,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } #endregion - public async Task CreateFollowerAsync(string acct, string host, int[] followings, Dictionary followingSyncStatus) + public async Task CreateFollowerAsync(string acct, string host, int[] followings, Dictionary followingSyncStatus, string inboxUrl) { var serializedDic = JsonConvert.SerializeObject(followingSyncStatus); @@ -32,8 +32,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers dbConnection.Open(); await dbConnection.ExecuteAsync( - $"INSERT INTO {_settings.FollowersTableName} (acct,host,followings,followingsSyncStatus) VALUES(@acct,@host,@followings, CAST(@followingsSyncStatus as json))", - new { acct, host, followings, followingsSyncStatus = serializedDic }); + $"INSERT INTO {_settings.FollowersTableName} (acct,host,inboxUrl,followings,followingsSyncStatus) VALUES(@acct,@host,@inboxUrl,@followings,CAST(@followingsSyncStatus as json))", + new { acct, host, inboxUrl, followings, followingsSyncStatus = serializedDic }); } } @@ -124,6 +124,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers Id = follower.Id, Acct = follower.Acct, Host = follower.Host, + InboxUrl = follower.InboxUrl, Followings = follower.Followings, FollowingsSyncStatus = JsonConvert.DeserializeObject>(follower.FollowingsSyncStatus) }; @@ -138,5 +139,6 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public string Acct { get; set; } public string Host { get; set; } + public string InboxUrl { get; set; } } } \ No newline at end of file diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs index 92e0cb3..19b3dbc 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs @@ -7,7 +7,7 @@ namespace BirdsiteLive.DAL.Contracts public interface IFollowersDal { Task GetFollowerAsync(string acct, string host); - Task CreateFollowerAsync(string acct, string host, int[] followings, Dictionary followingSyncStatus); + Task CreateFollowerAsync(string acct, string host, int[] followings, Dictionary followingSyncStatus, string inboxUrl); Task GetFollowersAsync(int followedUserId); Task UpdateFollowerAsync(int id, int[] followings, Dictionary followingSyncStatus); Task DeleteFollowerAsync(int id); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Models/Follower.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Models/Follower.cs index 5eedafb..32ee22b 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Models/Follower.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Models/Follower.cs @@ -11,5 +11,6 @@ namespace BirdsiteLive.DAL.Models public string Acct { get; set; } public string Host { get; set; } + public string InboxUrl { get; set; } } } \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs index f359f06..fed68ed 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs @@ -45,15 +45,17 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers {19, 166L}, {23, 167L} }; + var inboxUrl = "https://domain.ext/myhandle/inbox"; var dal = new FollowersPostgresDal(_settings); - await dal.CreateFollowerAsync(acct, host, following, followingSync); + await dal.CreateFollowerAsync(acct, host, following, followingSync, inboxUrl); var result = await dal.GetFollowerAsync(acct, host); Assert.IsNotNull(result); Assert.AreEqual(acct, result.Acct); Assert.AreEqual(host, result.Host); + Assert.AreEqual(inboxUrl, result.InboxUrl); Assert.AreEqual(following.Length, result.Followings.Length); Assert.AreEqual(following[0], result.Followings[0]); Assert.AreEqual(followingSync.Count, result.FollowingsSyncStatus.Count); @@ -71,19 +73,22 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers var host = "domain.ext"; var following = new[] { 1,2,3 }; var followingSync = new Dictionary(); - await dal.CreateFollowerAsync(acct, host, following, followingSync); + var inboxUrl = "https://domain.ext/myhandle1/inbox"; + await dal.CreateFollowerAsync(acct, host, following, followingSync, inboxUrl); //User 2 acct = "myhandle2"; host = "domain.ext"; following = new[] { 2, 4, 5 }; - await dal.CreateFollowerAsync(acct, host, following, followingSync); + inboxUrl = "https://domain.ext/myhandle2/inbox"; + await dal.CreateFollowerAsync(acct, host, following, followingSync, inboxUrl); //User 2 acct = "myhandle3"; host = "domain.ext"; following = new[] { 1 }; - await dal.CreateFollowerAsync(acct, host, following, followingSync); + inboxUrl = "https://domain.ext/myhandle3/inbox"; + await dal.CreateFollowerAsync(acct, host, following, followingSync, inboxUrl); var result = await dal.GetFollowersAsync(2); Assert.AreEqual(2, result.Length); @@ -107,9 +112,10 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers {19, 166L}, {23, 167L} }; + var inboxUrl = "https://domain.ext/myhandle/inbox"; var dal = new FollowersPostgresDal(_settings); - await dal.CreateFollowerAsync(acct, host, following, followingSync); + await dal.CreateFollowerAsync(acct, host, following, followingSync, inboxUrl); var result = await dal.GetFollowerAsync(acct, host); var updatedFollowing = new[] { 12, 19, 23, 24 }; @@ -143,9 +149,10 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers {19, 166L}, {23, 167L} }; + var inboxUrl = "https://domain.ext/myhandle/inbox"; var dal = new FollowersPostgresDal(_settings); - await dal.CreateFollowerAsync(acct, host, following, followingSync); + await dal.CreateFollowerAsync(acct, host, following, followingSync, inboxUrl); var result = await dal.GetFollowerAsync(acct, host); var updatedFollowing = new[] { 12, 19 }; @@ -177,9 +184,10 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers {19, 166L}, {23, 167L} }; + var inboxUrl = "https://domain.ext/myhandle/inbox"; var dal = new FollowersPostgresDal(_settings); - await dal.CreateFollowerAsync(acct, host, following, followingSync); + await dal.CreateFollowerAsync(acct, host, following, followingSync, inboxUrl); var result = await dal.GetFollowerAsync(acct, host); Assert.IsNotNull(result); @@ -201,9 +209,10 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers {19, 166L}, {23, 167L} }; + var inboxUrl = "https://domain.ext/myhandle/inbox"; var dal = new FollowersPostgresDal(_settings); - await dal.CreateFollowerAsync(acct, host, following, followingSync); + await dal.CreateFollowerAsync(acct, host, following, followingSync, inboxUrl); var result = await dal.GetFollowerAsync(acct, host); Assert.IsNotNull(result); From 5f5eeb95300194c71cf4796f2612149dae6c2048 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Tue, 7 Jul 2020 21:03:20 -0400 Subject: [PATCH 06/67] saving following in db --- .../Settings/InstanceSettings.cs | 2 + .../BirdsiteLive.Domain.csproj | 1 + .../BusinessUseCases/ProcessFollowUser.cs | 55 +++++++++++++++++++ .../BusinessUseCases/ProcessUnfollowUser.cs | 7 +++ src/BirdsiteLive.Domain/UserService.cs | 38 ++++++++++--- src/BirdsiteLive/BirdsiteLive.csproj | 1 + .../Controllers/WellKnownController.cs | 4 +- .../Services/FederationService.cs | 22 +++++++- src/BirdsiteLive/Startup.cs | 19 ++++++- src/BirdsiteLive/appsettings.json | 3 +- .../DataAccessLayers/FollowersPostgresDal.cs | 16 ++++-- .../Settings/PostgresSettings.cs | 6 +- .../Contracts/IFollowersDal.cs | 5 +- .../BirdsiteLive.DAL/Models/Follower.cs | 2 +- .../FollowersPostgresDalTests.cs | 36 ++++++------ 15 files changed, 174 insertions(+), 43 deletions(-) create mode 100644 src/BirdsiteLive.Domain/BusinessUseCases/ProcessFollowUser.cs create mode 100644 src/BirdsiteLive.Domain/BusinessUseCases/ProcessUnfollowUser.cs diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs index aabe822..c0fff2e 100644 --- a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs +++ b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs @@ -3,5 +3,7 @@ public class InstanceSettings { public string Domain { get; set; } + + public string PostgresConnString { get; set; } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj b/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj index 50eb4d2..cb89578 100644 --- a/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj +++ b/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj @@ -8,6 +8,7 @@ + diff --git a/src/BirdsiteLive.Domain/BusinessUseCases/ProcessFollowUser.cs b/src/BirdsiteLive.Domain/BusinessUseCases/ProcessFollowUser.cs new file mode 100644 index 0000000..c09c696 --- /dev/null +++ b/src/BirdsiteLive.Domain/BusinessUseCases/ProcessFollowUser.cs @@ -0,0 +1,55 @@ +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; + +namespace BirdsiteLive.Domain.BusinessUseCases +{ + public interface IProcessFollowUser + { + Task ExecuteAsync(string followerUsername, string followerDomain, string followerInbox, string twitterUser); + } + + public class ProcessFollowUser : IProcessFollowUser + { + private readonly IFollowersDal _followerDal; + private readonly ITwitterUserDal _twitterUserDal; + + #region Ctor + public ProcessFollowUser(IFollowersDal followerDal, ITwitterUserDal twitterUserDal) + { + _followerDal = followerDal; + _twitterUserDal = twitterUserDal; + } + #endregion + + public async Task ExecuteAsync(string followerUsername, string followerDomain, string followerInbox, string twitterUsername) + { + // Get Follower and Twitter Users + var follower = await _followerDal.GetFollowerAsync(followerUsername, followerDomain); + if (follower == null) + { + await _followerDal.CreateFollowerAsync(followerUsername, followerDomain, followerInbox); + follower = await _followerDal.GetFollowerAsync(followerUsername, followerDomain); + } + + var twitterUser = await _twitterUserDal.GetTwitterUserAsync(twitterUsername); + if (twitterUser == null) + { + await _twitterUserDal.CreateTwitterUserAsync(twitterUsername, -1); + twitterUser = await _twitterUserDal.GetTwitterUserAsync(twitterUsername); + } + + // Update Follower + var twitterUserId = twitterUser.Id; + if(!follower.Followings.Contains(twitterUserId)) + follower.Followings.Add(twitterUserId); + + if(!follower.FollowingsSyncStatus.ContainsKey(twitterUserId)) + follower.FollowingsSyncStatus.Add(twitterUserId, -1); + + follower.FollowingsSyncStatus[twitterUserId] = -1; + + // Save Follower + await _followerDal.UpdateFollowerAsync(follower); + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/BusinessUseCases/ProcessUnfollowUser.cs b/src/BirdsiteLive.Domain/BusinessUseCases/ProcessUnfollowUser.cs new file mode 100644 index 0000000..fe4035f --- /dev/null +++ b/src/BirdsiteLive.Domain/BusinessUseCases/ProcessUnfollowUser.cs @@ -0,0 +1,7 @@ +namespace BirdsiteLive.Domain.BusinessUseCases +{ + public class ProcessUnfollowUser + { + + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index d084e97..c6ead30 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using BirdsiteLive.ActivityPub; using BirdsiteLive.Common.Settings; using BirdsiteLive.Cryptography; +using BirdsiteLive.Domain.BusinessUseCases; using BirdsiteLive.Twitter.Models; using Tweetinvi.Core.Exceptions; using Tweetinvi.Models; @@ -23,15 +24,18 @@ namespace BirdsiteLive.Domain public class UserService : IUserService { + private readonly IProcessFollowUser _processFollowUser; + private readonly ICryptoService _cryptoService; private readonly IActivityPubService _activityPubService; private readonly string _host; #region Ctor - public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService) + public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser) { _cryptoService = cryptoService; _activityPubService = activityPubService; + _processFollowUser = processFollowUser; _host = $"https://{instanceSettings.Domain.Replace("https://",string.Empty).Replace("http://", string.Empty).TrimEnd('/')}"; } #endregion @@ -100,15 +104,21 @@ namespace BirdsiteLive.Domain return note; } - public async Task FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityFollow activity) + public async Task FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityFollow activity) { // Validate - if (!await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders)) return false; + var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders); + if (!sigValidation.SignatureIsValidated) return false; // Save Follow in DB - - // Send Accept Activity - var targetHost = activity.actor.Replace("https://", string.Empty).Split('/').First(); + var followerUserName = sigValidation.User.name.ToLowerInvariant(); + var followerHost = sigValidation.User.url.Replace("https://", string.Empty).Split('/').First(); + var followerInbox = sigValidation.User.inbox; + var twitterUser = activity.apObject.Split('/').Last().Replace("@", string.Empty); + await _processFollowUser.ExecuteAsync(followerUserName, followerHost, followerInbox, twitterUser); + + // Send Accept Activity + //var followerHost = activity.actor.Replace("https://", string.Empty).Split('/').First(); var acceptFollow = new ActivityAcceptFollow() { context = "https://www.w3.org/ns/activitystreams", @@ -123,11 +133,11 @@ namespace BirdsiteLive.Domain apObject = activity.apObject } }; - var result = await _activityPubService.PostDataAsync(acceptFollow, targetHost, activity.apObject); + var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject); return result == HttpStatusCode.Accepted; } - private async Task ValidateSignature(string actor, string rawSig, string method, string path, string queryString, Dictionary requestHeaders) + private async Task ValidateSignature(string actor, string rawSig, string method, string path, string queryString, Dictionary requestHeaders) { var signatures = rawSig.Split(','); var signature_header = new Dictionary(); @@ -184,7 +194,17 @@ namespace BirdsiteLive.Domain var result = signKey.VerifyData(Encoding.UTF8.GetBytes(toSign.ToString()), sig, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); - return result; + return new SignatureValidationResult() + { + SignatureIsValidated = result, + User = remoteUser + }; } } + + public class SignatureValidationResult + { + public bool SignatureIsValidated { get; set; } + public Actor User { get; set; } + } } diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index 332831e..5757c99 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -17,6 +17,7 @@ + diff --git a/src/BirdsiteLive/Controllers/WellKnownController.cs b/src/BirdsiteLive/Controllers/WellKnownController.cs index 3fce212..870801b 100644 --- a/src/BirdsiteLive/Controllers/WellKnownController.cs +++ b/src/BirdsiteLive/Controllers/WellKnownController.cs @@ -18,10 +18,10 @@ namespace BirdsiteLive.Controllers private readonly InstanceSettings _settings; #region Ctor - public WellKnownController(IOptions settings, ITwitterService twitterService) + public WellKnownController(InstanceSettings settings, ITwitterService twitterService) { _twitterService = twitterService; - _settings = settings.Value; + _settings = settings; } #endregion diff --git a/src/BirdsiteLive/Services/FederationService.cs b/src/BirdsiteLive/Services/FederationService.cs index ee07161..62f862b 100644 --- a/src/BirdsiteLive/Services/FederationService.cs +++ b/src/BirdsiteLive/Services/FederationService.cs @@ -1,6 +1,7 @@ using System; using System.Threading; using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; using BirdsiteLive.Domain; using Microsoft.Extensions.Hosting; @@ -8,22 +9,41 @@ namespace BirdsiteLive.Services { public class FederationService : BackgroundService { + private readonly IDbInitializerDal _dbInitializerDal; private readonly IUserService _userService; #region Ctor - public FederationService(IUserService userService) + public FederationService(IDbInitializerDal dbInitializerDal, IUserService userService) { + _dbInitializerDal = dbInitializerDal; _userService = userService; } #endregion protected override async Task ExecuteAsync(CancellationToken stoppingToken) { + await DbInitAsync(); + for (;;) { Console.WriteLine("RUNNING SERVICE"); await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken); } } + + private async Task DbInitAsync() + { + var currentVersion = await _dbInitializerDal.GetCurrentDbVersionAsync(); + var mandatoryVersion = _dbInitializerDal.GetMandatoryDbVersion(); + + if (currentVersion == null) + { + await _dbInitializerDal.InitDbAsync(); + } + else if (currentVersion != mandatoryVersion) + { + throw new NotImplementedException(); + } + } } } \ No newline at end of file diff --git a/src/BirdsiteLive/Startup.cs b/src/BirdsiteLive/Startup.cs index 6d07aaa..3eaecd8 100644 --- a/src/BirdsiteLive/Startup.cs +++ b/src/BirdsiteLive/Startup.cs @@ -3,6 +3,9 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using BirdsiteLive.Common.Settings; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Postgres.DataAccessLayers; +using BirdsiteLive.DAL.Postgres.Settings; using BirdsiteLive.Models; using Lamar; using Microsoft.AspNetCore.Builder; @@ -34,7 +37,7 @@ namespace BirdsiteLive // This method gets called by the runtime. Use this method to add services to the container. public void ConfigureServices(IServiceCollection services) { - services.Configure(Configuration.GetSection("Instance")); + //services.Configure(Configuration.GetSection("Instance")); //services.Configure(Configuration.GetSection("Twitter")); services.AddControllersWithViews(); @@ -48,15 +51,27 @@ namespace BirdsiteLive var instanceSettings = Configuration.GetSection("Instance").Get(); services.For().Use(x => instanceSettings); + var postgresSettings = new PostgresSettings + { + ConnString = instanceSettings.PostgresConnString + }; + services.For().Use(x => postgresSettings); + + services.For().Use().Singleton(); + services.For().Use().Singleton(); + services.For().Use().Singleton(); + services.Scan(_ => { _.Assembly("BirdsiteLive.Twitter"); _.Assembly("BirdsiteLive.Domain"); + _.Assembly("BirdsiteLive.DAL"); + _.Assembly("BirdsiteLive.DAL.Postgres"); _.TheCallingAssembly(); //_.AssemblyContainingType(); //_.Exclude(type => type.Name.Contains("Settings")); - + _.WithDefaultConventions(); _.LookForRegistries(); diff --git a/src/BirdsiteLive/appsettings.json b/src/BirdsiteLive/appsettings.json index 6de0ca3..5054e67 100644 --- a/src/BirdsiteLive/appsettings.json +++ b/src/BirdsiteLive/appsettings.json @@ -8,7 +8,8 @@ }, "AllowedHosts": "*", "Instance": { - "Domain": "domain.name" + "Domain": "domain.name", + "PostgresConnString": "Host=127.0.0.1;Username=username;Password=password;Database=mydb" }, "Twitter": { "ConsumerKey": "twitter.api.key", diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs index fdfd7e7..0ec78dc 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs @@ -20,8 +20,11 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } #endregion - public async Task CreateFollowerAsync(string acct, string host, int[] followings, Dictionary followingSyncStatus, string inboxUrl) + public async Task CreateFollowerAsync(string acct, string host, string inboxUrl, int[] followings = null, Dictionary followingSyncStatus = null) { + if(followings == null) followings = new int[0]; + if(followingSyncStatus == null) followingSyncStatus = new Dictionary(); + var serializedDic = JsonConvert.SerializeObject(followingSyncStatus); acct = acct.ToLowerInvariant(); @@ -68,18 +71,19 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } } - public async Task UpdateFollowerAsync(int id, int[] followings, Dictionary followingsSyncStatus) + public async Task UpdateFollowerAsync(Follower follower) { - if (id == default) throw new ArgumentException("id"); + if (follower == default) throw new ArgumentException("follower"); + if (follower.Id == default) throw new ArgumentException("id"); - var serializedDic = JsonConvert.SerializeObject(followingsSyncStatus); + var serializedDic = JsonConvert.SerializeObject(follower.FollowingsSyncStatus); var query = $"UPDATE {_settings.FollowersTableName} SET followings = @followings, followingsSyncStatus = CAST(@followingsSyncStatus as json) WHERE id = @id"; using (var dbConnection = Connection) { dbConnection.Open(); - await dbConnection.QueryAsync(query, new { id, followings, followingsSyncStatus = serializedDic }); + await dbConnection.QueryAsync(query, new { follower.Id, follower.Followings, followingsSyncStatus = serializedDic }); } } @@ -125,7 +129,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers Acct = follower.Acct, Host = follower.Host, InboxUrl = follower.InboxUrl, - Followings = follower.Followings, + Followings = follower.Followings.ToList(), FollowingsSyncStatus = JsonConvert.DeserializeObject>(follower.FollowingsSyncStatus) }; } diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Settings/PostgresSettings.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Settings/PostgresSettings.cs index 8037a42..c7504ef 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Settings/PostgresSettings.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Settings/PostgresSettings.cs @@ -4,9 +4,9 @@ { public string ConnString { get; set; } - public string DbVersionTableName { get; set; } = "db-version"; - public string TwitterUserTableName { get; set; } = "twitter-users"; + public string DbVersionTableName { get; set; } = "db_version"; + public string TwitterUserTableName { get; set; } = "twitter_users"; public string FollowersTableName { get; set; } = "followers"; - public string CachedTweetsTableName { get; set; } = "cached-tweets"; + public string CachedTweetsTableName { get; set; } = "cached_tweets"; } } \ No newline at end of file diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs index 19b3dbc..f7108a0 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs @@ -7,9 +7,10 @@ namespace BirdsiteLive.DAL.Contracts public interface IFollowersDal { Task GetFollowerAsync(string acct, string host); - Task CreateFollowerAsync(string acct, string host, int[] followings, Dictionary followingSyncStatus, string inboxUrl); + Task CreateFollowerAsync(string acct, string host, string inboxUrl, int[] followings = null, + Dictionary followingSyncStatus = null); Task GetFollowersAsync(int followedUserId); - Task UpdateFollowerAsync(int id, int[] followings, Dictionary followingSyncStatus); + Task UpdateFollowerAsync(Follower follower); Task DeleteFollowerAsync(int id); Task DeleteFollowerAsync(string acct, string host); } diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Models/Follower.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Models/Follower.cs index 32ee22b..2499263 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Models/Follower.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Models/Follower.cs @@ -6,7 +6,7 @@ namespace BirdsiteLive.DAL.Models { public int Id { get; set; } - public int[] Followings { get; set; } + public List Followings { get; set; } public Dictionary FollowingsSyncStatus { get; set; } public string Acct { get; set; } diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs index fed68ed..cdb0dc0 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs @@ -48,7 +48,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers var inboxUrl = "https://domain.ext/myhandle/inbox"; var dal = new FollowersPostgresDal(_settings); - await dal.CreateFollowerAsync(acct, host, following, followingSync, inboxUrl); + await dal.CreateFollowerAsync(acct, host, inboxUrl, following, followingSync); var result = await dal.GetFollowerAsync(acct, host); @@ -56,7 +56,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers Assert.AreEqual(acct, result.Acct); Assert.AreEqual(host, result.Host); Assert.AreEqual(inboxUrl, result.InboxUrl); - Assert.AreEqual(following.Length, result.Followings.Length); + Assert.AreEqual(following.Length, result.Followings.Count); Assert.AreEqual(following[0], result.Followings[0]); Assert.AreEqual(followingSync.Count, result.FollowingsSyncStatus.Count); Assert.AreEqual(followingSync.First().Key, result.FollowingsSyncStatus.First().Key); @@ -74,21 +74,21 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers var following = new[] { 1,2,3 }; var followingSync = new Dictionary(); var inboxUrl = "https://domain.ext/myhandle1/inbox"; - await dal.CreateFollowerAsync(acct, host, following, followingSync, inboxUrl); + await dal.CreateFollowerAsync(acct, host, inboxUrl, following, followingSync); //User 2 acct = "myhandle2"; host = "domain.ext"; following = new[] { 2, 4, 5 }; inboxUrl = "https://domain.ext/myhandle2/inbox"; - await dal.CreateFollowerAsync(acct, host, following, followingSync, inboxUrl); + await dal.CreateFollowerAsync(acct, host, inboxUrl, following, followingSync); //User 2 acct = "myhandle3"; host = "domain.ext"; following = new[] { 1 }; inboxUrl = "https://domain.ext/myhandle3/inbox"; - await dal.CreateFollowerAsync(acct, host, following, followingSync, inboxUrl); + await dal.CreateFollowerAsync(acct, host, inboxUrl, following, followingSync); var result = await dal.GetFollowersAsync(2); Assert.AreEqual(2, result.Length); @@ -115,22 +115,24 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers var inboxUrl = "https://domain.ext/myhandle/inbox"; var dal = new FollowersPostgresDal(_settings); - await dal.CreateFollowerAsync(acct, host, following, followingSync, inboxUrl); + await dal.CreateFollowerAsync(acct, host, inboxUrl, following, followingSync); var result = await dal.GetFollowerAsync(acct, host); - var updatedFollowing = new[] { 12, 19, 23, 24 }; - var updatedFollowingSync = new Dictionary() - { + var updatedFollowing = new List { 12, 19, 23, 24 }; + var updatedFollowingSync = new Dictionary(){ {12, 170L}, {19, 171L}, {23, 172L}, {24, 173L} }; + result.Followings = updatedFollowing.ToList(); + result.FollowingsSyncStatus = updatedFollowingSync; + - await dal.UpdateFollowerAsync(result.Id, updatedFollowing, updatedFollowingSync); + await dal.UpdateFollowerAsync(result); result = await dal.GetFollowerAsync(acct, host); - Assert.AreEqual(updatedFollowing.Length, result.Followings.Length); + Assert.AreEqual(updatedFollowing.Count, result.Followings.Count); Assert.AreEqual(updatedFollowing[0], result.Followings[0]); Assert.AreEqual(updatedFollowingSync.Count, result.FollowingsSyncStatus.Count); Assert.AreEqual(updatedFollowingSync.First().Key, result.FollowingsSyncStatus.First().Key); @@ -152,7 +154,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers var inboxUrl = "https://domain.ext/myhandle/inbox"; var dal = new FollowersPostgresDal(_settings); - await dal.CreateFollowerAsync(acct, host, following, followingSync, inboxUrl); + await dal.CreateFollowerAsync(acct, host, inboxUrl, following, followingSync); var result = await dal.GetFollowerAsync(acct, host); var updatedFollowing = new[] { 12, 19 }; @@ -161,11 +163,13 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers {12, 170L}, {19, 171L} }; + result.Followings = updatedFollowing.ToList(); + result.FollowingsSyncStatus = updatedFollowingSync; - await dal.UpdateFollowerAsync(result.Id, updatedFollowing, updatedFollowingSync); + await dal.UpdateFollowerAsync(result); result = await dal.GetFollowerAsync(acct, host); - Assert.AreEqual(updatedFollowing.Length, result.Followings.Length); + Assert.AreEqual(updatedFollowing.Length, result.Followings.Count); Assert.AreEqual(updatedFollowing[0], result.Followings[0]); Assert.AreEqual(updatedFollowingSync.Count, result.FollowingsSyncStatus.Count); Assert.AreEqual(updatedFollowingSync.First().Key, result.FollowingsSyncStatus.First().Key); @@ -187,7 +191,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers var inboxUrl = "https://domain.ext/myhandle/inbox"; var dal = new FollowersPostgresDal(_settings); - await dal.CreateFollowerAsync(acct, host, following, followingSync, inboxUrl); + await dal.CreateFollowerAsync(acct, host, inboxUrl, following, followingSync); var result = await dal.GetFollowerAsync(acct, host); Assert.IsNotNull(result); @@ -212,7 +216,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers var inboxUrl = "https://domain.ext/myhandle/inbox"; var dal = new FollowersPostgresDal(_settings); - await dal.CreateFollowerAsync(acct, host, following, followingSync, inboxUrl); + await dal.CreateFollowerAsync(acct, host, inboxUrl, following, followingSync); var result = await dal.GetFollowerAsync(acct, host); Assert.IsNotNull(result); From e52f2b8c7379e92cd490f7a059c67ee8fb32a800 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 8 Jul 2020 18:41:21 -0400 Subject: [PATCH 07/67] fix unsupported message --- src/BirdsiteLive/Controllers/UsersController.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index 38f87ee..2430cae 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -81,7 +81,7 @@ namespace BirdsiteLive.Controllers var activity = ApDeserializer.ProcessActivity(body); // Do something - switch (activity.type) + switch (activity?.type) { case "Follow": var succeeded = await _userService.FollowRequestedAsync(r.Headers["Signature"].First(), r.Method, r.Path, r.QueryString.ToString(), RequestHeaders(r.Headers), activity as ActivityFollow); @@ -95,7 +95,7 @@ namespace BirdsiteLive.Controllers } } - return Ok(); + return Accepted(); } private Dictionary RequestHeaders(IHeaderDictionary header) From a23271613d6a7e44589bfa7ecc0c0d4af23bd588 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 8 Jul 2020 18:41:47 -0400 Subject: [PATCH 08/67] add admin email to nodeinfo metadata --- src/BirdsiteLive.Common/Settings/InstanceSettings.cs | 2 +- src/BirdsiteLive/Controllers/WellKnownController.cs | 8 ++++++++ src/BirdsiteLive/Models/WellKnownModels/Metadata.cs | 7 +++++++ src/BirdsiteLive/Models/WellKnownModels/NodeInfoV20.cs | 6 ++++-- src/BirdsiteLive/Models/WellKnownModels/NodeInfoV21.cs | 2 +- src/BirdsiteLive/appsettings.json | 1 + 6 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 src/BirdsiteLive/Models/WellKnownModels/Metadata.cs diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs index c0fff2e..ce200d9 100644 --- a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs +++ b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs @@ -3,7 +3,7 @@ public class InstanceSettings { public string Domain { get; set; } - + public string AdminEmail { get; set; } public string PostgresConnString { get; set; } } } \ No newline at end of file diff --git a/src/BirdsiteLive/Controllers/WellKnownController.cs b/src/BirdsiteLive/Controllers/WellKnownController.cs index 870801b..30a6f22 100644 --- a/src/BirdsiteLive/Controllers/WellKnownController.cs +++ b/src/BirdsiteLive/Controllers/WellKnownController.cs @@ -77,6 +77,10 @@ namespace BirdsiteLive.Controllers { inbound = new object[0], outbound = new object[0] + }, + metadata = new Metadata() + { + email = _settings.AdminEmail } }; return new JsonResult(nodeInfo); @@ -109,6 +113,10 @@ namespace BirdsiteLive.Controllers { inbound = new object[0], outbound = new object[0] + }, + metadata = new Metadata() + { + email = _settings.AdminEmail } }; return new JsonResult(nodeInfo); diff --git a/src/BirdsiteLive/Models/WellKnownModels/Metadata.cs b/src/BirdsiteLive/Models/WellKnownModels/Metadata.cs new file mode 100644 index 0000000..9f5007e --- /dev/null +++ b/src/BirdsiteLive/Models/WellKnownModels/Metadata.cs @@ -0,0 +1,7 @@ +namespace BirdsiteLive.Models.WellKnownModels +{ + public class Metadata + { + public string email { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Models/WellKnownModels/NodeInfoV20.cs b/src/BirdsiteLive/Models/WellKnownModels/NodeInfoV20.cs index f2c1f92..032fc51 100644 --- a/src/BirdsiteLive/Models/WellKnownModels/NodeInfoV20.cs +++ b/src/BirdsiteLive/Models/WellKnownModels/NodeInfoV20.cs @@ -1,4 +1,6 @@ -namespace BirdsiteLive.Models.WellKnownModels +using System.ComponentModel.DataAnnotations; + +namespace BirdsiteLive.Models.WellKnownModels { public class NodeInfoV20 { @@ -8,6 +10,6 @@ public Usage usage { get; set; } public bool openRegistrations { get; set; } public Services services { get; set; } - //public object metadata { get; set; } + public Metadata metadata { get; set; } } } \ No newline at end of file diff --git a/src/BirdsiteLive/Models/WellKnownModels/NodeInfoV21.cs b/src/BirdsiteLive/Models/WellKnownModels/NodeInfoV21.cs index ba50f5b..ce397cb 100644 --- a/src/BirdsiteLive/Models/WellKnownModels/NodeInfoV21.cs +++ b/src/BirdsiteLive/Models/WellKnownModels/NodeInfoV21.cs @@ -8,6 +8,6 @@ public bool openRegistrations { get; set; } public SoftwareV21 software { get; set; } public Services services { get; set; } - //public object metadata { get; set; } + public Metadata metadata { get; set; } } } \ No newline at end of file diff --git a/src/BirdsiteLive/appsettings.json b/src/BirdsiteLive/appsettings.json index 5054e67..08c587a 100644 --- a/src/BirdsiteLive/appsettings.json +++ b/src/BirdsiteLive/appsettings.json @@ -9,6 +9,7 @@ "AllowedHosts": "*", "Instance": { "Domain": "domain.name", + "AdminEmail": "me@domain.name", "PostgresConnString": "Host=127.0.0.1;Username=username;Password=password;Database=mydb" }, "Twitter": { From 60eb4727527713b4030f5409cac3997c59fb3a94 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 8 Jul 2020 19:50:31 -0400 Subject: [PATCH 09/67] remove reinitialization in case of refollowing --- src/BirdsiteLive.Domain/BusinessUseCases/ProcessFollowUser.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/BirdsiteLive.Domain/BusinessUseCases/ProcessFollowUser.cs b/src/BirdsiteLive.Domain/BusinessUseCases/ProcessFollowUser.cs index c09c696..ce50673 100644 --- a/src/BirdsiteLive.Domain/BusinessUseCases/ProcessFollowUser.cs +++ b/src/BirdsiteLive.Domain/BusinessUseCases/ProcessFollowUser.cs @@ -45,9 +45,7 @@ namespace BirdsiteLive.Domain.BusinessUseCases if(!follower.FollowingsSyncStatus.ContainsKey(twitterUserId)) follower.FollowingsSyncStatus.Add(twitterUserId, -1); - - follower.FollowingsSyncStatus[twitterUserId] = -1; - + // Save Follower await _followerDal.UpdateFollowerAsync(follower); } From 387285e64586465b1673a0c73b061fcf042fe535 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 8 Jul 2020 19:50:58 -0400 Subject: [PATCH 10/67] added undo follow workflow --- .../Models/ActivityAcceptUndoFollow.cs | 10 +++++ .../BusinessUseCases/ProcessUnfollowUser.cs | 44 ++++++++++++++++-- src/BirdsiteLive.Domain/UserService.cs | 45 +++++++++++++++++-- .../Controllers/UsersController.cs | 22 ++++++--- 4 files changed, 108 insertions(+), 13 deletions(-) create mode 100644 src/BirdsiteLive.ActivityPub/Models/ActivityAcceptUndoFollow.cs diff --git a/src/BirdsiteLive.ActivityPub/Models/ActivityAcceptUndoFollow.cs b/src/BirdsiteLive.ActivityPub/Models/ActivityAcceptUndoFollow.cs new file mode 100644 index 0000000..9453f25 --- /dev/null +++ b/src/BirdsiteLive.ActivityPub/Models/ActivityAcceptUndoFollow.cs @@ -0,0 +1,10 @@ +using Newtonsoft.Json; + +namespace BirdsiteLive.ActivityPub +{ + public class ActivityAcceptUndoFollow : Activity + { + [JsonProperty("object")] + public ActivityUndoFollow apObject { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/BusinessUseCases/ProcessUnfollowUser.cs b/src/BirdsiteLive.Domain/BusinessUseCases/ProcessUnfollowUser.cs index fe4035f..4d5483a 100644 --- a/src/BirdsiteLive.Domain/BusinessUseCases/ProcessUnfollowUser.cs +++ b/src/BirdsiteLive.Domain/BusinessUseCases/ProcessUnfollowUser.cs @@ -1,7 +1,45 @@ -namespace BirdsiteLive.Domain.BusinessUseCases +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; + +namespace BirdsiteLive.Domain.BusinessUseCases { - public class ProcessUnfollowUser + public interface IProcessUndoFollowUser { - + Task ExecuteAsync(string followerUsername, string followerDomain, string twitterUsername); + } + + public class ProcessUndoFollowUser : IProcessUndoFollowUser + { + private readonly IFollowersDal _followerDal; + private readonly ITwitterUserDal _twitterUserDal; + + #region Ctor + public ProcessUndoFollowUser(IFollowersDal followerDal, ITwitterUserDal twitterUserDal) + { + _followerDal = followerDal; + _twitterUserDal = twitterUserDal; + } + #endregion + + public async Task ExecuteAsync(string followerUsername, string followerDomain, string twitterUsername) + { + // Get Follower and Twitter Users + var follower = await _followerDal.GetFollowerAsync(followerUsername, followerDomain); + if (follower == null) return; + + var twitterUser = await _twitterUserDal.GetTwitterUserAsync(twitterUsername); + if (twitterUser == null) return; + + // Update Follower + var twitterUserId = twitterUser.Id; + if (follower.Followings.Contains(twitterUserId)) + follower.Followings.Remove(twitterUserId); + + if (follower.FollowingsSyncStatus.ContainsKey(twitterUserId)) + follower.FollowingsSyncStatus.Remove(twitterUserId); + + // Save Follower + await _followerDal.UpdateFollowerAsync(follower); + } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index c6ead30..fe9b627 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -18,24 +18,27 @@ namespace BirdsiteLive.Domain public interface IUserService { Actor GetUser(TwitterUser twitterUser); - Task FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityFollow activity); Note GetStatus(TwitterUser user, ITweet tweet); + Task FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityFollow activity); + Task UndoFollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityUndoFollow activity); } public class UserService : IUserService { private readonly IProcessFollowUser _processFollowUser; + private readonly IProcessUndoFollowUser _processUndoFollowUser; private readonly ICryptoService _cryptoService; private readonly IActivityPubService _activityPubService; private readonly string _host; #region Ctor - public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser) + public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser) { _cryptoService = cryptoService; _activityPubService = activityPubService; _processFollowUser = processFollowUser; + _processUndoFollowUser = processUndoFollowUser; _host = $"https://{instanceSettings.Domain.Replace("https://",string.Empty).Replace("http://", string.Empty).TrimEnd('/')}"; } #endregion @@ -104,6 +107,8 @@ namespace BirdsiteLive.Domain return note; } + + public async Task FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityFollow activity) { // Validate @@ -118,7 +123,6 @@ namespace BirdsiteLive.Domain await _processFollowUser.ExecuteAsync(followerUserName, followerHost, followerInbox, twitterUser); // Send Accept Activity - //var followerHost = activity.actor.Replace("https://", string.Empty).Split('/').First(); var acceptFollow = new ActivityAcceptFollow() { context = "https://www.w3.org/ns/activitystreams", @@ -136,7 +140,40 @@ namespace BirdsiteLive.Domain var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject); return result == HttpStatusCode.Accepted; } - + + public async Task UndoFollowRequestedAsync(string signature, string method, string path, string queryString, + Dictionary requestHeaders, ActivityUndoFollow activity) + { + // Validate + var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders); + if (!sigValidation.SignatureIsValidated) return false; + + // Save Follow in DB + var followerUserName = sigValidation.User.name.ToLowerInvariant(); + var followerHost = sigValidation.User.url.Replace("https://", string.Empty).Split('/').First(); + //var followerInbox = sigValidation.User.inbox; + var twitterUser = activity.apObject.apObject.Split('/').Last().Replace("@", string.Empty); + await _processUndoFollowUser.ExecuteAsync(followerUserName, followerHost, twitterUser); + + // Send Accept Activity + var acceptFollow = new ActivityAcceptUndoFollow() + { + context = "https://www.w3.org/ns/activitystreams", + id = $"{activity.apObject.apObject}#accepts/undofollows/{Guid.NewGuid()}", + type = "Accept", + actor = activity.apObject.apObject, + apObject = new ActivityUndoFollow() + { + id = activity.id, + type = activity.type, + actor = activity.actor, + apObject = activity.apObject + } + }; + var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject.apObject); + return result == HttpStatusCode.Accepted; + } + private async Task ValidateSignature(string actor, string rawSig, string method, string path, string queryString, Dictionary requestHeaders) { var signatures = rawSig.Split(','); diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index 2430cae..5c756d4 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -54,9 +54,9 @@ namespace BirdsiteLive.Controllers { if (!long.TryParse(statusId, out var parsedStatusId)) return NotFound(); - + var tweet = _twitterService.GetTweet(parsedStatusId); - if(tweet == null) + if (tweet == null) return NotFound(); var user = _twitterService.GetUser(id); @@ -80,15 +80,25 @@ namespace BirdsiteLive.Controllers var body = await reader.ReadToEndAsync(); var activity = ApDeserializer.ProcessActivity(body); // Do something + var signature = r.Headers["Signature"].First(); switch (activity?.type) { case "Follow": - var succeeded = await _userService.FollowRequestedAsync(r.Headers["Signature"].First(), r.Method, r.Path, r.QueryString.ToString(), RequestHeaders(r.Headers), activity as ActivityFollow); - if (succeeded) return Accepted(); - else return Unauthorized(); - break; + { + var succeeded = await _userService.FollowRequestedAsync(signature, r.Method, r.Path, + r.QueryString.ToString(), RequestHeaders(r.Headers), activity as ActivityFollow); + if (succeeded) return Accepted(); + else return Unauthorized(); + } case "Undo": + if (activity is ActivityUndoFollow) + { + var succeeded = await _userService.UndoFollowRequestedAsync(signature, r.Method, r.Path, + r.QueryString.ToString(), RequestHeaders(r.Headers), activity as ActivityUndoFollow); + if (succeeded) return Accepted(); + else return Unauthorized(); + } return Accepted(); default: return Accepted(); From c03d5901f83d919d0c8f56367904ae4faf4e8868 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Thu, 16 Jul 2020 01:18:14 -0400 Subject: [PATCH 11/67] added debug infos --- src/BirdsiteLive/Controllers/UsersController.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index 5c756d4..fcc44df 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -82,6 +82,9 @@ namespace BirdsiteLive.Controllers // Do something var signature = r.Headers["Signature"].First(); + Console.WriteLine(body); + Console.WriteLine(); + switch (activity?.type) { case "Follow": From d13f60ec3cfaabf5552c3099e4964d75762d425c Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Thu, 16 Jul 2020 01:19:41 -0400 Subject: [PATCH 12/67] init pipeline --- .../BirdsiteLive.Pipeline.csproj | 15 ++++++ .../Contracts/IRetrieveFollowersProcessor.cs | 12 +++++ .../Contracts/IRetrieveTweetsProcessor.cs | 12 +++++ .../IRetrieveTwitterAccountsProcessor.cs | 7 +++ .../ISendTweetsToFollowersProcessor.cs | 11 +++++ .../Models/UserWithTweetsToSync.cs | 12 +++++ .../Processors/RetrieveTweetsProcessor.cs | 16 +++++++ .../StatusPublicationPipeline.cs | 47 +++++++++++++++++++ src/BirdsiteLive.sln | 15 ++++-- src/BirdsiteLive/BirdsiteLive.csproj | 1 + .../Services/FederationService.cs | 15 ++---- src/BirdsiteLive/Startup.cs | 1 + 12 files changed, 151 insertions(+), 13 deletions(-) create mode 100644 src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj create mode 100644 src/BirdsiteLive.Pipeline/Contracts/IRetrieveFollowersProcessor.cs create mode 100644 src/BirdsiteLive.Pipeline/Contracts/IRetrieveTweetsProcessor.cs create mode 100644 src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterAccountsProcessor.cs create mode 100644 src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs create mode 100644 src/BirdsiteLive.Pipeline/Models/UserWithTweetsToSync.cs create mode 100644 src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs create mode 100644 src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs diff --git a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj new file mode 100644 index 0000000..5f24b03 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj @@ -0,0 +1,15 @@ + + + + netstandard2.0 + + + + + + + + + + + diff --git a/src/BirdsiteLive.Pipeline/Contracts/IRetrieveFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/IRetrieveFollowersProcessor.cs new file mode 100644 index 0000000..557362f --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Contracts/IRetrieveFollowersProcessor.cs @@ -0,0 +1,12 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.Pipeline.Models; + +namespace BirdsiteLive.Pipeline.Contracts +{ + public interface IRetrieveFollowersProcessor + { + Task> ProcessAsync(UserWithTweetsToSync[] userWithTweetsToSyncs, CancellationToken ct); + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTweetsProcessor.cs new file mode 100644 index 0000000..451f1d1 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTweetsProcessor.cs @@ -0,0 +1,12 @@ +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Pipeline.Models; + +namespace BirdsiteLive.Pipeline.Contracts +{ + public interface IRetrieveTweetsProcessor + { + Task ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct); + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterAccountsProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterAccountsProcessor.cs new file mode 100644 index 0000000..219f74d --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterAccountsProcessor.cs @@ -0,0 +1,7 @@ +namespace BirdsiteLive.Pipeline.Contracts +{ + public interface IRetrieveTwitterAccountsProcessor + { + + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs new file mode 100644 index 0000000..df18fa9 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs @@ -0,0 +1,11 @@ +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.Pipeline.Models; + +namespace BirdsiteLive.Pipeline.Contracts +{ + public interface ISendTweetsToFollowersProcessor + { + Task ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct); + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Models/UserWithTweetsToSync.cs b/src/BirdsiteLive.Pipeline/Models/UserWithTweetsToSync.cs new file mode 100644 index 0000000..133e2a5 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Models/UserWithTweetsToSync.cs @@ -0,0 +1,12 @@ +using BirdsiteLive.DAL.Models; +using Tweetinvi.Models; + +namespace BirdsiteLive.Pipeline.Models +{ + public class UserWithTweetsToSync + { + public SyncTwitterUser User { get; set; } + public ITweet[] Tweets { get; set; } + public Follower[] Followers { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs new file mode 100644 index 0000000..46c658e --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs @@ -0,0 +1,16 @@ +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Models; + +namespace BirdsiteLive.Pipeline.Processors +{ + public class RetrieveTweetsProcessor : IRetrieveTweetsProcessor + { + public Task ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct) + { + throw new System.NotImplementedException(); + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs new file mode 100644 index 0000000..d1ddf17 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs @@ -0,0 +1,47 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Models; + +namespace BirdsiteLive.Pipeline +{ + public interface IStatusPublicationPipeline + { + Task ExecuteAsync(CancellationToken ct); + } + + public class StatusPublicationPipeline : IStatusPublicationPipeline + { + private readonly IRetrieveTwitterAccountsProcessor _retrieveTwitterAccountsProcessor; + private readonly IRetrieveTweetsProcessor _retrieveTweetsProcessor; + private readonly IRetrieveFollowersProcessor _retrieveFollowersProcessor; + private readonly ISendTweetsToFollowersProcessor _sendTweetsToFollowersProcessor; + + #region Ctor + public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor) + { + _retrieveTweetsProcessor = retrieveTweetsProcessor; + } + #endregion + + public async Task ExecuteAsync(CancellationToken ct) + { + // Create blocks + var twitterUsersBufferBlock = new BufferBlock(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct}); + var retrieveTweetsBlock = new TransformBlock(async x => await _retrieveTweetsProcessor.ProcessAsync(x, ct)); + var retrieveTweetsBufferBlock = new BufferBlock(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct }); + var retrieveFollowersBlock = new TransformManyBlock(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct)); + var retrieveFollowersBufferBlock = new BufferBlock(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct }); + var sendTweetsToFollowersBlock = new ActionBlock(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct}); + + // Link pipeline + + // Launch twitter user retriever + + // Wait + } + } +} diff --git a/src/BirdsiteLive.sln b/src/BirdsiteLive.sln index d600aa6..5491d81 100644 --- a/src/BirdsiteLive.sln +++ b/src/BirdsiteLive.sln @@ -25,11 +25,15 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.ActivityPub.Te EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataAccessLayers", "DataAccessLayers", "{CFAB3509-3931-42DB-AC97-4F91FC2D849C}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdsiteLive.DAL", "DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj", "{47058CAB-DC43-4DD1-8F68-D3D625332905}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.DAL", "DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj", "{47058CAB-DC43-4DD1-8F68-D3D625332905}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdsiteLive.DAL.Postgres", "DataAccessLayers\BirdsiteLive.DAL.Postgres\BirdsiteLive.DAL.Postgres.csproj", "{87E46519-BBF2-437C-8A5B-CF6CDE7CDAA6}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.DAL.Postgres", "DataAccessLayers\BirdsiteLive.DAL.Postgres\BirdsiteLive.DAL.Postgres.csproj", "{87E46519-BBF2-437C-8A5B-CF6CDE7CDAA6}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdsiteLive.DAL.Postgres.Tests", "Tests\BirdsiteLive.DAL.Postgres.Tests\BirdsiteLive.DAL.Postgres.Tests.csproj", "{CD9489BF-69C8-4705-8774-81C45F4F8FE1}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.DAL.Postgres.Tests", "Tests\BirdsiteLive.DAL.Postgres.Tests\BirdsiteLive.DAL.Postgres.Tests.csproj", "{CD9489BF-69C8-4705-8774-81C45F4F8FE1}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Pipeline", "Pipeline", "{DA3C160C-4811-4E26-A5AD-42B81FAF2D7C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdsiteLive.Pipeline", "BirdsiteLive.Pipeline\BirdsiteLive.Pipeline.csproj", "{2A8CC30D-D775-47D1-9388-F72A5C32DE2A}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -81,6 +85,10 @@ Global {CD9489BF-69C8-4705-8774-81C45F4F8FE1}.Debug|Any CPU.Build.0 = Debug|Any CPU {CD9489BF-69C8-4705-8774-81C45F4F8FE1}.Release|Any CPU.ActiveCfg = Release|Any CPU {CD9489BF-69C8-4705-8774-81C45F4F8FE1}.Release|Any CPU.Build.0 = Release|Any CPU + {2A8CC30D-D775-47D1-9388-F72A5C32DE2A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2A8CC30D-D775-47D1-9388-F72A5C32DE2A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2A8CC30D-D775-47D1-9388-F72A5C32DE2A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2A8CC30D-D775-47D1-9388-F72A5C32DE2A}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -96,6 +104,7 @@ Global {47058CAB-DC43-4DD1-8F68-D3D625332905} = {CFAB3509-3931-42DB-AC97-4F91FC2D849C} {87E46519-BBF2-437C-8A5B-CF6CDE7CDAA6} = {CFAB3509-3931-42DB-AC97-4F91FC2D849C} {CD9489BF-69C8-4705-8774-81C45F4F8FE1} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94} + {2A8CC30D-D775-47D1-9388-F72A5C32DE2A} = {DA3C160C-4811-4E26-A5AD-42B81FAF2D7C} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {69E8DCAD-4C37-4010-858F-5F94E6FBABCE} diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index 5757c99..2195422 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -16,6 +16,7 @@ + diff --git a/src/BirdsiteLive/Services/FederationService.cs b/src/BirdsiteLive/Services/FederationService.cs index 62f862b..f2c2e94 100644 --- a/src/BirdsiteLive/Services/FederationService.cs +++ b/src/BirdsiteLive/Services/FederationService.cs @@ -2,7 +2,7 @@ using System.Threading; using System.Threading.Tasks; using BirdsiteLive.DAL.Contracts; -using BirdsiteLive.Domain; +using BirdsiteLive.Pipeline; using Microsoft.Extensions.Hosting; namespace BirdsiteLive.Services @@ -10,25 +10,20 @@ namespace BirdsiteLive.Services public class FederationService : BackgroundService { private readonly IDbInitializerDal _dbInitializerDal; - private readonly IUserService _userService; + private readonly IStatusPublicationPipeline _statusPublicationPipeline; #region Ctor - public FederationService(IDbInitializerDal dbInitializerDal, IUserService userService) + public FederationService(IDbInitializerDal dbInitializerDal, IStatusPublicationPipeline statusPublicationPipeline) { _dbInitializerDal = dbInitializerDal; - _userService = userService; + _statusPublicationPipeline = statusPublicationPipeline; } #endregion protected override async Task ExecuteAsync(CancellationToken stoppingToken) { await DbInitAsync(); - - for (;;) - { - Console.WriteLine("RUNNING SERVICE"); - await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken); - } + await _statusPublicationPipeline.ExecuteAsync(stoppingToken); } private async Task DbInitAsync() diff --git a/src/BirdsiteLive/Startup.cs b/src/BirdsiteLive/Startup.cs index 3eaecd8..e31945f 100644 --- a/src/BirdsiteLive/Startup.cs +++ b/src/BirdsiteLive/Startup.cs @@ -67,6 +67,7 @@ namespace BirdsiteLive _.Assembly("BirdsiteLive.Domain"); _.Assembly("BirdsiteLive.DAL"); _.Assembly("BirdsiteLive.DAL.Postgres"); + _.Assembly("BirdsiteLive.Pipeline"); _.TheCallingAssembly(); //_.AssemblyContainingType(); From d91ddd420434f86734fda2a94e2ac5a750920102 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 18 Jul 2020 23:35:19 -0400 Subject: [PATCH 13/67] starting pipeline implementation --- .../BirdsiteLive.Pipeline.csproj | 3 + .../Contracts/IRetrieveFollowersProcessor.cs | 1 + .../IRetrieveTwitterAccountsProcessor.cs | 11 +++- .../Processors/RetrieveFollowersProcessor.cs | 33 +++++++++++ .../Processors/RetrieveTweetsProcessor.cs | 55 ++++++++++++++++++- .../RetrieveTwitterAccountsProcessor.cs | 43 +++++++++++++++ .../SendTweetsToFollowersProcessor.cs | 15 +++++ .../StatusPublicationPipeline.cs | 21 ++++++- src/BirdsiteLive.Twitter/TwitterService.cs | 37 ++++++++++++- 9 files changed, 207 insertions(+), 12 deletions(-) create mode 100644 src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs create mode 100644 src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterAccountsProcessor.cs create mode 100644 src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs diff --git a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj index 5f24b03..d1da203 100644 --- a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj +++ b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj @@ -2,13 +2,16 @@ netstandard2.0 + latest + + diff --git a/src/BirdsiteLive.Pipeline/Contracts/IRetrieveFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/IRetrieveFollowersProcessor.cs index 557362f..e0d45dc 100644 --- a/src/BirdsiteLive.Pipeline/Contracts/IRetrieveFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Contracts/IRetrieveFollowersProcessor.cs @@ -8,5 +8,6 @@ namespace BirdsiteLive.Pipeline.Contracts public interface IRetrieveFollowersProcessor { Task> ProcessAsync(UserWithTweetsToSync[] userWithTweetsToSyncs, CancellationToken ct); + //IAsyncEnumerable ProcessAsync(UserWithTweetsToSync[] userWithTweetsToSyncs, CancellationToken ct); } } \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterAccountsProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterAccountsProcessor.cs index 219f74d..b71ae93 100644 --- a/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterAccountsProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterAccountsProcessor.cs @@ -1,7 +1,12 @@ -namespace BirdsiteLive.Pipeline.Contracts +using System.Threading; +using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; +using BirdsiteLive.DAL.Models; + +namespace BirdsiteLive.Pipeline.Contracts { - public interface IRetrieveTwitterAccountsProcessor + public interface IRetrieveTwitterUsersProcessor { - + Task GetTwitterUsersAsync(BufferBlock twitterUsersBufferBlock, CancellationToken ct); } } \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs new file mode 100644 index 0000000..4b2f150 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs @@ -0,0 +1,33 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Models; + +namespace BirdsiteLive.Pipeline.Processors +{ + public class RetrieveFollowersProcessor : IRetrieveFollowersProcessor + { + private readonly IFollowersDal _followersDal; + + #region Ctor + public RetrieveFollowersProcessor(IFollowersDal followersDal) + { + _followersDal = followersDal; + } + #endregion + + public async Task> ProcessAsync(UserWithTweetsToSync[] userWithTweetsToSyncs, CancellationToken ct) + { + //TODO multithread this + foreach (var user in userWithTweetsToSyncs) + { + var followers = await _followersDal.GetFollowersAsync(user.User.Id); + user.Followers = followers; + } + + return userWithTweetsToSyncs; + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs index 46c658e..22416be 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs @@ -1,16 +1,65 @@ -using System.Threading; +using System.Collections.Generic; +using System.Linq; +using System.Threading; using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Contracts; using BirdsiteLive.Pipeline.Models; +using BirdsiteLive.Twitter; +using Tweetinvi.Models; namespace BirdsiteLive.Pipeline.Processors { public class RetrieveTweetsProcessor : IRetrieveTweetsProcessor { - public Task ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct) + private readonly ITwitterService _twitterService; + private readonly ITwitterUserDal _twitterUserDal; + + #region Ctor + public RetrieveTweetsProcessor(ITwitterService twitterService, ITwitterUserDal twitterUserDal) { - throw new System.NotImplementedException(); + _twitterService = twitterService; + _twitterUserDal = twitterUserDal; + } + #endregion + + public async Task ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct) + { + var usersWtTweets = new List(); + + //TODO multithread this + foreach (var user in syncTwitterUsers) + { + var tweets = RetrieveNewTweets(user); + if (tweets.Length > 0 && user.LastTweetPostedId != -1) + { + var userWtTweets = new UserWithTweetsToSync + { + User = user, + Tweets = tweets + }; + usersWtTweets.Add(userWtTweets); + } + else if (tweets.Length > 0 && user.LastTweetPostedId == -1) + { + var tweetId = tweets.First().Id; + await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId); + } + } + + return usersWtTweets.ToArray(); + } + + private ITweet[] RetrieveNewTweets(SyncTwitterUser user) + { + ITweet[] tweets; + if (user.LastTweetPostedId == -1) + tweets = _twitterService.GetTimeline(user.Acct, 1); + else + tweets = _twitterService.GetTimeline(user.Acct, 200, user.LastTweetSynchronizedForAllFollowersId); + + return tweets; } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterAccountsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterAccountsProcessor.cs new file mode 100644 index 0000000..dcc9d6b --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterAccountsProcessor.cs @@ -0,0 +1,43 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Pipeline.Contracts; + +namespace BirdsiteLive.Pipeline.Processors +{ + public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor + { + private readonly ITwitterUserDal _twitterUserDal; + private const int SyncPeriod = 15; //in minutes + + #region Ctor + public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal) + { + _twitterUserDal = twitterUserDal; + } + #endregion + + public async Task GetTwitterUsersAsync(BufferBlock twitterUsersBufferBlock, CancellationToken ct) + { + for (;;) + { + ct.ThrowIfCancellationRequested(); + + try + { + var users = await _twitterUserDal.GetAllTwitterUsersAsync(); + await twitterUsersBufferBlock.SendAsync(users, ct); + } + catch (Exception e) + { + Console.WriteLine(e); + } + + await Task.Delay(SyncPeriod * 1000 * 60, ct); + } + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs new file mode 100644 index 0000000..51bcb39 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs @@ -0,0 +1,15 @@ +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Models; + +namespace BirdsiteLive.Pipeline.Processors +{ + public class SendTweetsToFollowersProcessor : ISendTweetsToFollowersProcessor + { + public Task ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct) + { + throw new System.NotImplementedException(); + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs index d1ddf17..0e0da40 100644 --- a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs +++ b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs @@ -15,15 +15,18 @@ namespace BirdsiteLive.Pipeline public class StatusPublicationPipeline : IStatusPublicationPipeline { - private readonly IRetrieveTwitterAccountsProcessor _retrieveTwitterAccountsProcessor; + private readonly IRetrieveTwitterUsersProcessor _retrieveTwitterAccountsProcessor; private readonly IRetrieveTweetsProcessor _retrieveTweetsProcessor; private readonly IRetrieveFollowersProcessor _retrieveFollowersProcessor; private readonly ISendTweetsToFollowersProcessor _sendTweetsToFollowersProcessor; #region Ctor - public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor) + public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor) { _retrieveTweetsProcessor = retrieveTweetsProcessor; + _retrieveTwitterAccountsProcessor = retrieveTwitterAccountsProcessor; + _retrieveFollowersProcessor = retrieveFollowersProcessor; + _sendTweetsToFollowersProcessor = sendTweetsToFollowersProcessor; } #endregion @@ -36,12 +39,24 @@ namespace BirdsiteLive.Pipeline var retrieveFollowersBlock = new TransformManyBlock(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct)); var retrieveFollowersBufferBlock = new BufferBlock(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct }); var sendTweetsToFollowersBlock = new ActionBlock(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct}); - + // Link pipeline + twitterUsersBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions {PropagateCompletion = true}); + retrieveTweetsBlock.LinkTo(retrieveTweetsBufferBlock); + retrieveTweetsBufferBlock.LinkTo(retrieveFollowersBlock); + retrieveFollowersBlock.LinkTo(retrieveFollowersBufferBlock); + retrieveFollowersBufferBlock.LinkTo(sendTweetsToFollowersBlock); // Launch twitter user retriever + var retrieveTwitterAccountsTask = _retrieveTwitterAccountsProcessor.GetTwitterUsersAsync(twitterUsersBufferBlock, ct); // Wait + await Task.WhenAll(retrieveTwitterAccountsTask, sendTweetsToFollowersBlock.Completion); + + var foreground = Console.ForegroundColor; + Console.ForegroundColor = ConsoleColor.Red; + Console.WriteLine("An error occured, pipeline stopped"); + Console.ForegroundColor = foreground; } } } diff --git a/src/BirdsiteLive.Twitter/TwitterService.cs b/src/BirdsiteLive.Twitter/TwitterService.cs index 3bec34a..4c8878d 100644 --- a/src/BirdsiteLive.Twitter/TwitterService.cs +++ b/src/BirdsiteLive.Twitter/TwitterService.cs @@ -1,9 +1,12 @@ using System; +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using BirdsiteLive.Common.Settings; using BirdsiteLive.Twitter.Models; using Tweetinvi; using Tweetinvi.Models; +using Tweetinvi.Parameters; namespace BirdsiteLive.Twitter { @@ -11,6 +14,7 @@ namespace BirdsiteLive.Twitter { TwitterUser GetUser(string username); ITweet GetTweet(long statusId); + ITweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1); } public class TwitterService : ITwitterService @@ -21,13 +25,13 @@ namespace BirdsiteLive.Twitter public TwitterService(TwitterSettings settings) { _settings = settings; + Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true); } #endregion public TwitterUser GetUser(string username) { - //Auth.SetUserCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, _settings.AccessToken, _settings.AccessTokenSecret); - Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true); + //Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true); var user = User.GetUserFromScreenName(username); if (user == null) return null; @@ -45,9 +49,36 @@ namespace BirdsiteLive.Twitter public ITweet GetTweet(long statusId) { - Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true); + //Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true); var tweet = Tweet.GetTweet(statusId); return tweet; } + + public ITweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1) + { + //Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true); + TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended; + + var user = User.GetUserFromScreenName(username); + var tweets = new List(); + if (fromTweetId == -1) + { + var timeline = Timeline.GetUserTimeline(user.Id, nberTweets); + if (timeline != null) tweets.AddRange(timeline); + } + else + { + var timelineRequestParameters = new UserTimelineParameters + { + SinceId = fromTweetId, + MaximumNumberOfTweetsToRetrieve = nberTweets + }; + var timeline = Timeline.GetUserTimeline(user.Id, timelineRequestParameters); + if (timeline != null) tweets.AddRange(timeline); + } + + return tweets.ToArray(); + //return tweets.Where(x => returnReplies || string.IsNullOrWhiteSpace(x.InReplyToScreenName)).ToArray(); + } } } From baf8cd011c75d26b63539950274a17b35fc63445 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 22 Jul 2020 02:03:52 -0400 Subject: [PATCH 14/67] fix wrong actor usage --- src/BirdsiteLive.Domain/UserService.cs | 15 ++++++++------- src/BirdsiteLive/Controllers/UsersController.cs | 6 +++--- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index fe9b627..1ca60c1 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -18,7 +18,8 @@ namespace BirdsiteLive.Domain public interface IUserService { Actor GetUser(TwitterUser twitterUser); - Note GetStatus(TwitterUser user, ITweet tweet); + //Note GetStatus(TwitterUser user, ITweet tweet); + Note GetStatus(string username, ITweet tweet); Task FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityFollow activity); Task UndoFollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityUndoFollow activity); } @@ -74,15 +75,15 @@ namespace BirdsiteLive.Domain return user; } - public Note GetStatus(TwitterUser user, ITweet tweet) + public Note GetStatus(string username, ITweet tweet) { - var actor = GetUser(user); + //var actor = GetUser(user); - var actorUrl = $"{_host}/users/{user.Acct}"; - var noteId = $"{_host}/users/{user.Acct}/statuses/{tweet.Id}"; - var noteUrl = $"{_host}/@{user.Acct}/{tweet.Id}"; + var actorUrl = $"{_host}/users/{username}"; + var noteId = $"{_host}/users/{username}/statuses/{tweet.Id}"; + var noteUrl = $"{_host}/@{username}/{tweet.Id}"; - var to = $"{actor}/followers"; + var to = $"{actorUrl}/followers"; var apPublic = "https://www.w3.org/ns/activitystreams#Public"; var note = new Note diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index fcc44df..7320eb7 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -59,10 +59,10 @@ namespace BirdsiteLive.Controllers if (tweet == null) return NotFound(); - var user = _twitterService.GetUser(id); - if (user == null) return NotFound(); + //var user = _twitterService.GetUser(id); + //if (user == null) return NotFound(); - var status = _userService.GetStatus(user, tweet); + var status = _userService.GetStatus(id, tweet); var jsonApUser = JsonConvert.SerializeObject(status); return Content(jsonApUser, "application/activity+json; charset=utf-8"); } From 07e5f33613c6ffe8caf136569d196804e271a46d Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 22 Jul 2020 02:11:44 -0400 Subject: [PATCH 15/67] raw iteration of sending notes in pipeline --- src/BirdsiteLive.Domain/ActivityPubService.cs | 56 ++++++++++++++++++- .../BirdsiteLive.Pipeline.csproj | 3 +- .../SendTweetsToFollowersProcessor.cs | 51 ++++++++++++++++- 3 files changed, 105 insertions(+), 5 deletions(-) diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs index d30daf8..9f6e8e5 100644 --- a/src/BirdsiteLive.Domain/ActivityPubService.cs +++ b/src/BirdsiteLive.Domain/ActivityPubService.cs @@ -4,6 +4,7 @@ using System.Net.Http; using System.Text; using System.Threading.Tasks; using BirdsiteLive.ActivityPub; +using BirdsiteLive.Common.Settings; using Newtonsoft.Json; using Org.BouncyCastle.Bcpg; @@ -13,16 +14,20 @@ namespace BirdsiteLive.Domain { Task GetUser(string objectId); Task PostDataAsync(T data, string targetHost, string actorUrl, string inbox = null); + Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, + string targetInbox); } public class ActivityPubService : IActivityPubService { + private readonly InstanceSettings _instanceSettings; private readonly ICryptoService _cryptoService; #region Ctor - public ActivityPubService(ICryptoService cryptoService) + public ActivityPubService(ICryptoService cryptoService, InstanceSettings instanceSettings) { _cryptoService = cryptoService; + _instanceSettings = instanceSettings; } #endregion @@ -37,6 +42,55 @@ namespace BirdsiteLive.Domain } } + public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox) + { + //var username = "gra"; + var actor = $"https://{_instanceSettings.Domain}/users/{username}"; + //var targetHost = "mastodon.technology"; + //var target = $"{targetHost}/users/testtest"; + //var inbox = $"/users/testtest/inbox"; + + //var noteGuid = Guid.NewGuid(); + var noteUri = $"https://{_instanceSettings.Domain}/users/{username}/statuses/{noteId}"; + + //var noteUrl = $"https://{_instanceSettings.Domain}/@{username}/{noteId}"; + //var to = $"{actor}/followers"; + //var apPublic = "https://www.w3.org/ns/activitystreams#Public"; + + var now = DateTime.UtcNow; + var nowString = now.ToString("s") + "Z"; + + var noteActivity = new ActivityCreateNote() + { + context = "https://www.w3.org/ns/activitystreams", + id = $"{noteUri}/activity", + type = "Create", + actor = actor, + published = nowString, + + to = note.to, + cc = note.cc, + apObject = note + //apObject = new Note() + //{ + // id = noteUri, + // summary = null, + // inReplyTo = null, + // published = nowString, + // url = noteUrl, + // attributedTo = actor, + // to = new[] { to }, + // //cc = new [] { apPublic }, + // sensitive = false, + // content = "

Woooot

", + // attachment = new string[0], + // tag = new string[0] + //} + }; + + return await PostDataAsync(noteActivity, targetHost, actor, targetInbox); + } + public async Task PostDataAsync(T data, string targetHost, string actorUrl, string inbox = null) { var usedInbox = $"/inbox"; diff --git a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj index d1da203..89f6d5d 100644 --- a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj +++ b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj @@ -6,11 +6,12 @@ - + + diff --git a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs index 51bcb39..8fa297f 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs @@ -1,15 +1,60 @@ -using System.Threading; +using System.Linq; +using System.Net; +using System.Threading; using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.Domain; using BirdsiteLive.Pipeline.Contracts; using BirdsiteLive.Pipeline.Models; +using BirdsiteLive.Twitter; namespace BirdsiteLive.Pipeline.Processors { public class SendTweetsToFollowersProcessor : ISendTweetsToFollowersProcessor { - public Task ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct) + private readonly IActivityPubService _activityPubService; + private readonly IUserService _userService; + private readonly IFollowersDal _followersDal; + private readonly ITwitterUserDal _twitterUserDal; + + #region Ctor + public SendTweetsToFollowersProcessor(IActivityPubService activityPubService, IUserService userService, IFollowersDal followersDal, ITwitterUserDal twitterUserDal) { - throw new System.NotImplementedException(); + _activityPubService = activityPubService; + _userService = userService; + _followersDal = followersDal; + _twitterUserDal = twitterUserDal; + } + #endregion + + public async Task ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct) + { + var user = userWithTweetsToSync.User; + var userId = user.Id; + + foreach (var follower in userWithTweetsToSync.Followers) + { + var fromStatusId = follower.FollowingsSyncStatus[userId]; + var tweetsToSend = userWithTweetsToSync.Tweets.Where(x => x.Id > fromStatusId).OrderBy(x => x.Id).ToList(); + + var syncStatus = fromStatusId; + foreach (var tweet in tweetsToSend) + { + var note = _userService.GetStatus(user.Acct, tweet); + var result = await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, follower.InboxUrl); + if (result == HttpStatusCode.Accepted) + syncStatus = tweet.Id; + else + break; + } + + follower.FollowingsSyncStatus[userId] = syncStatus; + await _followersDal.UpdateFollowerAsync(follower); + } + + var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max(); + var minimumSync = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).Min(); + await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync); } } } \ No newline at end of file From 1bcb00cdd55a20c441bdebc5adc607b51ad9057d Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 22 Jul 2020 19:25:17 -0400 Subject: [PATCH 16/67] fix routing --- src/BirdsiteLive/Views/Debuging/Index.cshtml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BirdsiteLive/Views/Debuging/Index.cshtml b/src/BirdsiteLive/Views/Debuging/Index.cshtml index 04ea0bf..cb56c56 100644 --- a/src/BirdsiteLive/Views/Debuging/Index.cshtml +++ b/src/BirdsiteLive/Views/Debuging/Index.cshtml @@ -5,14 +5,14 @@

Debug

-
+
-
+ From 10104187d56a7249ac6a8eb55cd4ac649476f846 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 22 Jul 2020 19:27:25 -0400 Subject: [PATCH 17/67] support attachments --- .../Models/Attachment.cs | 9 ++ src/BirdsiteLive.ActivityPub/Models/Note.cs | 2 +- src/BirdsiteLive.Domain/StatusService.cs | 119 ++++++++++++++++++ src/BirdsiteLive.Domain/UserService.cs | 37 ------ .../Contracts/ISaveProgressionProcessor.cs | 11 ++ .../ISendTweetsToFollowersProcessor.cs | 2 +- .../Processors/SaveProgressionProcessor.cs | 29 +++++ .../SendTweetsToFollowersProcessor.cs | 72 +++++++---- .../Controllers/DebugingController.cs | 2 +- .../Controllers/UsersController.cs | 9 +- 10 files changed, 226 insertions(+), 66 deletions(-) create mode 100644 src/BirdsiteLive.ActivityPub/Models/Attachment.cs create mode 100644 src/BirdsiteLive.Domain/StatusService.cs create mode 100644 src/BirdsiteLive.Pipeline/Contracts/ISaveProgressionProcessor.cs create mode 100644 src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs diff --git a/src/BirdsiteLive.ActivityPub/Models/Attachment.cs b/src/BirdsiteLive.ActivityPub/Models/Attachment.cs new file mode 100644 index 0000000..d7b86dd --- /dev/null +++ b/src/BirdsiteLive.ActivityPub/Models/Attachment.cs @@ -0,0 +1,9 @@ +namespace BirdsiteLive.ActivityPub +{ + public class Attachment + { + public string type { get; set; } + public string mediaType { get; set; } + public string url { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.ActivityPub/Models/Note.cs b/src/BirdsiteLive.ActivityPub/Models/Note.cs index cc3d561..b0aba5e 100644 --- a/src/BirdsiteLive.ActivityPub/Models/Note.cs +++ b/src/BirdsiteLive.ActivityPub/Models/Note.cs @@ -24,7 +24,7 @@ namespace BirdsiteLive.ActivityPub //public string conversation { get; set; } public string content { get; set; } //public Dictionary contentMap { get; set; } - public string[] attachment { get; set; } + public Attachment[] attachment { get; set; } public string[] tag { get; set; } //public Dictionary replies; } diff --git a/src/BirdsiteLive.Domain/StatusService.cs b/src/BirdsiteLive.Domain/StatusService.cs new file mode 100644 index 0000000..cc2b688 --- /dev/null +++ b/src/BirdsiteLive.Domain/StatusService.cs @@ -0,0 +1,119 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using BirdsiteLive.ActivityPub; +using BirdsiteLive.Common.Settings; +using Tweetinvi.Models; +using Tweetinvi.Models.Entities; + +namespace BirdsiteLive.Domain +{ + public interface IStatusService + { + Note GetStatus(string username, ITweet tweet); + } + + public class StatusService : IStatusService + { + private readonly InstanceSettings _instanceSettings; + + #region Ctor + public StatusService(InstanceSettings instanceSettings) + { + _instanceSettings = instanceSettings; + } + #endregion + + public Note GetStatus(string username, ITweet tweet) + { + var actorUrl = $"https://{_instanceSettings.Domain}/users/{username}"; + var noteId = $"https://{_instanceSettings.Domain}/users/{username}/statuses/{tweet.Id}"; + var noteUrl = $"https://{_instanceSettings.Domain}/@{username}/{tweet.Id}"; + + var to = $"{actorUrl}/followers"; + var apPublic = "https://www.w3.org/ns/activitystreams#Public"; + + var note = new Note + { + id = $"{noteId}/activity", + + published = tweet.CreatedAt.ToString("s") + "Z", + url = noteUrl, + attributedTo = actorUrl, + + //to = new [] {to}, + //cc = new [] { apPublic }, + + to = new[] { to }, + cc = new[] { apPublic }, + //cc = new string[0], + + sensitive = false, + content = $"

{tweet.Text}

", + attachment = GetAttachments(tweet.Media), + tag = new string[0] + }; + + + return note; + } + + private Attachment[] GetAttachments(List media) + { + var result = new List(); + + foreach (var m in media) + { + var mediaUrl = GetMediaUrl(m); + var mediaType = GetMediaType(m.MediaType, mediaUrl); + if (mediaType == null) continue; + + var att = new Attachment + { + type = "Document", + mediaType = mediaType, + url = mediaUrl + }; + result.Add(att); + } + + return result.ToArray(); + } + + private string GetMediaUrl(IMediaEntity media) + { + switch (media.MediaType) + { + case "photo": return media.MediaURLHttps; + case "animated_gif": return media.VideoDetails.Variants[0].URL; + case "video": return media.VideoDetails.Variants.OrderByDescending(x => x.Bitrate).First().URL; + default: return null; + } + } + + private string GetMediaType(string mediaType, string mediaUrl) + { + switch (mediaType) + { + case "photo": + var ext = Path.GetExtension(mediaUrl); + switch (ext) + { + case ".jpg": + case ".jpeg": + return "image/jpeg"; + case ".png": + return "image/png"; + } + return null; + + case "animated_gif": + return "image/gif"; + + case "video": + return "video/mp4"; + } + return null; + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index 1ca60c1..2c0b045 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -18,8 +18,6 @@ namespace BirdsiteLive.Domain public interface IUserService { Actor GetUser(TwitterUser twitterUser); - //Note GetStatus(TwitterUser user, ITweet tweet); - Note GetStatus(string username, ITweet tweet); Task FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityFollow activity); Task UndoFollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityUndoFollow activity); } @@ -75,41 +73,6 @@ namespace BirdsiteLive.Domain return user; } - public Note GetStatus(string username, ITweet tweet) - { - //var actor = GetUser(user); - - var actorUrl = $"{_host}/users/{username}"; - var noteId = $"{_host}/users/{username}/statuses/{tweet.Id}"; - var noteUrl = $"{_host}/@{username}/{tweet.Id}"; - - var to = $"{actorUrl}/followers"; - var apPublic = "https://www.w3.org/ns/activitystreams#Public"; - - var note = new Note - { - id = $"{noteId}/activity", - - published = tweet.CreatedAt.ToString("s") + "Z", - url = noteUrl, - attributedTo = actorUrl, - - //to = new [] {to}, - //cc = new [] { apPublic }, - - to = new[] { apPublic }, - cc = new[] { to }, - - sensitive = false, - content = $"

{tweet.Text}

", - attachment = new string[0], - tag = new string[0] - }; - return note; - } - - - public async Task FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityFollow activity) { // Validate diff --git a/src/BirdsiteLive.Pipeline/Contracts/ISaveProgressionProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/ISaveProgressionProcessor.cs new file mode 100644 index 0000000..02efaef --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Contracts/ISaveProgressionProcessor.cs @@ -0,0 +1,11 @@ +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.Pipeline.Models; + +namespace BirdsiteLive.Pipeline.Contracts +{ + public interface ISaveProgressionProcessor + { + Task ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct); + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs index df18fa9..6d55957 100644 --- a/src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs @@ -6,6 +6,6 @@ namespace BirdsiteLive.Pipeline.Contracts { public interface ISendTweetsToFollowersProcessor { - Task ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct); + Task ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct); } } \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs new file mode 100644 index 0000000..5b305e7 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs @@ -0,0 +1,29 @@ +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Models; + +namespace BirdsiteLive.Pipeline.Processors +{ + public class SaveProgressionProcessor : ISaveProgressionProcessor + { + private readonly ITwitterUserDal _twitterUserDal; + + #region Ctor + public SaveProgressionProcessor(ITwitterUserDal twitterUserDal) + { + _twitterUserDal = twitterUserDal; + } + #endregion + + public async Task ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct) + { + var userId = userWithTweetsToSync.User.Id; + var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max(); + var minimumSync = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).Min(); + await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync); + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs index 8fa297f..a20f813 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs @@ -1,60 +1,84 @@ -using System.Linq; +using System; +using System.Collections.Generic; +using System.Linq; using System.Net; using System.Threading; using System.Threading.Tasks; using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; using BirdsiteLive.Domain; using BirdsiteLive.Pipeline.Contracts; using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Twitter; +using Tweetinvi.Models; namespace BirdsiteLive.Pipeline.Processors { public class SendTweetsToFollowersProcessor : ISendTweetsToFollowersProcessor { private readonly IActivityPubService _activityPubService; - private readonly IUserService _userService; + private readonly IStatusService _statusService; private readonly IFollowersDal _followersDal; - private readonly ITwitterUserDal _twitterUserDal; #region Ctor - public SendTweetsToFollowersProcessor(IActivityPubService activityPubService, IUserService userService, IFollowersDal followersDal, ITwitterUserDal twitterUserDal) + public SendTweetsToFollowersProcessor(IActivityPubService activityPubService, IFollowersDal followersDal, IStatusService statusService) { _activityPubService = activityPubService; - _userService = userService; _followersDal = followersDal; - _twitterUserDal = twitterUserDal; + _statusService = statusService; } #endregion - public async Task ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct) + public async Task ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct) { var user = userWithTweetsToSync.User; var userId = user.Id; foreach (var follower in userWithTweetsToSync.Followers) { - var fromStatusId = follower.FollowingsSyncStatus[userId]; - var tweetsToSend = userWithTweetsToSync.Tweets.Where(x => x.Id > fromStatusId).OrderBy(x => x.Id).ToList(); - - var syncStatus = fromStatusId; - foreach (var tweet in tweetsToSend) + try { - var note = _userService.GetStatus(user.Acct, tweet); - var result = await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, follower.InboxUrl); - if (result == HttpStatusCode.Accepted) - syncStatus = tweet.Id; - else - break; + await ProcessFollowerAsync(userWithTweetsToSync.Tweets, follower, userId, user); + } + catch (Exception e) + { + Console.WriteLine(e); + //TODO handle error } - - follower.FollowingsSyncStatus[userId] = syncStatus; - await _followersDal.UpdateFollowerAsync(follower); } - var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max(); - var minimumSync = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).Min(); - await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync); + return userWithTweetsToSync; + } + + private async Task ProcessFollowerAsync(IEnumerable tweets, Follower follower, int userId, + SyncTwitterUser user) + { + var fromStatusId = follower.FollowingsSyncStatus[userId]; + var tweetsToSend = tweets.Where(x => x.Id > fromStatusId).OrderBy(x => x.Id).ToList(); + + var syncStatus = fromStatusId; + try + { + foreach (var tweet in tweetsToSend) + { + var note = _statusService.GetStatus(user.Acct, tweet); + var result = await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, + follower.InboxUrl); + + if (result == HttpStatusCode.Accepted) + syncStatus = tweet.Id; + else + throw new Exception("Posting new note activity failed"); + } + } + finally + { + if (syncStatus != fromStatusId) + { + follower.FollowingsSyncStatus[userId] = syncStatus; + await _followersDal.UpdateFollowerAsync(follower); + } + } } } } \ No newline at end of file diff --git a/src/BirdsiteLive/Controllers/DebugingController.cs b/src/BirdsiteLive/Controllers/DebugingController.cs index 47196c7..164bb24 100644 --- a/src/BirdsiteLive/Controllers/DebugingController.cs +++ b/src/BirdsiteLive/Controllers/DebugingController.cs @@ -92,7 +92,7 @@ namespace BirdsiteLive.Controllers //cc = new [] { apPublic }, sensitive = false, content = "

Woooot

", - attachment = new string[0], + attachment = new Attachment[0], tag = new string[0] } }; diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index 7320eb7..5a98538 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Net.Mime; using System.Threading; using System.Threading.Tasks; using BirdsiteLive.ActivityPub; @@ -18,12 +19,14 @@ namespace BirdsiteLive.Controllers { private readonly ITwitterService _twitterService; private readonly IUserService _userService; + private readonly IStatusService _statusService; #region Ctor - public UsersController(ITwitterService twitterService, IUserService userService) + public UsersController(ITwitterService twitterService, IUserService userService, IStatusService statusService) { _twitterService = twitterService; _userService = userService; + _statusService = statusService; } #endregion @@ -62,7 +65,7 @@ namespace BirdsiteLive.Controllers //var user = _twitterService.GetUser(id); //if (user == null) return NotFound(); - var status = _userService.GetStatus(id, tweet); + var status = _statusService.GetStatus(id, tweet); var jsonApUser = JsonConvert.SerializeObject(status); return Content(jsonApUser, "application/activity+json; charset=utf-8"); } @@ -78,6 +81,8 @@ namespace BirdsiteLive.Controllers using (var reader = new StreamReader(Request.Body)) { var body = await reader.ReadToEndAsync(); + //System.IO.File.WriteAllText($@"C:\apdebug\{Guid.NewGuid()}.json", body); + var activity = ApDeserializer.ProcessActivity(body); // Do something var signature = r.Headers["Signature"].First(); From 66630324d795d35db152e853a1489072e7c729de Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 22 Jul 2020 19:27:49 -0400 Subject: [PATCH 18/67] handle exception in deserializer --- .../ApDeserializer.cs | 76 ++++++++++--------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/src/BirdsiteLive.ActivityPub/ApDeserializer.cs b/src/BirdsiteLive.ActivityPub/ApDeserializer.cs index 1eaffa8..e577fbb 100644 --- a/src/BirdsiteLive.ActivityPub/ApDeserializer.cs +++ b/src/BirdsiteLive.ActivityPub/ApDeserializer.cs @@ -1,4 +1,5 @@ -using Newtonsoft.Json; +using System; +using Newtonsoft.Json; namespace BirdsiteLive.ActivityPub { @@ -6,41 +7,48 @@ namespace BirdsiteLive.ActivityPub { public static Activity ProcessActivity(string json) { - var activity = JsonConvert.DeserializeObject(json); - switch (activity.type) + try { - case "Follow": - return JsonConvert.DeserializeObject(json); - case "Undo": - var a = JsonConvert.DeserializeObject(json); - if(a.apObject.type == "Follow") - return JsonConvert.DeserializeObject(json); - break; - case "Accept": - var accept = JsonConvert.DeserializeObject(json); - //var acceptType = JsonConvert.DeserializeObject(accept.apObject); - switch ((accept.apObject as dynamic).type.ToString()) - { - case "Follow": - var acceptFollow = new ActivityAcceptFollow() - { - type = accept.type, - id = accept.id, - actor = accept.actor, - context = accept.context, - apObject = new ActivityFollow() + var activity = JsonConvert.DeserializeObject(json); + switch (activity.type) + { + case "Follow": + return JsonConvert.DeserializeObject(json); + case "Undo": + var a = JsonConvert.DeserializeObject(json); + if(a.apObject.type == "Follow") + return JsonConvert.DeserializeObject(json); + break; + case "Accept": + var accept = JsonConvert.DeserializeObject(json); + //var acceptType = JsonConvert.DeserializeObject(accept.apObject); + switch ((accept.apObject as dynamic).type.ToString()) + { + case "Follow": + var acceptFollow = new ActivityAcceptFollow() { - id = (accept.apObject as dynamic).id?.ToString(), - type = (accept.apObject as dynamic).type?.ToString(), - actor = (accept.apObject as dynamic).actor?.ToString(), - context = (accept.apObject as dynamic).context?.ToString(), - apObject = (accept.apObject as dynamic).@object?.ToString() - } - }; - return acceptFollow; - break; - } - break; + type = accept.type, + id = accept.id, + actor = accept.actor, + context = accept.context, + apObject = new ActivityFollow() + { + id = (accept.apObject as dynamic).id?.ToString(), + type = (accept.apObject as dynamic).type?.ToString(), + actor = (accept.apObject as dynamic).actor?.ToString(), + context = (accept.apObject as dynamic).context?.ToString(), + apObject = (accept.apObject as dynamic).@object?.ToString() + } + }; + return acceptFollow; + break; + } + break; + } + } + catch (Exception e) + { + Console.WriteLine(e); } return null; From 1724d3a902a42606696665ed7eac9913b1029772 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 22 Jul 2020 19:49:08 -0400 Subject: [PATCH 19/67] added shared inbox --- src/BirdsiteLive.ActivityPub/Models/Actor.cs | 4 +++- .../Models/EndPoints.cs | 7 +++++++ src/BirdsiteLive.Domain/UserService.cs | 19 ++++++++++++------- .../Controllers/InboxController.cs | 2 +- 4 files changed, 23 insertions(+), 9 deletions(-) create mode 100644 src/BirdsiteLive.ActivityPub/Models/EndPoints.cs diff --git a/src/BirdsiteLive.ActivityPub/Models/Actor.cs b/src/BirdsiteLive.ActivityPub/Models/Actor.cs index 59ea71f..d517dc8 100644 --- a/src/BirdsiteLive.ActivityPub/Models/Actor.cs +++ b/src/BirdsiteLive.ActivityPub/Models/Actor.cs @@ -1,4 +1,5 @@ -using BirdsiteLive.ActivityPub.Converters; +using System.Net; +using BirdsiteLive.ActivityPub.Converters; using Newtonsoft.Json; namespace BirdsiteLive.ActivityPub @@ -19,5 +20,6 @@ namespace BirdsiteLive.ActivityPub public PublicKey publicKey { get; set; } public Image icon { get; set; } public Image image { get; set; } + public EndPoints endpoints { get; set; } } } diff --git a/src/BirdsiteLive.ActivityPub/Models/EndPoints.cs b/src/BirdsiteLive.ActivityPub/Models/EndPoints.cs new file mode 100644 index 0000000..8d671d5 --- /dev/null +++ b/src/BirdsiteLive.ActivityPub/Models/EndPoints.cs @@ -0,0 +1,7 @@ +namespace BirdsiteLive.ActivityPub +{ + public class EndPoints + { + public string sharedInbox { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index 2c0b045..25b8ad1 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -27,18 +27,19 @@ namespace BirdsiteLive.Domain private readonly IProcessFollowUser _processFollowUser; private readonly IProcessUndoFollowUser _processUndoFollowUser; + private readonly InstanceSettings _instanceSettings; private readonly ICryptoService _cryptoService; private readonly IActivityPubService _activityPubService; - private readonly string _host; #region Ctor public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser) { + _instanceSettings = instanceSettings; _cryptoService = cryptoService; _activityPubService = activityPubService; _processFollowUser = processFollowUser; _processUndoFollowUser = processUndoFollowUser; - _host = $"https://{instanceSettings.Domain.Replace("https://",string.Empty).Replace("http://", string.Empty).TrimEnd('/')}"; + //_host = $"https://{instanceSettings.Domain.Replace("https://",string.Empty).Replace("http://", string.Empty).TrimEnd('/')}"; } #endregion @@ -46,17 +47,17 @@ namespace BirdsiteLive.Domain { var user = new Actor { - id = $"{_host}/users/{twitterUser.Acct}", + id = $"https://{_instanceSettings.Domain}/users/{twitterUser.Acct}", type = "Person", preferredUsername = twitterUser.Acct, name = twitterUser.Name, - inbox = $"{_host}/users/{twitterUser.Acct}/inbox", + inbox = $"https://{_instanceSettings.Domain}/users/{twitterUser.Acct}/inbox", summary = twitterUser.Description, - url = $"{_host}/@{twitterUser.Acct}", + url = $"https://{_instanceSettings.Domain}/@{twitterUser.Acct}", publicKey = new PublicKey() { - id = $"{_host}/users/{twitterUser.Acct}#main-key", - owner = $"{_host}/users/{twitterUser.Acct}", + id = $"https://{_instanceSettings.Domain}/users/{twitterUser.Acct}#main-key", + owner = $"https://{_instanceSettings.Domain}/users/{twitterUser.Acct}", publicKeyPem = _cryptoService.GetUserPem(twitterUser.Acct) }, icon = new Image @@ -68,6 +69,10 @@ namespace BirdsiteLive.Domain { mediaType = "image/jpeg", url = twitterUser.ProfileBannerURL + }, + endpoints = new EndPoints + { + sharedInbox = $"{_instanceSettings.Domain}/inbox" } }; return user; diff --git a/src/BirdsiteLive/Controllers/InboxController.cs b/src/BirdsiteLive/Controllers/InboxController.cs index 82de7b3..9d7783c 100644 --- a/src/BirdsiteLive/Controllers/InboxController.cs +++ b/src/BirdsiteLive/Controllers/InboxController.cs @@ -19,7 +19,7 @@ namespace BirdsiteLive.Controllers using (var reader = new StreamReader(Request.Body)) { var body = await reader.ReadToEndAsync(); - + //System.IO.File.WriteAllText($@"C:\apdebug\inbox\{Guid.NewGuid()}.json", body); } From fffc9af534873733b75c00c6bf44c552519bde5e Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 22 Jul 2020 20:19:40 -0400 Subject: [PATCH 20/67] extract only needed info from tweets --- src/BirdsiteLive.Domain/StatusService.cs | 69 +++-------- .../Models/UserWithTweetsToSync.cs | 3 +- .../Processors/RetrieveTweetsProcessor.cs | 5 +- .../SendTweetsToFollowersProcessor.cs | 3 +- .../Models/ExtractedMedia.cs | 8 ++ .../Models/ExtractedTweet.cs | 14 +++ src/BirdsiteLive.Twitter/TwitterService.cs | 107 ++++++++++++++++-- 7 files changed, 140 insertions(+), 69 deletions(-) create mode 100644 src/BirdsiteLive.Twitter/Models/ExtractedMedia.cs create mode 100644 src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs diff --git a/src/BirdsiteLive.Domain/StatusService.cs b/src/BirdsiteLive.Domain/StatusService.cs index cc2b688..e50dddb 100644 --- a/src/BirdsiteLive.Domain/StatusService.cs +++ b/src/BirdsiteLive.Domain/StatusService.cs @@ -1,8 +1,10 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.IO; using System.Linq; using BirdsiteLive.ActivityPub; using BirdsiteLive.Common.Settings; +using BirdsiteLive.Twitter.Models; using Tweetinvi.Models; using Tweetinvi.Models.Entities; @@ -10,7 +12,7 @@ namespace BirdsiteLive.Domain { public interface IStatusService { - Note GetStatus(string username, ITweet tweet); + Note GetStatus(string username, ExtractedTweet tweet); } public class StatusService : IStatusService @@ -24,7 +26,7 @@ namespace BirdsiteLive.Domain } #endregion - public Note GetStatus(string username, ITweet tweet) + public Note GetStatus(string username, ExtractedTweet tweet) { var actorUrl = $"https://{_instanceSettings.Domain}/users/{username}"; var noteId = $"https://{_instanceSettings.Domain}/users/{username}/statuses/{tweet.Id}"; @@ -49,8 +51,8 @@ namespace BirdsiteLive.Domain //cc = new string[0], sensitive = false, - content = $"

{tweet.Text}

", - attachment = GetAttachments(tweet.Media), + content = $"

{tweet.MessageContent}

", + attachment = Convert(tweet.Media), tag = new string[0] }; @@ -58,62 +60,17 @@ namespace BirdsiteLive.Domain return note; } - private Attachment[] GetAttachments(List media) + private Attachment[] Convert(ExtractedMedia[] media) { - var result = new List(); - - foreach (var m in media) + return media.Select(x => { - var mediaUrl = GetMediaUrl(m); - var mediaType = GetMediaType(m.MediaType, mediaUrl); - if (mediaType == null) continue; - - var att = new Attachment + return new Attachment { type = "Document", - mediaType = mediaType, - url = mediaUrl + url = x.Url, + mediaType = x.MediaType }; - result.Add(att); - } - - return result.ToArray(); - } - - private string GetMediaUrl(IMediaEntity media) - { - switch (media.MediaType) - { - case "photo": return media.MediaURLHttps; - case "animated_gif": return media.VideoDetails.Variants[0].URL; - case "video": return media.VideoDetails.Variants.OrderByDescending(x => x.Bitrate).First().URL; - default: return null; - } - } - - private string GetMediaType(string mediaType, string mediaUrl) - { - switch (mediaType) - { - case "photo": - var ext = Path.GetExtension(mediaUrl); - switch (ext) - { - case ".jpg": - case ".jpeg": - return "image/jpeg"; - case ".png": - return "image/png"; - } - return null; - - case "animated_gif": - return "image/gif"; - - case "video": - return "video/mp4"; - } - return null; + }).ToArray(); } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Models/UserWithTweetsToSync.cs b/src/BirdsiteLive.Pipeline/Models/UserWithTweetsToSync.cs index 133e2a5..57810c7 100644 --- a/src/BirdsiteLive.Pipeline/Models/UserWithTweetsToSync.cs +++ b/src/BirdsiteLive.Pipeline/Models/UserWithTweetsToSync.cs @@ -1,4 +1,5 @@ using BirdsiteLive.DAL.Models; +using BirdsiteLive.Twitter.Models; using Tweetinvi.Models; namespace BirdsiteLive.Pipeline.Models @@ -6,7 +7,7 @@ namespace BirdsiteLive.Pipeline.Models public class UserWithTweetsToSync { public SyncTwitterUser User { get; set; } - public ITweet[] Tweets { get; set; } + public ExtractedTweet[] Tweets { get; set; } public Follower[] Followers { get; set; } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs index 22416be..dc556ba 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs @@ -7,6 +7,7 @@ using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Contracts; using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Twitter; +using BirdsiteLive.Twitter.Models; using Tweetinvi.Models; namespace BirdsiteLive.Pipeline.Processors @@ -51,9 +52,9 @@ namespace BirdsiteLive.Pipeline.Processors return usersWtTweets.ToArray(); } - private ITweet[] RetrieveNewTweets(SyncTwitterUser user) + private ExtractedTweet[] RetrieveNewTweets(SyncTwitterUser user) { - ITweet[] tweets; + ExtractedTweet[] tweets; if (user.LastTweetPostedId == -1) tweets = _twitterService.GetTimeline(user.Acct, 1); else diff --git a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs index a20f813..3dd37cd 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs @@ -10,6 +10,7 @@ using BirdsiteLive.Domain; using BirdsiteLive.Pipeline.Contracts; using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Twitter; +using BirdsiteLive.Twitter.Models; using Tweetinvi.Models; namespace BirdsiteLive.Pipeline.Processors @@ -50,7 +51,7 @@ namespace BirdsiteLive.Pipeline.Processors return userWithTweetsToSync; } - private async Task ProcessFollowerAsync(IEnumerable tweets, Follower follower, int userId, + private async Task ProcessFollowerAsync(IEnumerable tweets, Follower follower, int userId, SyncTwitterUser user) { var fromStatusId = follower.FollowingsSyncStatus[userId]; diff --git a/src/BirdsiteLive.Twitter/Models/ExtractedMedia.cs b/src/BirdsiteLive.Twitter/Models/ExtractedMedia.cs new file mode 100644 index 0000000..cdab034 --- /dev/null +++ b/src/BirdsiteLive.Twitter/Models/ExtractedMedia.cs @@ -0,0 +1,8 @@ +namespace BirdsiteLive.Twitter.Models +{ + public class ExtractedMedia + { + public string MediaType { get; set; } + public string Url { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs b/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs new file mode 100644 index 0000000..82c67bd --- /dev/null +++ b/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs @@ -0,0 +1,14 @@ +using System; +using System.Net.Sockets; + +namespace BirdsiteLive.Twitter.Models +{ + public class ExtractedTweet + { + public long Id { get; set; } + public long? InReplyToStatusId { get; set; } + public string MessageContent { get; set; } + public ExtractedMedia[] Media { get; set; } + public DateTime CreatedAt { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/TwitterService.cs b/src/BirdsiteLive.Twitter/TwitterService.cs index 4c8878d..ddbaaf0 100644 --- a/src/BirdsiteLive.Twitter/TwitterService.cs +++ b/src/BirdsiteLive.Twitter/TwitterService.cs @@ -1,11 +1,13 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading.Tasks; using BirdsiteLive.Common.Settings; using BirdsiteLive.Twitter.Models; using Tweetinvi; using Tweetinvi.Models; +using Tweetinvi.Models.Entities; using Tweetinvi.Parameters; namespace BirdsiteLive.Twitter @@ -13,8 +15,8 @@ namespace BirdsiteLive.Twitter public interface ITwitterService { TwitterUser GetUser(string username); - ITweet GetTweet(long statusId); - ITweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1); + ExtractedTweet GetTweet(long statusId); + ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1); } public class TwitterService : ITwitterService @@ -31,7 +33,6 @@ namespace BirdsiteLive.Twitter public TwitterUser GetUser(string username) { - //Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true); var user = User.GetUserFromScreenName(username); if (user == null) return null; @@ -47,16 +48,104 @@ namespace BirdsiteLive.Twitter }; } - public ITweet GetTweet(long statusId) + public ExtractedTweet GetTweet(long statusId) { - //Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true); + TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended; var tweet = Tweet.GetTweet(statusId); - return tweet; + return Extract(tweet); } - public ITweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1) + private ExtractedTweet Extract(ITweet tweet) + { + var extractedTweet = new ExtractedTweet + { + Id = tweet.Id, + InReplyToStatusId = tweet.InReplyToStatusId, + MessageContent = ExtractMessage(tweet), + Media = ExtractMedia(tweet.Media), + CreatedAt = tweet.CreatedAt + }; + return extractedTweet; + } + + private string ExtractMessage(ITweet tweet) + { + var tweetUrls = tweet.Media.Select(x => x.URL).Distinct(); + var message = tweet.FullText; + foreach (var tweetUrl in tweetUrls) + message = message.Replace(tweetUrl, string.Empty).Trim(); + + if (tweet.QuotedTweet != null) message = $"[Quote RT] {message}"; + if (tweet.IsRetweet) + { + if (tweet.RetweetedTweet != null) + message = $"[RT {tweet.RetweetedTweet.CreatedBy.ScreenName}] {tweet.RetweetedTweet.FullText}"; + else + message = message.Replace("RT", "[RT]"); + } + + return message; + } + + private ExtractedMedia[] ExtractMedia(List media) + { + var result = new List(); + + foreach (var m in media) + { + var mediaUrl = GetMediaUrl(m); + var mediaType = GetMediaType(m.MediaType, mediaUrl); + if (mediaType == null) continue; + + var att = new ExtractedMedia + { + MediaType = mediaType, + Url = mediaUrl + }; + result.Add(att); + } + + return result.ToArray(); + } + + private string GetMediaUrl(IMediaEntity media) + { + switch (media.MediaType) + { + case "photo": return media.MediaURLHttps; + case "animated_gif": return media.VideoDetails.Variants[0].URL; + case "video": return media.VideoDetails.Variants.OrderByDescending(x => x.Bitrate).First().URL; + default: return null; + } + } + + private string GetMediaType(string mediaType, string mediaUrl) + { + switch (mediaType) + { + case "photo": + var ext = Path.GetExtension(mediaUrl); + switch (ext) + { + case ".jpg": + case ".jpeg": + return "image/jpeg"; + case ".png": + return "image/png"; + } + return null; + + case "animated_gif": + return "image/gif"; + + case "video": + return "video/mp4"; + } + return null; + } + + public ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1) { - //Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true); TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended; var user = User.GetUserFromScreenName(username); @@ -77,7 +166,7 @@ namespace BirdsiteLive.Twitter if (timeline != null) tweets.AddRange(timeline); } - return tweets.ToArray(); + return tweets.Select(Extract).ToArray(); //return tweets.Where(x => returnReplies || string.IsNullOrWhiteSpace(x.InReplyToScreenName)).ToArray(); } } From 4bd3d4289dbf5402574d542767dd7a59acb27b93 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 22 Jul 2020 20:23:26 -0400 Subject: [PATCH 21/67] remove extraction methods from service --- .../Extractors/TweetExtractor.cs | 106 ++++++++++++++++++ src/BirdsiteLive.Twitter/TwitterService.cs | 98 ++-------------- 2 files changed, 113 insertions(+), 91 deletions(-) create mode 100644 src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs diff --git a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs new file mode 100644 index 0000000..7bdb67f --- /dev/null +++ b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs @@ -0,0 +1,106 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using BirdsiteLive.Twitter.Models; +using Tweetinvi.Models; +using Tweetinvi.Models.Entities; + +namespace BirdsiteLive.Twitter.Extractors +{ + public interface ITweetExtractor + { + ExtractedTweet Extract(ITweet tweet); + } + + public class TweetExtractor : ITweetExtractor + { + public ExtractedTweet Extract(ITweet tweet) + { + var extractedTweet = new ExtractedTweet + { + Id = tweet.Id, + InReplyToStatusId = tweet.InReplyToStatusId, + MessageContent = ExtractMessage(tweet), + Media = ExtractMedia(tweet.Media), + CreatedAt = tweet.CreatedAt + }; + return extractedTweet; + } + + public string ExtractMessage(ITweet tweet) + { + var tweetUrls = tweet.Media.Select(x => x.URL).Distinct(); + var message = tweet.FullText; + foreach (var tweetUrl in tweetUrls) + message = message.Replace(tweetUrl, string.Empty).Trim(); + + if (tweet.QuotedTweet != null) message = $"[Quote RT] {message}"; + if (tweet.IsRetweet) + { + if (tweet.RetweetedTweet != null) + message = $"[RT {tweet.RetweetedTweet.CreatedBy.ScreenName}] {tweet.RetweetedTweet.FullText}"; + else + message = message.Replace("RT", "[RT]"); + } + + return message; + } + + public ExtractedMedia[] ExtractMedia(List media) + { + var result = new List(); + + foreach (var m in media) + { + var mediaUrl = GetMediaUrl(m); + var mediaType = GetMediaType(m.MediaType, mediaUrl); + if (mediaType == null) continue; + + var att = new ExtractedMedia + { + MediaType = mediaType, + Url = mediaUrl + }; + result.Add(att); + } + + return result.ToArray(); + } + + public string GetMediaUrl(IMediaEntity media) + { + switch (media.MediaType) + { + case "photo": return media.MediaURLHttps; + case "animated_gif": return media.VideoDetails.Variants[0].URL; + case "video": return media.VideoDetails.Variants.OrderByDescending(x => x.Bitrate).First().URL; + default: return null; + } + } + + public string GetMediaType(string mediaType, string mediaUrl) + { + switch (mediaType) + { + case "photo": + var ext = Path.GetExtension(mediaUrl); + switch (ext) + { + case ".jpg": + case ".jpeg": + return "image/jpeg"; + case ".png": + return "image/png"; + } + return null; + + case "animated_gif": + return "image/gif"; + + case "video": + return "video/mp4"; + } + return null; + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/TwitterService.cs b/src/BirdsiteLive.Twitter/TwitterService.cs index ddbaaf0..21853ad 100644 --- a/src/BirdsiteLive.Twitter/TwitterService.cs +++ b/src/BirdsiteLive.Twitter/TwitterService.cs @@ -4,6 +4,7 @@ using System.IO; using System.Linq; using System.Threading.Tasks; using BirdsiteLive.Common.Settings; +using BirdsiteLive.Twitter.Extractors; using BirdsiteLive.Twitter.Models; using Tweetinvi; using Tweetinvi.Models; @@ -22,11 +23,13 @@ namespace BirdsiteLive.Twitter public class TwitterService : ITwitterService { private readonly TwitterSettings _settings; + private readonly ITweetExtractor _tweetExtractor; #region Ctor - public TwitterService(TwitterSettings settings) + public TwitterService(TwitterSettings settings, ITweetExtractor tweetExtractor) { _settings = settings; + _tweetExtractor = tweetExtractor; Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true); } #endregion @@ -52,97 +55,10 @@ namespace BirdsiteLive.Twitter { TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended; var tweet = Tweet.GetTweet(statusId); - return Extract(tweet); + return _tweetExtractor.Extract(tweet); } - private ExtractedTweet Extract(ITweet tweet) - { - var extractedTweet = new ExtractedTweet - { - Id = tweet.Id, - InReplyToStatusId = tweet.InReplyToStatusId, - MessageContent = ExtractMessage(tweet), - Media = ExtractMedia(tweet.Media), - CreatedAt = tweet.CreatedAt - }; - return extractedTweet; - } - - private string ExtractMessage(ITweet tweet) - { - var tweetUrls = tweet.Media.Select(x => x.URL).Distinct(); - var message = tweet.FullText; - foreach (var tweetUrl in tweetUrls) - message = message.Replace(tweetUrl, string.Empty).Trim(); - - if (tweet.QuotedTweet != null) message = $"[Quote RT] {message}"; - if (tweet.IsRetweet) - { - if (tweet.RetweetedTweet != null) - message = $"[RT {tweet.RetweetedTweet.CreatedBy.ScreenName}] {tweet.RetweetedTweet.FullText}"; - else - message = message.Replace("RT", "[RT]"); - } - - return message; - } - - private ExtractedMedia[] ExtractMedia(List media) - { - var result = new List(); - - foreach (var m in media) - { - var mediaUrl = GetMediaUrl(m); - var mediaType = GetMediaType(m.MediaType, mediaUrl); - if (mediaType == null) continue; - - var att = new ExtractedMedia - { - MediaType = mediaType, - Url = mediaUrl - }; - result.Add(att); - } - - return result.ToArray(); - } - - private string GetMediaUrl(IMediaEntity media) - { - switch (media.MediaType) - { - case "photo": return media.MediaURLHttps; - case "animated_gif": return media.VideoDetails.Variants[0].URL; - case "video": return media.VideoDetails.Variants.OrderByDescending(x => x.Bitrate).First().URL; - default: return null; - } - } - - private string GetMediaType(string mediaType, string mediaUrl) - { - switch (mediaType) - { - case "photo": - var ext = Path.GetExtension(mediaUrl); - switch (ext) - { - case ".jpg": - case ".jpeg": - return "image/jpeg"; - case ".png": - return "image/png"; - } - return null; - - case "animated_gif": - return "image/gif"; - - case "video": - return "video/mp4"; - } - return null; - } + public ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1) { @@ -166,7 +82,7 @@ namespace BirdsiteLive.Twitter if (timeline != null) tweets.AddRange(timeline); } - return tweets.Select(Extract).ToArray(); + return tweets.Select(_tweetExtractor.Extract).ToArray(); //return tweets.Where(x => returnReplies || string.IsNullOrWhiteSpace(x.InReplyToScreenName)).ToArray(); } } From 4a0df5d8093279429ce0a61ab9a711ceb0ceb9bd Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 22 Jul 2020 23:49:57 -0400 Subject: [PATCH 22/67] fix inbox url --- src/BirdsiteLive.Domain/UserService.cs | 2 +- src/BirdsiteLive.Twitter/TwitterService.cs | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index 25b8ad1..f7cf448 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -72,7 +72,7 @@ namespace BirdsiteLive.Domain }, endpoints = new EndPoints { - sharedInbox = $"{_instanceSettings.Domain}/inbox" + sharedInbox = $"https://{_instanceSettings.Domain}/inbox" } }; return user; diff --git a/src/BirdsiteLive.Twitter/TwitterService.cs b/src/BirdsiteLive.Twitter/TwitterService.cs index 21853ad..f49f089 100644 --- a/src/BirdsiteLive.Twitter/TwitterService.cs +++ b/src/BirdsiteLive.Twitter/TwitterService.cs @@ -58,8 +58,6 @@ namespace BirdsiteLive.Twitter return _tweetExtractor.Extract(tweet); } - - public ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1) { TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended; From bfc4dcb4fd35a424f941700346a4e6f5ad6726fb Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 31 Jul 2020 22:13:52 -0400 Subject: [PATCH 23/67] extracting tags --- .../Models/ActivityCreateNote.cs | 1 + src/BirdsiteLive.ActivityPub/Models/Note.cs | 8 ++-- src/BirdsiteLive.ActivityPub/Models/Tag.cs | 8 ++++ src/BirdsiteLive.Domain/ActivityPubService.cs | 1 + src/BirdsiteLive.Domain/StatusService.cs | 35 +++++++++++++- .../Tools/StatusExtractor.cs | 7 +++ src/BirdsiteLive.sln | 9 +++- .../Controllers/DebugingController.cs | 1 + .../BirdsiteLive.Domain.Tests.csproj | 20 ++++++++ .../StatusServiceTests.cs | 48 +++++++++++++++++++ 10 files changed, 130 insertions(+), 8 deletions(-) create mode 100644 src/BirdsiteLive.ActivityPub/Models/Tag.cs create mode 100644 src/BirdsiteLive.Domain/Tools/StatusExtractor.cs create mode 100644 src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj create mode 100644 src/Tests/BirdsiteLive.Domain.Tests/StatusServiceTests.cs diff --git a/src/BirdsiteLive.ActivityPub/Models/ActivityCreateNote.cs b/src/BirdsiteLive.ActivityPub/Models/ActivityCreateNote.cs index d100b3a..fe14ac7 100644 --- a/src/BirdsiteLive.ActivityPub/Models/ActivityCreateNote.cs +++ b/src/BirdsiteLive.ActivityPub/Models/ActivityCreateNote.cs @@ -1,4 +1,5 @@ using System; +using BirdsiteLive.ActivityPub.Models; using Newtonsoft.Json; namespace BirdsiteLive.ActivityPub diff --git a/src/BirdsiteLive.ActivityPub/Models/Note.cs b/src/BirdsiteLive.ActivityPub/Models/Note.cs index b0aba5e..fc6dc5b 100644 --- a/src/BirdsiteLive.ActivityPub/Models/Note.cs +++ b/src/BirdsiteLive.ActivityPub/Models/Note.cs @@ -1,9 +1,7 @@ -using System; -using System.Collections.Generic; -using BirdsiteLive.ActivityPub.Converters; +using BirdsiteLive.ActivityPub.Converters; using Newtonsoft.Json; -namespace BirdsiteLive.ActivityPub +namespace BirdsiteLive.ActivityPub.Models { public class Note { @@ -25,7 +23,7 @@ namespace BirdsiteLive.ActivityPub public string content { get; set; } //public Dictionary contentMap { get; set; } public Attachment[] attachment { get; set; } - public string[] tag { get; set; } + public Tag[] tag { get; set; } //public Dictionary replies; } } \ No newline at end of file diff --git a/src/BirdsiteLive.ActivityPub/Models/Tag.cs b/src/BirdsiteLive.ActivityPub/Models/Tag.cs new file mode 100644 index 0000000..0699c97 --- /dev/null +++ b/src/BirdsiteLive.ActivityPub/Models/Tag.cs @@ -0,0 +1,8 @@ +namespace BirdsiteLive.ActivityPub.Models +{ + public class Tag { + public string type { get; set; } //Hashtag + public string href { get; set; } //https://mastodon.social/tags/app + public string name { get; set; } //#app + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs index 9f6e8e5..b9b248e 100644 --- a/src/BirdsiteLive.Domain/ActivityPubService.cs +++ b/src/BirdsiteLive.Domain/ActivityPubService.cs @@ -4,6 +4,7 @@ using System.Net.Http; using System.Text; using System.Threading.Tasks; using BirdsiteLive.ActivityPub; +using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.Common.Settings; using Newtonsoft.Json; using Org.BouncyCastle.Bcpg; diff --git a/src/BirdsiteLive.Domain/StatusService.cs b/src/BirdsiteLive.Domain/StatusService.cs index e50dddb..32e7f30 100644 --- a/src/BirdsiteLive.Domain/StatusService.cs +++ b/src/BirdsiteLive.Domain/StatusService.cs @@ -2,7 +2,9 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text.RegularExpressions; using BirdsiteLive.ActivityPub; +using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.Common.Settings; using BirdsiteLive.Twitter.Models; using Tweetinvi.Models; @@ -35,6 +37,8 @@ namespace BirdsiteLive.Domain var to = $"{actorUrl}/followers"; var apPublic = "https://www.w3.org/ns/activitystreams#Public"; + var extractedTags = ExtractTags(tweet.MessageContent); + var note = new Note { id = $"{noteId}/activity", @@ -51,17 +55,44 @@ namespace BirdsiteLive.Domain //cc = new string[0], sensitive = false, - content = $"

{tweet.MessageContent}

", + content = $"

{extractedTags.content}

", attachment = Convert(tweet.Media), - tag = new string[0] + tag = extractedTags.tags }; return note; } + private (string content, Tag[] tags) ExtractTags(string messageContent) + { + var regex = new Regex(@"\W(\#[a-zA-Z0-9]+\b)(?!;)"); + var match = regex.Matches(messageContent); + + var tags = new List(); + foreach (var m in match) + { + var tag = m.ToString().Replace("#", string.Empty).Replace("\n", string.Empty).Trim(); + var url = $"https://{_instanceSettings.Domain}/tags/{tag}"; + + tags.Add(new Tag + { + name = $"#{tag}", + href = url, + type = "Hashtag" + }); + + messageContent = messageContent.Replace( + $"#{tag}", + $@"#{tag}"); + } + + return (messageContent, new Tag[0]); + } + private Attachment[] Convert(ExtractedMedia[] media) { + if(media == null) return new Attachment[0]; return media.Select(x => { return new Attachment diff --git a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs new file mode 100644 index 0000000..ed8951a --- /dev/null +++ b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs @@ -0,0 +1,7 @@ +namespace BirdsiteLive.Domain.Tools +{ + public class StatusExtractor + { + + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.sln b/src/BirdsiteLive.sln index 5491d81..6ef9eb6 100644 --- a/src/BirdsiteLive.sln +++ b/src/BirdsiteLive.sln @@ -33,7 +33,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.DAL.Postgres.T EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Pipeline", "Pipeline", "{DA3C160C-4811-4E26-A5AD-42B81FAF2D7C}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdsiteLive.Pipeline", "BirdsiteLive.Pipeline\BirdsiteLive.Pipeline.csproj", "{2A8CC30D-D775-47D1-9388-F72A5C32DE2A}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Pipeline", "BirdsiteLive.Pipeline\BirdsiteLive.Pipeline.csproj", "{2A8CC30D-D775-47D1-9388-F72A5C32DE2A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdsiteLive.Domain.Tests", "Tests\BirdsiteLive.Domain.Tests\BirdsiteLive.Domain.Tests.csproj", "{F544D745-89A8-4DEA-B61C-A7E6C53C1D63}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -89,6 +91,10 @@ Global {2A8CC30D-D775-47D1-9388-F72A5C32DE2A}.Debug|Any CPU.Build.0 = Debug|Any CPU {2A8CC30D-D775-47D1-9388-F72A5C32DE2A}.Release|Any CPU.ActiveCfg = Release|Any CPU {2A8CC30D-D775-47D1-9388-F72A5C32DE2A}.Release|Any CPU.Build.0 = Release|Any CPU + {F544D745-89A8-4DEA-B61C-A7E6C53C1D63}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F544D745-89A8-4DEA-B61C-A7E6C53C1D63}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F544D745-89A8-4DEA-B61C-A7E6C53C1D63}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F544D745-89A8-4DEA-B61C-A7E6C53C1D63}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -105,6 +111,7 @@ Global {87E46519-BBF2-437C-8A5B-CF6CDE7CDAA6} = {CFAB3509-3931-42DB-AC97-4F91FC2D849C} {CD9489BF-69C8-4705-8774-81C45F4F8FE1} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94} {2A8CC30D-D775-47D1-9388-F72A5C32DE2A} = {DA3C160C-4811-4E26-A5AD-42B81FAF2D7C} + {F544D745-89A8-4DEA-B61C-A7E6C53C1D63} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {69E8DCAD-4C37-4010-858F-5F94E6FBABCE} diff --git a/src/BirdsiteLive/Controllers/DebugingController.cs b/src/BirdsiteLive/Controllers/DebugingController.cs index 164bb24..cb67a6e 100644 --- a/src/BirdsiteLive/Controllers/DebugingController.cs +++ b/src/BirdsiteLive/Controllers/DebugingController.cs @@ -5,6 +5,7 @@ using System.Net.Http; using System.Text; using System.Threading.Tasks; using BirdsiteLive.ActivityPub; +using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.Common.Settings; using BirdsiteLive.Domain; using Microsoft.AspNetCore.Mvc; diff --git a/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj b/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj new file mode 100644 index 0000000..8762be8 --- /dev/null +++ b/src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj @@ -0,0 +1,20 @@ + + + + netcoreapp3.1 + + false + + + + + + + + + + + + + + diff --git a/src/Tests/BirdsiteLive.Domain.Tests/StatusServiceTests.cs b/src/Tests/BirdsiteLive.Domain.Tests/StatusServiceTests.cs new file mode 100644 index 0000000..b1f3f6f --- /dev/null +++ b/src/Tests/BirdsiteLive.Domain.Tests/StatusServiceTests.cs @@ -0,0 +1,48 @@ +using System; +using BirdsiteLive.Common.Settings; +using BirdsiteLive.Twitter.Models; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace BirdsiteLive.Domain.Tests +{ + [TestClass] + public class StatusServiceTests + { + private readonly InstanceSettings _settings; + + #region Ctor + public StatusServiceTests() + { + _settings = new InstanceSettings + { + Domain = "domain.name" + }; + } + #endregion + + [TestMethod] + public void ExtractMentionsTest() + { + #region Stubs + var username = "MyUserName"; + var extractedTweet = new ExtractedTweet + { + Id = 124L, + CreatedAt = DateTime.UtcNow, + MessageContent = @"Getting ready for the weekend...have a great one everyone! +⁠ +Photo by Tim Tronckoe | @timtronckoe +⁠ +#archenemy #michaelamott #alissawhitegluz #jeffloomis #danielerlandsson #sharleedangelo⁠" + }; + #endregion + + var service = new StatusService(_settings); + var result = service.GetStatus(username, extractedTweet); + + #region Validations + + #endregion + } + } +} From 83507614a4a508f93677d8966c69c6dba349f409 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 31 Jul 2020 22:49:00 -0400 Subject: [PATCH 24/67] testing hashtag extraction --- src/BirdsiteLive.Domain/StatusService.cs | 33 ++--------- .../Tools/StatusExtractor.cs | 52 +++++++++++++++- .../StatusServiceTests.cs | 42 ++++++------- .../Tools/StatusExtractorTests.cs | 59 +++++++++++++++++++ 4 files changed, 134 insertions(+), 52 deletions(-) create mode 100644 src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs diff --git a/src/BirdsiteLive.Domain/StatusService.cs b/src/BirdsiteLive.Domain/StatusService.cs index 32e7f30..172a366 100644 --- a/src/BirdsiteLive.Domain/StatusService.cs +++ b/src/BirdsiteLive.Domain/StatusService.cs @@ -6,6 +6,7 @@ using System.Text.RegularExpressions; using BirdsiteLive.ActivityPub; using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.Common.Settings; +using BirdsiteLive.Domain.Tools; using BirdsiteLive.Twitter.Models; using Tweetinvi.Models; using Tweetinvi.Models.Entities; @@ -20,11 +21,13 @@ namespace BirdsiteLive.Domain public class StatusService : IStatusService { private readonly InstanceSettings _instanceSettings; + private readonly IStatusExtractor _statusExtractor; #region Ctor - public StatusService(InstanceSettings instanceSettings) + public StatusService(InstanceSettings instanceSettings, IStatusExtractor statusExtractor) { _instanceSettings = instanceSettings; + _statusExtractor = statusExtractor; } #endregion @@ -37,7 +40,7 @@ namespace BirdsiteLive.Domain var to = $"{actorUrl}/followers"; var apPublic = "https://www.w3.org/ns/activitystreams#Public"; - var extractedTags = ExtractTags(tweet.MessageContent); + var extractedTags = _statusExtractor.ExtractTags(tweet.MessageContent); var note = new Note { @@ -64,32 +67,6 @@ namespace BirdsiteLive.Domain return note; } - private (string content, Tag[] tags) ExtractTags(string messageContent) - { - var regex = new Regex(@"\W(\#[a-zA-Z0-9]+\b)(?!;)"); - var match = regex.Matches(messageContent); - - var tags = new List(); - foreach (var m in match) - { - var tag = m.ToString().Replace("#", string.Empty).Replace("\n", string.Empty).Trim(); - var url = $"https://{_instanceSettings.Domain}/tags/{tag}"; - - tags.Add(new Tag - { - name = $"#{tag}", - href = url, - type = "Hashtag" - }); - - messageContent = messageContent.Replace( - $"#{tag}", - $@"#{tag}"); - } - - return (messageContent, new Tag[0]); - } - private Attachment[] Convert(ExtractedMedia[] media) { if(media == null) return new Attachment[0]; diff --git a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs index ed8951a..c19f97e 100644 --- a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs +++ b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs @@ -1,7 +1,53 @@ -namespace BirdsiteLive.Domain.Tools +using System.Collections.Generic; +using System.Text.RegularExpressions; +using BirdsiteLive.ActivityPub.Models; +using BirdsiteLive.Common.Settings; + +namespace BirdsiteLive.Domain.Tools { - public class StatusExtractor + public interface IStatusExtractor { - + (string content, Tag[] tags) ExtractTags(string messageContent); + } + + public class StatusExtractor : IStatusExtractor + { + private readonly InstanceSettings _instanceSettings; + + #region Ctor + public StatusExtractor(InstanceSettings instanceSettings) + { + _instanceSettings = instanceSettings; + } + #endregion + + public (string content, Tag[] tags) ExtractTags(string messageContent) + { + var regex = new Regex(@"\W(\#[a-zA-Z0-9]+\b)(?!;)"); + var match = regex.Matches(messageContent); + + var tags = new List(); + foreach (var m in match) + { + var tag = m.ToString().Replace("#", string.Empty).Replace("\n", string.Empty).Trim(); + var url = $"https://{_instanceSettings.Domain}/tags/{tag}"; + + tags.Add(new Tag + { + name = $"#{tag}", + href = url, + type = "Hashtag" + }); + + messageContent = Regex.Replace(messageContent, m.ToString(), + $@"#{tag}"); + + //messageContent = messageContent.Replace( + // $"#{tag}", + // $@"#{tag}"); + } + + return (messageContent, new Tag[0]); + } } } \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Domain.Tests/StatusServiceTests.cs b/src/Tests/BirdsiteLive.Domain.Tests/StatusServiceTests.cs index b1f3f6f..e6178c3 100644 --- a/src/Tests/BirdsiteLive.Domain.Tests/StatusServiceTests.cs +++ b/src/Tests/BirdsiteLive.Domain.Tests/StatusServiceTests.cs @@ -20,29 +20,29 @@ namespace BirdsiteLive.Domain.Tests } #endregion - [TestMethod] - public void ExtractMentionsTest() - { - #region Stubs - var username = "MyUserName"; - var extractedTweet = new ExtractedTweet - { - Id = 124L, - CreatedAt = DateTime.UtcNow, - MessageContent = @"Getting ready for the weekend...have a great one everyone! -⁠ -Photo by Tim Tronckoe | @timtronckoe -⁠ -#archenemy #michaelamott #alissawhitegluz #jeffloomis #danielerlandsson #sharleedangelo⁠" - }; - #endregion +// [TestMethod] +// public void ExtractMentionsTest() +// { +// #region Stubs +// var username = "MyUserName"; +// var extractedTweet = new ExtractedTweet +// { +// Id = 124L, +// CreatedAt = DateTime.UtcNow, +// MessageContent = @"Getting ready for the weekend...have a great one everyone! +//⁠ +//Photo by Tim Tronckoe | @timtronckoe +//⁠ +//#archenemy #michaelamott #alissawhitegluz #jeffloomis #danielerlandsson #sharleedangelo⁠" +// }; +// #endregion - var service = new StatusService(_settings); - var result = service.GetStatus(username, extractedTweet); +// var service = new StatusService(_settings); +// var result = service.GetStatus(username, extractedTweet); - #region Validations +// #region Validations - #endregion - } +// #endregion +// } } } diff --git a/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs new file mode 100644 index 0000000..d2b63bf --- /dev/null +++ b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs @@ -0,0 +1,59 @@ +using System; +using BirdsiteLive.Common.Settings; +using BirdsiteLive.Domain.Tools; +using BirdsiteLive.Twitter.Models; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace BirdsiteLive.Domain.Tests.Tools +{ + [TestClass] + public class StatusExtractorTests + { + private readonly InstanceSettings _settings; + + #region Ctor + public StatusExtractorTests() + { + _settings = new InstanceSettings + { + Domain = "domain.name" + }; + } + #endregion + + [TestMethod] + public void Extract_SingleTag_Test() + { + #region Stubs + var message = $"Bla!{Environment.NewLine}#mytag⁠"; + #endregion + + var service = new StatusExtractor(_settings); + var result = service.ExtractTags(message); + + #region Validations + Assert.IsTrue(result.content.Contains("Bla!")); + Assert.IsTrue(result.content.Contains(@"#mytag")); + #endregion + } + + [TestMethod] + public void Extract_MultiTags_Test() + { + #region Stubs + var message = $"Bla!{Environment.NewLine}#mytag #mytag2 #mytag3⁠{Environment.NewLine}Test #bal Test"; + #endregion + + var service = new StatusExtractor(_settings); + var result = service.ExtractTags(message); + + #region Validations + Assert.IsTrue(result.content.Contains("Bla!")); + Assert.IsTrue(result.content.Contains(@"#mytag")); + Assert.IsTrue(result.content.Contains(@"#mytag2")); + Assert.IsTrue(result.content.Contains(@"#mytag3")); + Assert.IsTrue(result.content.Contains(@"#bal")); + #endregion + } + } +} \ No newline at end of file From fdfe4c703c5327d860cd5204c1c7ee90ceea1d5d Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 31 Jul 2020 22:52:40 -0400 Subject: [PATCH 25/67] fix DI --- src/BirdsiteLive/Controllers/DebugingController.cs | 2 +- src/BirdsiteLive/Controllers/InboxController.cs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/BirdsiteLive/Controllers/DebugingController.cs b/src/BirdsiteLive/Controllers/DebugingController.cs index cb67a6e..12ac90e 100644 --- a/src/BirdsiteLive/Controllers/DebugingController.cs +++ b/src/BirdsiteLive/Controllers/DebugingController.cs @@ -94,7 +94,7 @@ namespace BirdsiteLive.Controllers sensitive = false, content = "

Woooot

", attachment = new Attachment[0], - tag = new string[0] + tag = new Tag[0] } }; diff --git a/src/BirdsiteLive/Controllers/InboxController.cs b/src/BirdsiteLive/Controllers/InboxController.cs index 9d7783c..a0d8748 100644 --- a/src/BirdsiteLive/Controllers/InboxController.cs +++ b/src/BirdsiteLive/Controllers/InboxController.cs @@ -23,8 +23,7 @@ namespace BirdsiteLive.Controllers } - - throw new NotImplementedException(); + return Accepted(); } } } \ No newline at end of file From 6073ee1ba47894d2c0c66ea659bd57e55c2dba80 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 31 Jul 2020 23:03:20 -0400 Subject: [PATCH 26/67] extract mentions --- .../Tools/StatusExtractor.cs | 33 ++++++++---- .../Tools/StatusExtractorTests.cs | 52 ++++++++++++++++++- 2 files changed, 74 insertions(+), 11 deletions(-) diff --git a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs index c19f97e..91dc453 100644 --- a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs +++ b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs @@ -12,6 +12,8 @@ namespace BirdsiteLive.Domain.Tools public class StatusExtractor : IStatusExtractor { + private readonly Regex _hastagRegex = new Regex(@"\W(\#[a-zA-Z0-9]+\b)(?!;)"); + private readonly Regex _mentionRegex = new Regex(@"\W(\@[a-zA-Z0-9]+\b)(?!;)"); private readonly InstanceSettings _instanceSettings; #region Ctor @@ -23,11 +25,10 @@ namespace BirdsiteLive.Domain.Tools public (string content, Tag[] tags) ExtractTags(string messageContent) { - var regex = new Regex(@"\W(\#[a-zA-Z0-9]+\b)(?!;)"); - var match = regex.Matches(messageContent); - var tags = new List(); - foreach (var m in match) + + var hashtagMatch = _hastagRegex.Matches(messageContent); + foreach (var m in hashtagMatch) { var tag = m.ToString().Replace("#", string.Empty).Replace("\n", string.Empty).Trim(); var url = $"https://{_instanceSettings.Domain}/tags/{tag}"; @@ -41,13 +42,27 @@ namespace BirdsiteLive.Domain.Tools messageContent = Regex.Replace(messageContent, m.ToString(), $@"#{tag}"); - - //messageContent = messageContent.Replace( - // $"#{tag}", - // $@"#{tag}"); } - return (messageContent, new Tag[0]); + var mentionMatch = _mentionRegex.Matches(messageContent); + foreach (var m in mentionMatch) + { + var mention = m.ToString().Replace("@", string.Empty).Replace("\n", string.Empty).Trim(); + var url = $"https://{_instanceSettings.Domain}/users/{mention}"; + var name = $"@{mention}@{_instanceSettings.Domain}"; + + tags.Add(new Tag + { + name = name, + href = url, + type = "Mention" + }); + + messageContent = Regex.Replace(messageContent, m.ToString(), + $@"@{mention}"); + } + + return (messageContent, tags.ToArray()); } } } \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs index d2b63bf..d40a519 100644 --- a/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs +++ b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using BirdsiteLive.Common.Settings; using BirdsiteLive.Domain.Tools; using BirdsiteLive.Twitter.Models; @@ -22,7 +23,7 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion [TestMethod] - public void Extract_SingleTag_Test() + public void Extract_SingleHashTag_Test() { #region Stubs var message = $"Bla!{Environment.NewLine}#mytag⁠"; @@ -32,13 +33,18 @@ namespace BirdsiteLive.Domain.Tests.Tools var result = service.ExtractTags(message); #region Validations + Assert.AreEqual(1, result.tags.Length); + Assert.AreEqual("#mytag", result.tags.First().name); + Assert.AreEqual("Hashtag", result.tags.First().type); + Assert.AreEqual("https://domain.name/tags/mytag", result.tags.First().href); + Assert.IsTrue(result.content.Contains("Bla!")); Assert.IsTrue(result.content.Contains(@"#mytag")); #endregion } [TestMethod] - public void Extract_MultiTags_Test() + public void Extract_MultiHashTags_Test() { #region Stubs var message = $"Bla!{Environment.NewLine}#mytag #mytag2 #mytag3⁠{Environment.NewLine}Test #bal Test"; @@ -48,6 +54,7 @@ namespace BirdsiteLive.Domain.Tests.Tools var result = service.ExtractTags(message); #region Validations + Assert.AreEqual(4, result.tags.Length); Assert.IsTrue(result.content.Contains("Bla!")); Assert.IsTrue(result.content.Contains(@"#mytag")); Assert.IsTrue(result.content.Contains(@"#mytag2")); @@ -55,5 +62,46 @@ namespace BirdsiteLive.Domain.Tests.Tools Assert.IsTrue(result.content.Contains(@"#bal")); #endregion } + + [TestMethod] + public void Extract_SingleMentionTag_Test() + { + #region Stubs + var message = $"Bla!{Environment.NewLine}@mynickname⁠"; + #endregion + + var service = new StatusExtractor(_settings); + var result = service.ExtractTags(message); + + #region Validations + Assert.AreEqual(1, result.tags.Length); + Assert.AreEqual("@mynickname@domain.name", result.tags.First().name); + Assert.AreEqual("Mention", result.tags.First().type); + Assert.AreEqual("https://domain.name/users/mynickname", result.tags.First().href); + + Assert.IsTrue(result.content.Contains("Bla!")); + Assert.IsTrue(result.content.Contains(@"@mynickname")); + #endregion + } + + [TestMethod] + public void Extract_MultiMentionTag_Test() + { + #region Stubs + var message = $"Bla!{Environment.NewLine}@mynickname⁠ @mynickname2 @mynickname3{Environment.NewLine}Test @dada Test"; + #endregion + + var service = new StatusExtractor(_settings); + var result = service.ExtractTags(message); + + #region Validations + Assert.AreEqual(4, result.tags.Length); + Assert.IsTrue(result.content.Contains("Bla!")); + Assert.IsTrue(result.content.Contains(@"@mynickname")); + Assert.IsTrue(result.content.Contains(@"@mynickname2")); + Assert.IsTrue(result.content.Contains(@"@mynickname3")); + Assert.IsTrue(result.content.Contains(@"@dada")); + #endregion + } } } \ No newline at end of file From 8643f3d366b8d2211a269c7159a081fbbd75f9ca Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 31 Jul 2020 23:05:09 -0400 Subject: [PATCH 27/67] added heteregeneous test --- .../Tools/StatusExtractorTests.cs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs index d40a519..839529d 100644 --- a/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs +++ b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs @@ -103,5 +103,26 @@ namespace BirdsiteLive.Domain.Tests.Tools Assert.IsTrue(result.content.Contains(@"@dada")); #endregion } + + [TestMethod] + public void Extract_HeterogeneousTag_Test() + { + #region Stubs + var message = $"Bla!{Environment.NewLine}@mynickname⁠ #mytag2 @mynickname3{Environment.NewLine}Test @dada #dada Test"; + #endregion + + var service = new StatusExtractor(_settings); + var result = service.ExtractTags(message); + + #region Validations + Assert.AreEqual(5, result.tags.Length); + Assert.IsTrue(result.content.Contains("Bla!")); + Assert.IsTrue(result.content.Contains(@"@mynickname")); + Assert.IsTrue(result.content.Contains(@"#mytag2")); + Assert.IsTrue(result.content.Contains(@"@mynickname3")); + Assert.IsTrue(result.content.Contains(@"@dada")); + Assert.IsTrue(result.content.Contains(@"#dada")); + #endregion + } } } \ No newline at end of file From 4fa8b1ade2d38137277dc9ad85b83dcdf0200dbe Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 31 Jul 2020 23:11:41 -0400 Subject: [PATCH 28/67] added special char --- .../Tools/StatusExtractor.cs | 4 ++-- .../Tools/StatusExtractorTests.cs | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs index 91dc453..8ea3ce9 100644 --- a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs +++ b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs @@ -12,8 +12,8 @@ namespace BirdsiteLive.Domain.Tools public class StatusExtractor : IStatusExtractor { - private readonly Regex _hastagRegex = new Regex(@"\W(\#[a-zA-Z0-9]+\b)(?!;)"); - private readonly Regex _mentionRegex = new Regex(@"\W(\@[a-zA-Z0-9]+\b)(?!;)"); + private readonly Regex _hastagRegex = new Regex(@"\W(\#[a-zA-Z0-9_]+\b)(?!;)"); + private readonly Regex _mentionRegex = new Regex(@"\W(\@[a-zA-Z0-9_]+\b)(?!;)"); private readonly InstanceSettings _instanceSettings; #region Ctor diff --git a/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs index 839529d..e0ab5f7 100644 --- a/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs +++ b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs @@ -84,6 +84,27 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion } + [TestMethod] + public void Extract_SingleMentionTag_SpecialChar_Test() + { + #region Stubs + var message = $"Bla!{Environment.NewLine}@my___nickname⁠"; + #endregion + + var service = new StatusExtractor(_settings); + var result = service.ExtractTags(message); + + #region Validations + Assert.AreEqual(1, result.tags.Length); + Assert.AreEqual("@my___nickname@domain.name", result.tags.First().name); + Assert.AreEqual("Mention", result.tags.First().type); + Assert.AreEqual("https://domain.name/users/my___nickname", result.tags.First().href); + + Assert.IsTrue(result.content.Contains("Bla!")); + Assert.IsTrue(result.content.Contains(@"@my___nickname")); + #endregion + } + [TestMethod] public void Extract_MultiMentionTag_Test() { From 4e9e096a6582ff9f76e1ca030cc61580c1494982 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 31 Jul 2020 23:13:17 -0400 Subject: [PATCH 29/67] added spacing on urls --- src/BirdsiteLive.Domain/Tools/StatusExtractor.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs index 8ea3ce9..9ba889c 100644 --- a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs +++ b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs @@ -41,7 +41,7 @@ namespace BirdsiteLive.Domain.Tools }); messageContent = Regex.Replace(messageContent, m.ToString(), - $@"#{tag}"); + $@" #{tag}"); } var mentionMatch = _mentionRegex.Matches(messageContent); @@ -59,7 +59,7 @@ namespace BirdsiteLive.Domain.Tools }); messageContent = Regex.Replace(messageContent, m.ToString(), - $@"@{mention}"); + $@" @{mention}"); } return (messageContent, tags.ToArray()); From d9a1dc84bea52cc852e80fcb1b14a3ff03644f92 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 31 Jul 2020 23:16:23 -0400 Subject: [PATCH 30/67] =?UTF-8?q?added=20=E3=83=BC=20char=20support?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Tools/StatusExtractor.cs | 4 ++-- .../Tools/StatusExtractorTests.cs | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs index 9ba889c..134fc3f 100644 --- a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs +++ b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs @@ -12,8 +12,8 @@ namespace BirdsiteLive.Domain.Tools public class StatusExtractor : IStatusExtractor { - private readonly Regex _hastagRegex = new Regex(@"\W(\#[a-zA-Z0-9_]+\b)(?!;)"); - private readonly Regex _mentionRegex = new Regex(@"\W(\@[a-zA-Z0-9_]+\b)(?!;)"); + private readonly Regex _hastagRegex = new Regex(@"\W(\#[a-zA-Z0-9_ー]+\b)(?!;)"); + private readonly Regex _mentionRegex = new Regex(@"\W(\@[a-zA-Z0-9_ー]+\b)(?!;)"); private readonly InstanceSettings _instanceSettings; #region Ctor diff --git a/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs index e0ab5f7..865fa76 100644 --- a/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs +++ b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs @@ -43,6 +43,27 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion } + [TestMethod] + public void Extract_SingleHashTag_SpecialChar_Test() + { + #region Stubs + var message = $"Bla!{Environment.NewLine}#COVIDー19⁠"; + #endregion + + var service = new StatusExtractor(_settings); + var result = service.ExtractTags(message); + + #region Validations + Assert.AreEqual(1, result.tags.Length); + Assert.AreEqual("#COVIDー19", result.tags.First().name); + Assert.AreEqual("Hashtag", result.tags.First().type); + Assert.AreEqual("https://domain.name/tags/COVIDー19", result.tags.First().href); + + Assert.IsTrue(result.content.Contains("Bla!")); + Assert.IsTrue(result.content.Contains(@"#COVIDー19")); + #endregion + } + [TestMethod] public void Extract_MultiHashTags_Test() { From ce996c20b2fc34f233ea884ef0cb6c510ac3018f Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 1 Aug 2020 00:00:27 -0400 Subject: [PATCH 31/67] fix start tag detection --- .../Tools/StatusExtractor.cs | 7 +++- .../Tools/StatusExtractorTests.cs | 42 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs index 134fc3f..c8067c1 100644 --- a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs +++ b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs @@ -13,7 +13,11 @@ namespace BirdsiteLive.Domain.Tools public class StatusExtractor : IStatusExtractor { private readonly Regex _hastagRegex = new Regex(@"\W(\#[a-zA-Z0-9_ー]+\b)(?!;)"); + //private readonly Regex _hastagRegex = new Regex(@"(?<=[\s>]|^)#(\w*[a-zA-Z0-9_ー]+\w*)\b(?!;)"); + //private readonly Regex _hastagRegex = new Regex(@"(?<=[\s>]|^)#(\w*[a-zA-Z0-9_ー]+)\b(?!;)"); private readonly Regex _mentionRegex = new Regex(@"\W(\@[a-zA-Z0-9_ー]+\b)(?!;)"); + //private readonly Regex _mentionRegex = new Regex(@"(?<=[\s>]|^)@(\w*[a-zA-Z0-9_ー]+\w*)\b(?!;)"); + //private readonly Regex _mentionRegex = new Regex(@"(?<=[\s>]|^)@(\w*[a-zA-Z0-9_ー]+)\b(?!;)"); private readonly InstanceSettings _instanceSettings; #region Ctor @@ -26,6 +30,7 @@ namespace BirdsiteLive.Domain.Tools public (string content, Tag[] tags) ExtractTags(string messageContent) { var tags = new List(); + messageContent = $" {messageContent} "; var hashtagMatch = _hastagRegex.Matches(messageContent); foreach (var m in hashtagMatch) @@ -62,7 +67,7 @@ namespace BirdsiteLive.Domain.Tools $@" @{mention}"); } - return (messageContent, tags.ToArray()); + return (messageContent.Trim(), tags.ToArray()); } } } \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs index 865fa76..d2f3bc9 100644 --- a/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs +++ b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs @@ -43,6 +43,27 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion } + [TestMethod] + public void Extract_SingleHashTag_AtStart_Test() + { + #region Stubs + var message = $"#mytag⁠ Bla!"; + #endregion + + var service = new StatusExtractor(_settings); + var result = service.ExtractTags(message); + + #region Validations + Assert.AreEqual(1, result.tags.Length); + Assert.AreEqual("#mytag", result.tags.First().name); + Assert.AreEqual("Hashtag", result.tags.First().type); + Assert.AreEqual("https://domain.name/tags/mytag", result.tags.First().href); + + Assert.IsTrue(result.content.Contains("Bla!")); + Assert.IsTrue(result.content.Contains(@"#mytag")); + #endregion + } + [TestMethod] public void Extract_SingleHashTag_SpecialChar_Test() { @@ -126,6 +147,27 @@ namespace BirdsiteLive.Domain.Tests.Tools #endregion } + [TestMethod] + public void Extract_SingleMentionTag_AtStart_Test() + { + #region Stubs + var message = $"@mynickname Bla!"; + #endregion + + var service = new StatusExtractor(_settings); + var result = service.ExtractTags(message); + + #region Validations + Assert.AreEqual(1, result.tags.Length); + Assert.AreEqual("@mynickname@domain.name", result.tags.First().name); + Assert.AreEqual("Mention", result.tags.First().type); + Assert.AreEqual("https://domain.name/users/mynickname", result.tags.First().href); + + Assert.IsTrue(result.content.Contains("Bla!")); + Assert.IsTrue(result.content.Contains(@"@mynickname")); + #endregion + } + [TestMethod] public void Extract_MultiMentionTag_Test() { From fce355c2e8f7f97790b1d7d875107b60102ad640 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 1 Aug 2020 00:32:04 -0400 Subject: [PATCH 32/67] added replyTo infos in status --- src/BirdsiteLive.Domain/StatusService.cs | 10 ++++++++-- src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs | 1 + src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs | 1 + 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/BirdsiteLive.Domain/StatusService.cs b/src/BirdsiteLive.Domain/StatusService.cs index 172a366..51cddfc 100644 --- a/src/BirdsiteLive.Domain/StatusService.cs +++ b/src/BirdsiteLive.Domain/StatusService.cs @@ -41,15 +41,21 @@ namespace BirdsiteLive.Domain var apPublic = "https://www.w3.org/ns/activitystreams#Public"; var extractedTags = _statusExtractor.ExtractTags(tweet.MessageContent); - + + string inReplyTo = null; + if (tweet.InReplyToStatusId != default) + inReplyTo = $"https://{_instanceSettings.Domain}/users/{tweet.InReplyToAccount}/statuses/{tweet.InReplyToStatusId}"; + var note = new Note { - id = $"{noteId}/activity", + //id = $"{noteId}/activity", + id = $"{noteId}", published = tweet.CreatedAt.ToString("s") + "Z", url = noteUrl, attributedTo = actorUrl, + inReplyTo = inReplyTo, //to = new [] {to}, //cc = new [] { apPublic }, diff --git a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs index 7bdb67f..4a0892a 100644 --- a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs +++ b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs @@ -20,6 +20,7 @@ namespace BirdsiteLive.Twitter.Extractors { Id = tweet.Id, InReplyToStatusId = tweet.InReplyToStatusId, + InReplyToAccount = tweet.InReplyToScreenName, MessageContent = ExtractMessage(tweet), Media = ExtractMedia(tweet.Media), CreatedAt = tweet.CreatedAt diff --git a/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs b/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs index 82c67bd..0363973 100644 --- a/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs +++ b/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs @@ -10,5 +10,6 @@ namespace BirdsiteLive.Twitter.Models public string MessageContent { get; set; } public ExtractedMedia[] Media { get; set; } public DateTime CreatedAt { get; set; } + public string InReplyToAccount { get; set; } } } \ No newline at end of file From 16d310a37ee71ebc0dd2d641f2e0e779fe9d6e18 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 1 Aug 2020 13:19:41 -0400 Subject: [PATCH 33/67] hack to handle saved inbox --- src/BirdsiteLive.Domain/ActivityPubService.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs index b9b248e..043a7fd 100644 --- a/src/BirdsiteLive.Domain/ActivityPubService.cs +++ b/src/BirdsiteLive.Domain/ActivityPubService.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using System.Net; using System.Net.Http; using System.Text; @@ -89,6 +90,10 @@ namespace BirdsiteLive.Domain //} }; + //TODO Remove this + if (targetInbox.Contains(targetHost)) + targetInbox = targetInbox.Split(new []{ targetHost }, StringSplitOptions.RemoveEmptyEntries).Last(); + return await PostDataAsync(noteActivity, targetHost, actor, targetInbox); } From d33cbbfb1ce068db2fc1db5ceef9ce07de51b91f Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 1 Aug 2020 13:56:59 -0400 Subject: [PATCH 34/67] extract urls --- .../Tools/StatusExtractor.cs | 34 +++++++++ .../Tools/StatusExtractorTests.cs | 74 +++++++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs index c8067c1..722f369 100644 --- a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs +++ b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs @@ -18,6 +18,7 @@ namespace BirdsiteLive.Domain.Tools private readonly Regex _mentionRegex = new Regex(@"\W(\@[a-zA-Z0-9_ー]+\b)(?!;)"); //private readonly Regex _mentionRegex = new Regex(@"(?<=[\s>]|^)@(\w*[a-zA-Z0-9_ー]+\w*)\b(?!;)"); //private readonly Regex _mentionRegex = new Regex(@"(?<=[\s>]|^)@(\w*[a-zA-Z0-9_ー]+)\b(?!;)"); + private readonly Regex _urlRegex = new Regex(@"((http|ftp|https):\/\/[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])?)"); private readonly InstanceSettings _instanceSettings; #region Ctor @@ -32,6 +33,38 @@ namespace BirdsiteLive.Domain.Tools var tags = new List(); messageContent = $" {messageContent} "; + // Extract Urls + var urlMatch = _urlRegex.Matches(messageContent); + foreach (var m in urlMatch) + { + var url = m.ToString().Replace("\n", string.Empty).Trim(); + + var protocol = "https://"; + if (url.StartsWith("http://")) protocol = "http://"; + else if (url.StartsWith("ftp://")) protocol = "ftp://"; + + var truncatedUrl = url.Replace(protocol, string.Empty); + + if (truncatedUrl.StartsWith("www.")) + { + protocol += "www."; + truncatedUrl = truncatedUrl.Replace("www.", string.Empty); + } + + var firstPart = truncatedUrl; + var secondPart = string.Empty; + + if (truncatedUrl.Length > 30) + { + firstPart = truncatedUrl.Substring(0, 30); + secondPart = truncatedUrl.Substring(30); + } + + messageContent = Regex.Replace(messageContent, m.ToString(), + $@" {protocol}{firstPart}{secondPart}"); + } + + // Extract Hashtags var hashtagMatch = _hastagRegex.Matches(messageContent); foreach (var m in hashtagMatch) { @@ -49,6 +82,7 @@ namespace BirdsiteLive.Domain.Tools $@" #{tag}"); } + // Extract Mentions var mentionMatch = _mentionRegex.Matches(messageContent); foreach (var m in mentionMatch) { diff --git a/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs index d2f3bc9..790ba62 100644 --- a/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs +++ b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs @@ -22,6 +22,80 @@ namespace BirdsiteLive.Domain.Tests.Tools } #endregion + [TestMethod] + public void Extract_FormatUrl_Test() + { + #region Stubs + var message = $"Bla!{Environment.NewLine}https://t.co/L8BpyHgg25"; + #endregion + + var service = new StatusExtractor(_settings); + var result = service.ExtractTags(message); + + #region Validations + Assert.AreEqual(0, result.tags.Length); + + Assert.IsTrue(result.content.Contains("Bla!")); + Assert.IsTrue(result.content.Contains(@"https://t.co/L8BpyHgg25")); + #endregion + } + + [TestMethod] + public void Extract_FormatUrl_Long_Test() + { + #region Stubs + var message = $"Bla!{Environment.NewLine}https://www.eff.org/deeplinks/2020/07/pact-act-not-solution-problem-harmful-online-content"; + #endregion + + var service = new StatusExtractor(_settings); + var result = service.ExtractTags(message); + + #region Validations + Assert.AreEqual(0, result.tags.Length); + + Assert.IsTrue(result.content.Contains("Bla!")); + Assert.IsTrue(result.content.Contains(@"https://www.eff.org/deeplinks/2020/07/pact-act-not-solution-problem-harmful-online-content")); + #endregion + } + + [TestMethod] + public void Extract_FormatUrl_Exact_Test() + { + #region Stubs + var message = $"Bla!{Environment.NewLine}https://www.eff.org/deeplinks/2020/07/pact"; + #endregion + + var service = new StatusExtractor(_settings); + var result = service.ExtractTags(message); + + #region Validations + Assert.AreEqual(0, result.tags.Length); + + Assert.IsTrue(result.content.Contains("Bla!")); + Assert.IsTrue(result.content.Contains(@"https://www.eff.org/deeplinks/2020/07/pact")); + #endregion + } + + [TestMethod] + public void Extract_MultiUrls__Test() + { + #region Stubs + var message = $"https://t.co/L8BpyHgg25 Bla!{Environment.NewLine}https://www.eff.org/deeplinks/2020/07/pact-act-not-solution-problem-harmful-online-content"; + #endregion + + var service = new StatusExtractor(_settings); + var result = service.ExtractTags(message); + + #region Validations + Assert.AreEqual(0, result.tags.Length); + + Assert.IsTrue(result.content.Contains("Bla!")); + Assert.IsTrue(result.content.Contains(@"https://t.co/L8BpyHgg25")); + + Assert.IsTrue(result.content.Contains(@"https://www.eff.org/deeplinks/2020/07/pact-act-not-solution-problem-harmful-online-content")); + #endregion + } + [TestMethod] public void Extract_SingleHashTag_Test() { From 7a6d854aeff4d469be1ec75455f1cc1f09590abe Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 1 Aug 2020 14:04:08 -0400 Subject: [PATCH 35/67] added mention on RTs --- src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs index 4a0892a..6bc871b 100644 --- a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs +++ b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs @@ -39,7 +39,7 @@ namespace BirdsiteLive.Twitter.Extractors if (tweet.IsRetweet) { if (tweet.RetweetedTweet != null) - message = $"[RT {tweet.RetweetedTweet.CreatedBy.ScreenName}] {tweet.RetweetedTweet.FullText}"; + message = $"[RT @{tweet.RetweetedTweet.CreatedBy.ScreenName}] {tweet.RetweetedTweet.FullText}"; else message = message.Replace("RT", "[RT]"); } From 54fc131d002f0e32fac3ccb0d12e997d197be013 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 1 Aug 2020 16:42:32 -0400 Subject: [PATCH 36/67] fix date --- src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs index 6bc871b..2cb7dde 100644 --- a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs +++ b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.IO; using System.Linq; using BirdsiteLive.Twitter.Models; @@ -23,7 +24,7 @@ namespace BirdsiteLive.Twitter.Extractors InReplyToAccount = tweet.InReplyToScreenName, MessageContent = ExtractMessage(tweet), Media = ExtractMedia(tweet.Media), - CreatedAt = tweet.CreatedAt + CreatedAt = tweet.CreatedAt.ToUniversalTime() }; return extractedTweet; } @@ -35,11 +36,11 @@ namespace BirdsiteLive.Twitter.Extractors foreach (var tweetUrl in tweetUrls) message = message.Replace(tweetUrl, string.Empty).Trim(); - if (tweet.QuotedTweet != null) message = $"[Quote RT] {message}"; + if (tweet.QuotedTweet != null) message = $"[Quote RT]{Environment.NewLine}{message}"; if (tweet.IsRetweet) { if (tweet.RetweetedTweet != null) - message = $"[RT @{tweet.RetweetedTweet.CreatedBy.ScreenName}] {tweet.RetweetedTweet.FullText}"; + message = $"[RT @{tweet.RetweetedTweet.CreatedBy.ScreenName}]{Environment.NewLine}{tweet.RetweetedTweet.FullText}"; else message = message.Replace("RT", "[RT]"); } From f9bb6b153ca4f90fdc3867250e06576d2eaab6f8 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 1 Aug 2020 19:26:00 -0400 Subject: [PATCH 37/67] make sure pipeline error are seen --- src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs index 0e0da40..bcb896f 100644 --- a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs +++ b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs @@ -42,16 +42,16 @@ namespace BirdsiteLive.Pipeline // Link pipeline twitterUsersBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions {PropagateCompletion = true}); - retrieveTweetsBlock.LinkTo(retrieveTweetsBufferBlock); - retrieveTweetsBufferBlock.LinkTo(retrieveFollowersBlock); - retrieveFollowersBlock.LinkTo(retrieveFollowersBufferBlock); - retrieveFollowersBufferBlock.LinkTo(sendTweetsToFollowersBlock); + retrieveTweetsBlock.LinkTo(retrieveTweetsBufferBlock, new DataflowLinkOptions { PropagateCompletion = true }); + retrieveTweetsBufferBlock.LinkTo(retrieveFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true }); + retrieveFollowersBlock.LinkTo(retrieveFollowersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true }); + retrieveFollowersBufferBlock.LinkTo(sendTweetsToFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true }); // Launch twitter user retriever var retrieveTwitterAccountsTask = _retrieveTwitterAccountsProcessor.GetTwitterUsersAsync(twitterUsersBufferBlock, ct); // Wait - await Task.WhenAll(retrieveTwitterAccountsTask, sendTweetsToFollowersBlock.Completion); + await Task.WhenAny(new []{ retrieveTwitterAccountsTask , sendTweetsToFollowersBlock.Completion}); var foreground = Console.ForegroundColor; Console.ForegroundColor = ConsoleColor.Red; From 31cd662db9f8206b07df1e63fcaaf86aa6a1b19f Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 1 Aug 2020 19:26:33 -0400 Subject: [PATCH 38/67] make following only statuses --- src/BirdsiteLive.Domain/StatusService.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BirdsiteLive.Domain/StatusService.cs b/src/BirdsiteLive.Domain/StatusService.cs index 51cddfc..54a95ca 100644 --- a/src/BirdsiteLive.Domain/StatusService.cs +++ b/src/BirdsiteLive.Domain/StatusService.cs @@ -60,8 +60,8 @@ namespace BirdsiteLive.Domain //cc = new [] { apPublic }, to = new[] { to }, - cc = new[] { apPublic }, - //cc = new string[0], + //cc = new[] { apPublic }, + cc = new string[0], sensitive = false, content = $"

{extractedTags.content}

", From a9a2fad375e5a3f5a2ce8f9fcfda168c53dcec20 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sun, 2 Aug 2020 19:40:02 -0400 Subject: [PATCH 39/67] added return lines parsing --- src/BirdsiteLive.Domain/Tools/StatusExtractor.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs index 722f369..a1c0245 100644 --- a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs +++ b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs @@ -33,6 +33,10 @@ namespace BirdsiteLive.Domain.Tools var tags = new List(); messageContent = $" {messageContent} "; + // Replace return lines + messageContent = Regex.Replace(messageContent, @"\r\n\r\n?|\n\n", "

"); + messageContent = Regex.Replace(messageContent, @"\r\n?|\n", "
"); + // Extract Urls var urlMatch = _urlRegex.Matches(messageContent); foreach (var m in urlMatch) From 0354bce8e9f18fea42e08f328ad3294000416a24 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sun, 2 Aug 2020 20:21:59 -0400 Subject: [PATCH 40/67] added welcome page --- .../Controllers/HomeController.cs | 6 +++++ src/BirdsiteLive/Views/Home/Index.cshtml | 23 ++++++++++++++++--- src/BirdsiteLive/Views/Shared/_Layout.cshtml | 10 ++++---- .../lib/bootstrap/dist/css/bootstrap.css | 12 ++++++++++ 4 files changed, 43 insertions(+), 8 deletions(-) diff --git a/src/BirdsiteLive/Controllers/HomeController.cs b/src/BirdsiteLive/Controllers/HomeController.cs index 7270240..ef41b65 100644 --- a/src/BirdsiteLive/Controllers/HomeController.cs +++ b/src/BirdsiteLive/Controllers/HomeController.cs @@ -33,5 +33,11 @@ namespace BirdsiteLive.Controllers { return View(new ErrorViewModel { RequestId = Activity.Current?.Id ?? HttpContext.TraceIdentifier }); } + + [HttpPost] + public IActionResult Index(string handle) + { + return RedirectToAction("Index", "Users", new {id = handle}); + } } } diff --git a/src/BirdsiteLive/Views/Home/Index.cshtml b/src/BirdsiteLive/Views/Home/Index.cshtml index 4c62fc0..866a295 100644 --- a/src/BirdsiteLive/Views/Home/Index.cshtml +++ b/src/BirdsiteLive/Views/Home/Index.cshtml @@ -5,11 +5,28 @@

Welcome

-

Learn about building Web apps with ASP.NET Core.

+

+
+ BirdsiteLIVE is a Twitter to ActivityPub bridge.
+ Find a Twitter account below: +

+ + + @*
+ + + We'll never share your email with anyone else. +
*@ +
+ @**@ + +
+ + - @if (HtmlHelperExtensions.IsDebug()) + @*@if (HtmlHelperExtensions.IsDebug()) { Debug - } + }*@
diff --git a/src/BirdsiteLive/Views/Shared/_Layout.cshtml b/src/BirdsiteLive/Views/Shared/_Layout.cshtml index 426caa2..633fd0d 100644 --- a/src/BirdsiteLive/Views/Shared/_Layout.cshtml +++ b/src/BirdsiteLive/Views/Shared/_Layout.cshtml @@ -3,7 +3,7 @@ - @ViewData["Title"] - BirdsiteLive + @ViewData["Title"] - BirdsiteLIVE @@ -12,8 +12,8 @@
@@ -38,7 +38,7 @@ diff --git a/src/BirdsiteLive/wwwroot/lib/bootstrap/dist/css/bootstrap.css b/src/BirdsiteLive/wwwroot/lib/bootstrap/dist/css/bootstrap.css index 8f47589..6f49c34 100644 --- a/src/BirdsiteLive/wwwroot/lib/bootstrap/dist/css/bootstrap.css +++ b/src/BirdsiteLive/wwwroot/lib/bootstrap/dist/css/bootstrap.css @@ -419,6 +419,18 @@ h6, .h6 { line-height: 1.2; } +.display-5 { + font-size: 2.5rem; + font-weight: 100; + line-height: 1.2; +} + +.display-6 { + font-size: 1.5rem; + font-weight: 300; + line-height: 1.2; +} + hr { margin-top: 1rem; margin-bottom: 1rem; From cb0d0db441d1ac2a0610918d2c02b58a7f4dae11 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sun, 2 Aug 2020 21:04:38 -0400 Subject: [PATCH 41/67] better profile display --- src/BirdsiteLive/Views/Users/Index.cshtml | 34 ++++++++--- src/BirdsiteLive/wwwroot/css/birdsite.css | 69 ++++++++++++++++------- 2 files changed, 74 insertions(+), 29 deletions(-) diff --git a/src/BirdsiteLive/Views/Users/Index.cshtml b/src/BirdsiteLive/Views/Users/Index.cshtml index 97344a8..2915493 100644 --- a/src/BirdsiteLive/Views/Users/Index.cshtml +++ b/src/BirdsiteLive/Views/Users/Index.cshtml @@ -3,16 +3,32 @@ ViewData["Title"] = "User"; } -
-
- -

@ViewData.Model.Name

 -

@@@ViewData.Model.Acct

+
+ + @*
*@ +
+ +
+ + + + +

@ViewData.Model.Name @@@ViewData.Model.Acct

+ + @*

@@@ViewData.Model.Acct

*@ -
- @ViewData.Model.Description +
+ @ViewData.Model.Description +
+
+
-
-
+ +
+

Search this handle to find it in your instance:

+ +
\ No newline at end of file diff --git a/src/BirdsiteLive/wwwroot/css/birdsite.css b/src/BirdsiteLive/wwwroot/css/birdsite.css index 91b0dc6..c18719f 100644 --- a/src/BirdsiteLive/wwwroot/css/birdsite.css +++ b/src/BirdsiteLive/wwwroot/css/birdsite.css @@ -1,43 +1,72 @@ -.profile { - border: 1px #dddddd solid; - border-radius: 5px; - background-repeat: no-repeat; - /*background-attachment: fixed;*/ - background-position: center; +.nounderline { + text-decoration: none !important } +.logo { + width: 25px; + height: 25px; + float: right; +} + +.logo-twitter { + filter: invert(51%) sepia(92%) saturate(1166%) hue-rotate(180deg) brightness(94%) contrast(98%); + /*background: #349fef;*/ +} + +.profile { + border: 1px #dddddd solid; + border-radius: 15px; + /*background-repeat: no-repeat;*/ + /*background-attachment: fixed;*/ + /*background-position: center;*/ + color: black; +} + + .profile:hover { + transition: all .2s; + background-color: #f5f8fa; + } + .profile h1 { - font-size: 32px; - margin-left: 120px; - padding-top: 8px; + font-size: 18px; + margin-left: 60px; + padding-top: 0px; } .profile h2 { font-size: 20px; - margin-left: 120px; + margin-left: 0px; } - .profile a { +/*.profile a { color: black; } - .profile a:hover { - color: #555555; - } + .profile a:hover { + color: #555555; + }*/ + +.handle { + color: gray; + font-weight: normal; +} .sub-profile { - padding: 20px; - background-color: rgba(255, 255, 255, 0.7); + padding: 10px 15px; } +/*.sub-profile a { + color: black; + }*/ + .avatar { float: left; - width: 100px; + width: 50px; border-radius: 50%; } .description { - margin-top: 40px; - margin-left: 20px; - font-weight: bold; + margin-top: 0px; + margin-left: 60px; + /*font-weight: bold;*/ } From 9f574ea4b24c8fd3e43d5c1e6796f15177130dda Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Mon, 3 Aug 2020 02:10:20 -0400 Subject: [PATCH 42/67] fix return line parsing --- .../Tools/StatusExtractor.cs | 33 +++++++++++++++---- .../Tools/StatusExtractorTests.cs | 32 ++++++++++++++++++ 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs index a1c0245..d78bf6e 100644 --- a/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs +++ b/src/BirdsiteLive.Domain/Tools/StatusExtractor.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Linq; using System.Text.RegularExpressions; using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.Common.Settings; @@ -13,11 +14,15 @@ namespace BirdsiteLive.Domain.Tools public class StatusExtractor : IStatusExtractor { private readonly Regex _hastagRegex = new Regex(@"\W(\#[a-zA-Z0-9_ー]+\b)(?!;)"); + //private readonly Regex _hastagRegex = new Regex(@"#\w+"); //private readonly Regex _hastagRegex = new Regex(@"(?<=[\s>]|^)#(\w*[a-zA-Z0-9_ー]+\w*)\b(?!;)"); //private readonly Regex _hastagRegex = new Regex(@"(?<=[\s>]|^)#(\w*[a-zA-Z0-9_ー]+)\b(?!;)"); + private readonly Regex _mentionRegex = new Regex(@"\W(\@[a-zA-Z0-9_ー]+\b)(?!;)"); + //private readonly Regex _mentionRegex = new Regex(@"@\w+"); //private readonly Regex _mentionRegex = new Regex(@"(?<=[\s>]|^)@(\w*[a-zA-Z0-9_ー]+\w*)\b(?!;)"); //private readonly Regex _mentionRegex = new Regex(@"(?<=[\s>]|^)@(\w*[a-zA-Z0-9_ー]+)\b(?!;)"); + private readonly Regex _urlRegex = new Regex(@"((http|ftp|https):\/\/[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])?)"); private readonly InstanceSettings _instanceSettings; @@ -34,12 +39,12 @@ namespace BirdsiteLive.Domain.Tools messageContent = $" {messageContent} "; // Replace return lines - messageContent = Regex.Replace(messageContent, @"\r\n\r\n?|\n\n", "

"); - messageContent = Regex.Replace(messageContent, @"\r\n?|\n", "
"); + messageContent = Regex.Replace(messageContent, @"\r\n\r\n?|\n\n", "

"); + messageContent = Regex.Replace(messageContent, @"\r\n?|\n", "
"); // Extract Urls var urlMatch = _urlRegex.Matches(messageContent); - foreach (var m in urlMatch) + foreach (Match m in urlMatch) { var url = m.ToString().Replace("\n", string.Empty).Trim(); @@ -69,8 +74,8 @@ namespace BirdsiteLive.Domain.Tools } // Extract Hashtags - var hashtagMatch = _hastagRegex.Matches(messageContent); - foreach (var m in hashtagMatch) + var hashtagMatch = OrderByLength(_hastagRegex.Matches(messageContent)); + foreach (Match m in hashtagMatch) { var tag = m.ToString().Replace("#", string.Empty).Replace("\n", string.Empty).Trim(); var url = $"https://{_instanceSettings.Domain}/tags/{tag}"; @@ -87,8 +92,8 @@ namespace BirdsiteLive.Domain.Tools } // Extract Mentions - var mentionMatch = _mentionRegex.Matches(messageContent); - foreach (var m in mentionMatch) + var mentionMatch = OrderByLength(_mentionRegex.Matches(messageContent)); + foreach (Match m in mentionMatch) { var mention = m.ToString().Replace("@", string.Empty).Replace("\n", string.Empty).Trim(); var url = $"https://{_instanceSettings.Domain}/users/{mention}"; @@ -105,7 +110,21 @@ namespace BirdsiteLive.Domain.Tools $@" @{mention}"); } + // Clean up return lines + messageContent = Regex.Replace(messageContent, @"

", "

"); + messageContent = Regex.Replace(messageContent, @"
", "
"); + return (messageContent.Trim(), tags.ToArray()); } + + private IEnumerable OrderByLength(MatchCollection matches) + { + var result = new List(); + + foreach (Match m in matches) result.Add(m); + result = result.OrderByDescending(x => x.Length).ToList(); + + return result; + } } } \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs index 790ba62..5728cc5 100644 --- a/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs +++ b/src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs @@ -22,6 +22,38 @@ namespace BirdsiteLive.Domain.Tests.Tools } #endregion + [TestMethod] + public void Extract_ReturnLines_Test() + { + #region Stubs + var message = "Bla.\n\n@Mention blo. https://t.co/pgtrJi9600"; + #endregion + + var service = new StatusExtractor(_settings); + var result = service.ExtractTags(message); + + #region Validations + Assert.IsTrue(result.content.Contains("Bla.")); + Assert.IsTrue(result.content.Contains("

")); + #endregion + } + + [TestMethod] + public void Extract_ReturnSingleLines_Test() + { + #region Stubs + var message = "Bla.\n@Mention blo. https://t.co/pgtrJi9600"; + #endregion + + var service = new StatusExtractor(_settings); + var result = service.ExtractTags(message); + + #region Validations + Assert.IsTrue(result.content.Contains("Bla.")); + Assert.IsTrue(result.content.Contains("
")); + #endregion + } + [TestMethod] public void Extract_FormatUrl_Test() { From d5276c120e7aed416b3410f530d0ddec89e47ebe Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Mon, 10 Aug 2020 20:04:12 -0400 Subject: [PATCH 43/67] added shared inbox serialization --- src/BirdsiteLive.Domain/ActivityPubService.cs | 19 ----- .../BusinessUseCases/ProcessFollowUser.cs | 6 +- src/BirdsiteLive.Domain/UserService.cs | 20 ++++- .../SendTweetsToFollowersProcessor.cs | 9 ++- .../DbInitializerPostgresDal.cs | 3 +- .../DataAccessLayers/FollowersPostgresDal.cs | 12 +-- .../Contracts/IFollowersDal.cs | 2 +- .../BirdsiteLive.DAL/Models/Follower.cs | 3 +- .../FollowersPostgresDalTests.cs | 75 ++++++++++++++----- 9 files changed, 97 insertions(+), 52 deletions(-) diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs index 043a7fd..64f5705 100644 --- a/src/BirdsiteLive.Domain/ActivityPubService.cs +++ b/src/BirdsiteLive.Domain/ActivityPubService.cs @@ -73,27 +73,8 @@ namespace BirdsiteLive.Domain to = note.to, cc = note.cc, apObject = note - //apObject = new Note() - //{ - // id = noteUri, - // summary = null, - // inReplyTo = null, - // published = nowString, - // url = noteUrl, - // attributedTo = actor, - // to = new[] { to }, - // //cc = new [] { apPublic }, - // sensitive = false, - // content = "

Woooot

", - // attachment = new string[0], - // tag = new string[0] - //} }; - //TODO Remove this - if (targetInbox.Contains(targetHost)) - targetInbox = targetInbox.Split(new []{ targetHost }, StringSplitOptions.RemoveEmptyEntries).Last(); - return await PostDataAsync(noteActivity, targetHost, actor, targetInbox); } diff --git a/src/BirdsiteLive.Domain/BusinessUseCases/ProcessFollowUser.cs b/src/BirdsiteLive.Domain/BusinessUseCases/ProcessFollowUser.cs index ce50673..ac657e4 100644 --- a/src/BirdsiteLive.Domain/BusinessUseCases/ProcessFollowUser.cs +++ b/src/BirdsiteLive.Domain/BusinessUseCases/ProcessFollowUser.cs @@ -5,7 +5,7 @@ namespace BirdsiteLive.Domain.BusinessUseCases { public interface IProcessFollowUser { - Task ExecuteAsync(string followerUsername, string followerDomain, string followerInbox, string twitterUser); + Task ExecuteAsync(string followerUsername, string followerDomain, string twitterUsername, string followerInbox, string sharedInbox); } public class ProcessFollowUser : IProcessFollowUser @@ -21,13 +21,13 @@ namespace BirdsiteLive.Domain.BusinessUseCases } #endregion - public async Task ExecuteAsync(string followerUsername, string followerDomain, string followerInbox, string twitterUsername) + public async Task ExecuteAsync(string followerUsername, string followerDomain, string twitterUsername, string followerInbox, string sharedInbox) { // Get Follower and Twitter Users var follower = await _followerDal.GetFollowerAsync(followerUsername, followerDomain); if (follower == null) { - await _followerDal.CreateFollowerAsync(followerUsername, followerDomain, followerInbox); + await _followerDal.CreateFollowerAsync(followerUsername, followerDomain, followerInbox, sharedInbox); follower = await _followerDal.GetFollowerAsync(followerUsername, followerDomain); } diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index f7cf448..fa2f5aa 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -88,8 +88,15 @@ namespace BirdsiteLive.Domain var followerUserName = sigValidation.User.name.ToLowerInvariant(); var followerHost = sigValidation.User.url.Replace("https://", string.Empty).Split('/').First(); var followerInbox = sigValidation.User.inbox; + var followerSharedInbox = sigValidation.User?.endpoints?.sharedInbox; var twitterUser = activity.apObject.Split('/').Last().Replace("@", string.Empty); - await _processFollowUser.ExecuteAsync(followerUserName, followerHost, followerInbox, twitterUser); + + // Make sure to only keep routes + followerInbox = OnlyKeepRoute(followerInbox, followerHost); + followerSharedInbox = OnlyKeepRoute(followerSharedInbox, followerHost); + + // Execute + await _processFollowUser.ExecuteAsync(followerUserName, followerHost, twitterUser, followerInbox, followerSharedInbox); // Send Accept Activity var acceptFollow = new ActivityAcceptFollow() @@ -110,6 +117,17 @@ namespace BirdsiteLive.Domain return result == HttpStatusCode.Accepted; } + private string OnlyKeepRoute(string inbox, string host) + { + if (string.IsNullOrWhiteSpace(inbox)) + return null; + + if (inbox.Contains(host)) + inbox = inbox.Split(new[] { host }, StringSplitOptions.RemoveEmptyEntries).Last(); + + return inbox; + } + public async Task UndoFollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityUndoFollow activity) { diff --git a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs index 3dd37cd..7a06c8a 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs @@ -51,11 +51,13 @@ namespace BirdsiteLive.Pipeline.Processors return userWithTweetsToSync; } - private async Task ProcessFollowerAsync(IEnumerable tweets, Follower follower, int userId, - SyncTwitterUser user) + private async Task ProcessFollowerAsync(IEnumerable tweets, Follower follower, int userId, SyncTwitterUser user) { var fromStatusId = follower.FollowingsSyncStatus[userId]; var tweetsToSend = tweets.Where(x => x.Id > fromStatusId).OrderBy(x => x.Id).ToList(); + var inbox = string.IsNullOrWhiteSpace(follower.SharedInboxRoute) + ? follower.InboxRoute + : follower.SharedInboxRoute; var syncStatus = fromStatusId; try @@ -63,8 +65,7 @@ namespace BirdsiteLive.Pipeline.Processors foreach (var tweet in tweetsToSend) { var note = _statusService.GetStatus(user.Acct, tweet); - var result = await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, - follower.InboxUrl); + var result = await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox); if (result == HttpStatusCode.Accepted) syncStatus = tweet.Id; diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs index 832eaf4..c6d5c81 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs @@ -108,7 +108,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers acct VARCHAR(50) NOT NULL, host VARCHAR(253) NOT NULL, - inboxUrl VARCHAR(2048) NOT NULL, + inboxRoute VARCHAR(2048) NOT NULL, + sharedInboxRoute VARCHAR(2048), UNIQUE (acct, host) );"; await _tools.ExecuteRequestAsync(createFollowers); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs index 0ec78dc..961aa7c 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs @@ -20,7 +20,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } #endregion - public async Task CreateFollowerAsync(string acct, string host, string inboxUrl, int[] followings = null, Dictionary followingSyncStatus = null) + public async Task CreateFollowerAsync(string acct, string host, string inboxRoute, string sharedInboxRoute, int[] followings = null, Dictionary followingSyncStatus = null) { if(followings == null) followings = new int[0]; if(followingSyncStatus == null) followingSyncStatus = new Dictionary(); @@ -35,8 +35,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers dbConnection.Open(); await dbConnection.ExecuteAsync( - $"INSERT INTO {_settings.FollowersTableName} (acct,host,inboxUrl,followings,followingsSyncStatus) VALUES(@acct,@host,@inboxUrl,@followings,CAST(@followingsSyncStatus as json))", - new { acct, host, inboxUrl, followings, followingsSyncStatus = serializedDic }); + $"INSERT INTO {_settings.FollowersTableName} (acct,host,inboxRoute,sharedInboxRoute,followings,followingsSyncStatus) VALUES(@acct,@host,@inboxRoute,@sharedInboxRoute,@followings,CAST(@followingsSyncStatus as json))", + new { acct, host, inboxRoute, sharedInboxRoute, followings, followingsSyncStatus = serializedDic }); } } @@ -128,7 +128,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers Id = follower.Id, Acct = follower.Acct, Host = follower.Host, - InboxUrl = follower.InboxUrl, + InboxRoute = follower.InboxRoute, + SharedInboxRoute = follower.SharedInboxRoute, Followings = follower.Followings.ToList(), FollowingsSyncStatus = JsonConvert.DeserializeObject>(follower.FollowingsSyncStatus) }; @@ -143,6 +144,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers public string Acct { get; set; } public string Host { get; set; } - public string InboxUrl { get; set; } + public string InboxRoute { get; set; } + public string SharedInboxRoute { get; set; } } } \ No newline at end of file diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs index f7108a0..8b5e6e1 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs @@ -7,7 +7,7 @@ namespace BirdsiteLive.DAL.Contracts public interface IFollowersDal { Task GetFollowerAsync(string acct, string host); - Task CreateFollowerAsync(string acct, string host, string inboxUrl, int[] followings = null, + Task CreateFollowerAsync(string acct, string host, string inboxRoute, string sharedInboxRoute, int[] followings = null, Dictionary followingSyncStatus = null); Task GetFollowersAsync(int followedUserId); Task UpdateFollowerAsync(Follower follower); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Models/Follower.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Models/Follower.cs index 2499263..8fbc97b 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Models/Follower.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Models/Follower.cs @@ -11,6 +11,7 @@ namespace BirdsiteLive.DAL.Models public string Acct { get; set; } public string Host { get; set; } - public string InboxUrl { get; set; } + public string InboxRoute { get; set; } + public string SharedInboxRoute { get; set; } } } \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs index cdb0dc0..e12d08a 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs @@ -45,17 +45,51 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers {19, 166L}, {23, 167L} }; - var inboxUrl = "https://domain.ext/myhandle/inbox"; + var inboxRoute = "/myhandle/inbox"; + var sharedInboxRoute = "/inbox"; var dal = new FollowersPostgresDal(_settings); - await dal.CreateFollowerAsync(acct, host, inboxUrl, following, followingSync); + await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, following, followingSync); var result = await dal.GetFollowerAsync(acct, host); Assert.IsNotNull(result); Assert.AreEqual(acct, result.Acct); Assert.AreEqual(host, result.Host); - Assert.AreEqual(inboxUrl, result.InboxUrl); + Assert.AreEqual(inboxRoute, result.InboxRoute); + Assert.AreEqual(sharedInboxRoute, result.SharedInboxRoute); + Assert.AreEqual(following.Length, result.Followings.Count); + Assert.AreEqual(following[0], result.Followings[0]); + Assert.AreEqual(followingSync.Count, result.FollowingsSyncStatus.Count); + Assert.AreEqual(followingSync.First().Key, result.FollowingsSyncStatus.First().Key); + Assert.AreEqual(followingSync.First().Value, result.FollowingsSyncStatus.First().Value); + } + + [TestMethod] + public async Task CreateAndGetFollower_NoSharedInbox() + { + var acct = "myhandle"; + var host = "domain.ext"; + var following = new[] { 12, 19, 23 }; + var followingSync = new Dictionary() + { + {12, 165L}, + {19, 166L}, + {23, 167L} + }; + var inboxRoute = "/myhandle/inbox"; + string sharedInboxRoute = null; + + var dal = new FollowersPostgresDal(_settings); + await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, following, followingSync); + + var result = await dal.GetFollowerAsync(acct, host); + + Assert.IsNotNull(result); + Assert.AreEqual(acct, result.Acct); + Assert.AreEqual(host, result.Host); + Assert.AreEqual(inboxRoute, result.InboxRoute); + Assert.AreEqual(sharedInboxRoute, result.SharedInboxRoute); Assert.AreEqual(following.Length, result.Followings.Count); Assert.AreEqual(following[0], result.Followings[0]); Assert.AreEqual(followingSync.Count, result.FollowingsSyncStatus.Count); @@ -73,22 +107,25 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers var host = "domain.ext"; var following = new[] { 1,2,3 }; var followingSync = new Dictionary(); - var inboxUrl = "https://domain.ext/myhandle1/inbox"; - await dal.CreateFollowerAsync(acct, host, inboxUrl, following, followingSync); + var inboxRoute = "/myhandle1/inbox"; + var sharedInboxRoute = "/inbox"; + await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, following, followingSync); //User 2 acct = "myhandle2"; host = "domain.ext"; following = new[] { 2, 4, 5 }; - inboxUrl = "https://domain.ext/myhandle2/inbox"; - await dal.CreateFollowerAsync(acct, host, inboxUrl, following, followingSync); + inboxRoute = "/myhandle2/inbox"; + sharedInboxRoute = "/inbox2"; + await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, following, followingSync); //User 2 acct = "myhandle3"; host = "domain.ext"; following = new[] { 1 }; - inboxUrl = "https://domain.ext/myhandle3/inbox"; - await dal.CreateFollowerAsync(acct, host, inboxUrl, following, followingSync); + inboxRoute = "/myhandle3/inbox"; + sharedInboxRoute = "/inbox3"; + await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, following, followingSync); var result = await dal.GetFollowersAsync(2); Assert.AreEqual(2, result.Length); @@ -112,10 +149,11 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers {19, 166L}, {23, 167L} }; - var inboxUrl = "https://domain.ext/myhandle/inbox"; + var inboxRoute = "/myhandle/inbox"; + var sharedInboxRoute = "/inbox"; var dal = new FollowersPostgresDal(_settings); - await dal.CreateFollowerAsync(acct, host, inboxUrl, following, followingSync); + await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, following, followingSync); var result = await dal.GetFollowerAsync(acct, host); var updatedFollowing = new List { 12, 19, 23, 24 }; @@ -151,10 +189,11 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers {19, 166L}, {23, 167L} }; - var inboxUrl = "https://domain.ext/myhandle/inbox"; + var inboxRoute = "/myhandle/inbox"; + var sharedInboxRoute = "/inbox"; var dal = new FollowersPostgresDal(_settings); - await dal.CreateFollowerAsync(acct, host, inboxUrl, following, followingSync); + await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, following, followingSync); var result = await dal.GetFollowerAsync(acct, host); var updatedFollowing = new[] { 12, 19 }; @@ -188,10 +227,11 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers {19, 166L}, {23, 167L} }; - var inboxUrl = "https://domain.ext/myhandle/inbox"; + var inboxRoute = "/myhandle/inbox"; + var sharedInboxRoute = "/inbox"; var dal = new FollowersPostgresDal(_settings); - await dal.CreateFollowerAsync(acct, host, inboxUrl, following, followingSync); + await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, following, followingSync); var result = await dal.GetFollowerAsync(acct, host); Assert.IsNotNull(result); @@ -213,10 +253,11 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers {19, 166L}, {23, 167L} }; - var inboxUrl = "https://domain.ext/myhandle/inbox"; + var inboxRoute = "/myhandle/inbox"; + var sharedInboxRoute = "/inbox"; var dal = new FollowersPostgresDal(_settings); - await dal.CreateFollowerAsync(acct, host, inboxUrl, following, followingSync); + await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, following, followingSync); var result = await dal.GetFollowerAsync(acct, host); Assert.IsNotNull(result); From afa05a72d203ff6b44e913f90f07ee61eb3a2c0c Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 12 Aug 2020 18:34:01 -0400 Subject: [PATCH 44/67] added shared inbox publication --- .../SendTweetsToFollowersProcessor.cs | 105 ++++++++++++++++-- 1 file changed, 94 insertions(+), 11 deletions(-) diff --git a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs index 7a06c8a..8c5f6b9 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Net; using System.Threading; using System.Threading.Tasks; +using System.Xml; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Domain; @@ -33,13 +34,31 @@ namespace BirdsiteLive.Pipeline.Processors public async Task ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, CancellationToken ct) { var user = userWithTweetsToSync.User; - var userId = user.Id; - foreach (var follower in userWithTweetsToSync.Followers) + // Process Shared Inbox + var followersWtSharedInbox = userWithTweetsToSync.Followers + .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute)) + .ToList(); + await ProcessFollowersWithSharedInbox(userWithTweetsToSync.Tweets, followersWtSharedInbox, user); + + // Process Inbox + var followerWtInbox = userWithTweetsToSync.Followers + .Where(x => string.IsNullOrWhiteSpace(x.SharedInboxRoute)) + .ToList(); + await ProcessFollowersWithInbox(userWithTweetsToSync.Tweets, followerWtInbox, user); + + return userWithTweetsToSync; + } + + private async Task ProcessFollowersWithSharedInbox(ExtractedTweet[] tweets, List followers, SyncTwitterUser user) + { + var followersPerInstances = followers.GroupBy(x => x.Host); + + foreach (var followersPerInstance in followersPerInstances) { try { - await ProcessFollowerAsync(userWithTweetsToSync.Tweets, follower, userId, user); + await ProcessInstanceFollowersWithSharedInbox(tweets, user, followersPerInstance); } catch (Exception e) { @@ -47,17 +66,81 @@ namespace BirdsiteLive.Pipeline.Processors //TODO handle error } } - - return userWithTweetsToSync; } - private async Task ProcessFollowerAsync(IEnumerable tweets, Follower follower, int userId, SyncTwitterUser user) + private async Task ProcessInstanceFollowersWithSharedInbox(ExtractedTweet[] tweets, SyncTwitterUser user, + IGrouping followersPerInstance) { + var userId = user.Id; + var host = followersPerInstance.Key; + var groupedFollowers = followersPerInstance.ToList(); + var inbox = groupedFollowers.First().SharedInboxRoute; + + var fromStatusId = groupedFollowers + .Max(x => x.FollowingsSyncStatus[userId]); + + var tweetsToSend = tweets + .Where(x => x.Id > fromStatusId) + .OrderBy(x => x.Id) + .ToList(); + + var syncStatus = fromStatusId; + try + { + foreach (var tweet in tweetsToSend) + { + var note = _statusService.GetStatus(user.Acct, tweet); + var result = + await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox); + + if (result == HttpStatusCode.Accepted) + syncStatus = tweet.Id; + else + throw new Exception("Posting new note activity failed"); + } + } + finally + { + if (syncStatus != fromStatusId) + { + foreach (var f in groupedFollowers) + { + f.FollowingsSyncStatus[userId] = syncStatus; + await _followersDal.UpdateFollowerAsync(f); + } + } + } + } + + private async Task ProcessFollowersWithInbox(ExtractedTweet[] tweets, List followerWtInbox, SyncTwitterUser user) + { + foreach (var follower in followerWtInbox) + { + try + { + await ProcessFollowerWithInboxAsync(tweets, follower, user); + } + catch (Exception e) + { + Console.WriteLine(e); + //TODO handle error + } + } + } + + private async Task ProcessFollowerWithInboxAsync(IEnumerable tweets, Follower follower, SyncTwitterUser user) + { + var userId = user.Id; var fromStatusId = follower.FollowingsSyncStatus[userId]; - var tweetsToSend = tweets.Where(x => x.Id > fromStatusId).OrderBy(x => x.Id).ToList(); - var inbox = string.IsNullOrWhiteSpace(follower.SharedInboxRoute) - ? follower.InboxRoute - : follower.SharedInboxRoute; + var tweetsToSend = tweets + .Where(x => x.Id > fromStatusId) + .OrderBy(x => x.Id) + .ToList(); + + var inbox = follower.InboxRoute; + //var inbox = string.IsNullOrWhiteSpace(follower.SharedInboxRoute) + // ? follower.InboxRoute + // : follower.SharedInboxRoute; var syncStatus = fromStatusId; try @@ -67,7 +150,7 @@ namespace BirdsiteLive.Pipeline.Processors var note = _statusService.GetStatus(user.Acct, tweet); var result = await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox); - if (result == HttpStatusCode.Accepted) + if (result == HttpStatusCode.Accepted) syncStatus = tweet.Id; else throw new Exception("Posting new note activity failed"); From 4436b533190194d5cd0ed82bb130974093cbadb3 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 12 Aug 2020 19:05:01 -0400 Subject: [PATCH 45/67] refactorization of sendtweets pipeline stage --- .../BirdsiteLive.Pipeline.csproj | 2 +- .../SendTweetsToFollowersProcessor.cs | 101 ++---------------- .../SubTasks/SendTweetsToInboxTask.cs | 71 ++++++++++++ .../SubTasks/SendTweetsToSharedInboxTask.cs | 75 +++++++++++++ 4 files changed, 156 insertions(+), 93 deletions(-) create mode 100644 src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs create mode 100644 src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs diff --git a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj index 89f6d5d..6b8b510 100644 --- a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj +++ b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj @@ -1,4 +1,4 @@ - + netstandard2.0 diff --git a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs index 8c5f6b9..29bfb12 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs @@ -10,6 +10,7 @@ using BirdsiteLive.DAL.Models; using BirdsiteLive.Domain; using BirdsiteLive.Pipeline.Contracts; using BirdsiteLive.Pipeline.Models; +using BirdsiteLive.Pipeline.Processors.SubTasks; using BirdsiteLive.Twitter; using BirdsiteLive.Twitter.Models; using Tweetinvi.Models; @@ -18,16 +19,14 @@ namespace BirdsiteLive.Pipeline.Processors { public class SendTweetsToFollowersProcessor : ISendTweetsToFollowersProcessor { - private readonly IActivityPubService _activityPubService; - private readonly IStatusService _statusService; - private readonly IFollowersDal _followersDal; + private readonly ISendTweetsToInboxTask _sendTweetsToInboxTask; + private readonly ISendTweetsToSharedInboxTask _sendTweetsToSharedInbox; #region Ctor - public SendTweetsToFollowersProcessor(IActivityPubService activityPubService, IFollowersDal followersDal, IStatusService statusService) + public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox) { - _activityPubService = activityPubService; - _followersDal = followersDal; - _statusService = statusService; + _sendTweetsToInboxTask = sendTweetsToInboxTask; + _sendTweetsToSharedInbox = sendTweetsToSharedInbox; } #endregion @@ -58,7 +57,7 @@ namespace BirdsiteLive.Pipeline.Processors { try { - await ProcessInstanceFollowersWithSharedInbox(tweets, user, followersPerInstance); + await _sendTweetsToSharedInbox.ExecuteAsync(tweets, user, followersPerInstance); } catch (Exception e) { @@ -67,58 +66,14 @@ namespace BirdsiteLive.Pipeline.Processors } } } - - private async Task ProcessInstanceFollowersWithSharedInbox(ExtractedTweet[] tweets, SyncTwitterUser user, - IGrouping followersPerInstance) - { - var userId = user.Id; - var host = followersPerInstance.Key; - var groupedFollowers = followersPerInstance.ToList(); - var inbox = groupedFollowers.First().SharedInboxRoute; - - var fromStatusId = groupedFollowers - .Max(x => x.FollowingsSyncStatus[userId]); - - var tweetsToSend = tweets - .Where(x => x.Id > fromStatusId) - .OrderBy(x => x.Id) - .ToList(); - - var syncStatus = fromStatusId; - try - { - foreach (var tweet in tweetsToSend) - { - var note = _statusService.GetStatus(user.Acct, tweet); - var result = - await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox); - - if (result == HttpStatusCode.Accepted) - syncStatus = tweet.Id; - else - throw new Exception("Posting new note activity failed"); - } - } - finally - { - if (syncStatus != fromStatusId) - { - foreach (var f in groupedFollowers) - { - f.FollowingsSyncStatus[userId] = syncStatus; - await _followersDal.UpdateFollowerAsync(f); - } - } - } - } - + private async Task ProcessFollowersWithInbox(ExtractedTweet[] tweets, List followerWtInbox, SyncTwitterUser user) { foreach (var follower in followerWtInbox) { try { - await ProcessFollowerWithInboxAsync(tweets, follower, user); + await _sendTweetsToInboxTask.ExecuteAsync(tweets, follower, user); } catch (Exception e) { @@ -127,43 +82,5 @@ namespace BirdsiteLive.Pipeline.Processors } } } - - private async Task ProcessFollowerWithInboxAsync(IEnumerable tweets, Follower follower, SyncTwitterUser user) - { - var userId = user.Id; - var fromStatusId = follower.FollowingsSyncStatus[userId]; - var tweetsToSend = tweets - .Where(x => x.Id > fromStatusId) - .OrderBy(x => x.Id) - .ToList(); - - var inbox = follower.InboxRoute; - //var inbox = string.IsNullOrWhiteSpace(follower.SharedInboxRoute) - // ? follower.InboxRoute - // : follower.SharedInboxRoute; - - var syncStatus = fromStatusId; - try - { - foreach (var tweet in tweetsToSend) - { - var note = _statusService.GetStatus(user.Acct, tweet); - var result = await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox); - - if (result == HttpStatusCode.Accepted) - syncStatus = tweet.Id; - else - throw new Exception("Posting new note activity failed"); - } - } - finally - { - if (syncStatus != fromStatusId) - { - follower.FollowingsSyncStatus[userId] = syncStatus; - await _followersDal.UpdateFollowerAsync(follower); - } - } - } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs new file mode 100644 index 0000000..3624f45 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs @@ -0,0 +1,71 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Domain; +using BirdsiteLive.Twitter.Models; + +namespace BirdsiteLive.Pipeline.Processors.SubTasks +{ + public interface ISendTweetsToInboxTask + { + Task ExecuteAsync(IEnumerable tweets, Follower follower, SyncTwitterUser user); + } + + public class SendTweetsToInboxTask : ISendTweetsToInboxTask + { + private readonly IActivityPubService _activityPubService; + private readonly IStatusService _statusService; + private readonly IFollowersDal _followersDal; + + #region Ctor + public SendTweetsToInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal) + { + _activityPubService = activityPubService; + _statusService = statusService; + _followersDal = followersDal; + } + #endregion + + public async Task ExecuteAsync(IEnumerable tweets, Follower follower, SyncTwitterUser user) + { + var userId = user.Id; + var fromStatusId = follower.FollowingsSyncStatus[userId]; + var tweetsToSend = tweets + .Where(x => x.Id > fromStatusId) + .OrderBy(x => x.Id) + .ToList(); + + var inbox = follower.InboxRoute; + //var inbox = string.IsNullOrWhiteSpace(follower.SharedInboxRoute) + // ? follower.InboxRoute + // : follower.SharedInboxRoute; + + var syncStatus = fromStatusId; + try + { + foreach (var tweet in tweetsToSend) + { + var note = _statusService.GetStatus(user.Acct, tweet); + var result = await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox); + + if (result == HttpStatusCode.Accepted) + syncStatus = tweet.Id; + else + throw new Exception("Posting new note activity failed"); + } + } + finally + { + if (syncStatus != fromStatusId) + { + follower.FollowingsSyncStatus[userId] = syncStatus; + await _followersDal.UpdateFollowerAsync(follower); + } + } + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs new file mode 100644 index 0000000..0aeafd6 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs @@ -0,0 +1,75 @@ +using System; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Domain; +using BirdsiteLive.Twitter.Models; + +namespace BirdsiteLive.Pipeline.Processors.SubTasks +{ + public interface ISendTweetsToSharedInboxTask + { + Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, IGrouping followersPerInstance); + } + + public class SendTweetsToSharedInboxTask : ISendTweetsToSharedInboxTask + { + private readonly IStatusService _statusService; + private readonly IActivityPubService _activityPubService; + private readonly IFollowersDal _followersDal; + + #region Ctor + public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal) + { + _activityPubService = activityPubService; + _statusService = statusService; + _followersDal = followersDal; + } + #endregion + + public async Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, IGrouping followersPerInstance) + { + var userId = user.Id; + var host = followersPerInstance.Key; + var groupedFollowers = followersPerInstance.ToList(); + var inbox = groupedFollowers.First().SharedInboxRoute; + + var fromStatusId = groupedFollowers + .Max(x => x.FollowingsSyncStatus[userId]); + + var tweetsToSend = tweets + .Where(x => x.Id > fromStatusId) + .OrderBy(x => x.Id) + .ToList(); + + var syncStatus = fromStatusId; + try + { + foreach (var tweet in tweetsToSend) + { + var note = _statusService.GetStatus(user.Acct, tweet); + var result = + await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox); + + if (result == HttpStatusCode.Accepted) + syncStatus = tweet.Id; + else + throw new Exception("Posting new note activity failed"); + } + } + finally + { + if (syncStatus != fromStatusId) + { + foreach (var f in groupedFollowers) + { + f.FollowingsSyncStatus[userId] = syncStatus; + await _followersDal.UpdateFollowerAsync(f); + } + } + } + } + } +} \ No newline at end of file From c3acb19e7c3967abb0550e88243dc1dfa3c2dcf6 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 12 Aug 2020 20:23:19 -0400 Subject: [PATCH 46/67] added SendTweetToSharedInbox Tests --- .../BirdsiteLive.ActivityPub.csproj | 2 +- .../SendTweetsToFollowersProcessor.cs | 2 +- .../SubTasks/SendTweetsToSharedInboxTask.cs | 12 +- src/BirdsiteLive.sln | 9 +- .../BirdsiteLive.Pipeline.Tests.csproj | 21 ++ .../SubTasks/SendTweetsToInboxTaskTests.cs | 7 + .../SubTasks/SendTweetsToSharedInboxTests.cs | 322 ++++++++++++++++++ 7 files changed, 365 insertions(+), 10 deletions(-) create mode 100644 src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj create mode 100644 src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs create mode 100644 src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs diff --git a/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj b/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj index 8dfebd7..a690b63 100644 --- a/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj +++ b/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj @@ -6,7 +6,7 @@ - + diff --git a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs index 29bfb12..95fd0c8 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs @@ -57,7 +57,7 @@ namespace BirdsiteLive.Pipeline.Processors { try { - await _sendTweetsToSharedInbox.ExecuteAsync(tweets, user, followersPerInstance); + await _sendTweetsToSharedInbox.ExecuteAsync(tweets, user, followersPerInstance.Key, followersPerInstance.ToArray()); } catch (Exception e) { diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs index 0aeafd6..bdebdcd 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs @@ -11,7 +11,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks { public interface ISendTweetsToSharedInboxTask { - Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, IGrouping followersPerInstance); + Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, string host, Follower[] followersPerInstance); } public class SendTweetsToSharedInboxTask : ISendTweetsToSharedInboxTask @@ -29,14 +29,12 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks } #endregion - public async Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, IGrouping followersPerInstance) + public async Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, string host, Follower[] followersPerInstance) { var userId = user.Id; - var host = followersPerInstance.Key; - var groupedFollowers = followersPerInstance.ToList(); - var inbox = groupedFollowers.First().SharedInboxRoute; + var inbox = followersPerInstance.First().SharedInboxRoute; - var fromStatusId = groupedFollowers + var fromStatusId = followersPerInstance .Max(x => x.FollowingsSyncStatus[userId]); var tweetsToSend = tweets @@ -63,7 +61,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks { if (syncStatus != fromStatusId) { - foreach (var f in groupedFollowers) + foreach (var f in followersPerInstance) { f.FollowingsSyncStatus[userId] = syncStatus; await _followersDal.UpdateFollowerAsync(f); diff --git a/src/BirdsiteLive.sln b/src/BirdsiteLive.sln index 6ef9eb6..bf78d55 100644 --- a/src/BirdsiteLive.sln +++ b/src/BirdsiteLive.sln @@ -35,7 +35,9 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Pipeline", "Pipeline", "{DA EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Pipeline", "BirdsiteLive.Pipeline\BirdsiteLive.Pipeline.csproj", "{2A8CC30D-D775-47D1-9388-F72A5C32DE2A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdsiteLive.Domain.Tests", "Tests\BirdsiteLive.Domain.Tests\BirdsiteLive.Domain.Tests.csproj", "{F544D745-89A8-4DEA-B61C-A7E6C53C1D63}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Domain.Tests", "Tests\BirdsiteLive.Domain.Tests\BirdsiteLive.Domain.Tests.csproj", "{F544D745-89A8-4DEA-B61C-A7E6C53C1D63}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Pipeline.Tests", "Tests\BirdsiteLive.Pipeline.Tests\BirdsiteLive.Pipeline.Tests.csproj", "{BF51CA81-5A7A-46F8-B4FB-861C6BE59298}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -95,6 +97,10 @@ Global {F544D745-89A8-4DEA-B61C-A7E6C53C1D63}.Debug|Any CPU.Build.0 = Debug|Any CPU {F544D745-89A8-4DEA-B61C-A7E6C53C1D63}.Release|Any CPU.ActiveCfg = Release|Any CPU {F544D745-89A8-4DEA-B61C-A7E6C53C1D63}.Release|Any CPU.Build.0 = Release|Any CPU + {BF51CA81-5A7A-46F8-B4FB-861C6BE59298}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BF51CA81-5A7A-46F8-B4FB-861C6BE59298}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BF51CA81-5A7A-46F8-B4FB-861C6BE59298}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BF51CA81-5A7A-46F8-B4FB-861C6BE59298}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -112,6 +118,7 @@ Global {CD9489BF-69C8-4705-8774-81C45F4F8FE1} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94} {2A8CC30D-D775-47D1-9388-F72A5C32DE2A} = {DA3C160C-4811-4E26-A5AD-42B81FAF2D7C} {F544D745-89A8-4DEA-B61C-A7E6C53C1D63} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94} + {BF51CA81-5A7A-46F8-B4FB-861C6BE59298} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {69E8DCAD-4C37-4010-858F-5F94E6FBABCE} diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj b/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj new file mode 100644 index 0000000..3dd6984 --- /dev/null +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj @@ -0,0 +1,21 @@ + + + + netcoreapp3.1 + + false + + + + + + + + + + + + + + + diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs new file mode 100644 index 0000000..0193442 --- /dev/null +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs @@ -0,0 +1,7 @@ +namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks +{ + public class SendTweetsToInboxTaskTests + { + + } +} \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs new file mode 100644 index 0000000..a052a5c --- /dev/null +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs @@ -0,0 +1,322 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using BirdsiteLive.ActivityPub.Models; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Domain; +using BirdsiteLive.Pipeline.Processors.SubTasks; +using BirdsiteLive.Twitter.Models; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks +{ + [TestClass] + public class SendTweetsToSharedInboxTests + { + [TestMethod] + public async Task ExecuteAsync_SingleTweet_Test() + { + #region Stubs + var tweetId = 10; + var tweets = new List + { + new ExtractedTweet + { + Id = tweetId, + } + }; + + var noteId = "noteId"; + var note = new Note() + { + id = noteId + }; + + var twitterHandle = "Test"; + var twitterUserId = 7; + var twitterUser = new SyncTwitterUser + { + Id = twitterUserId, + Acct = twitterHandle + }; + + var host = "domain.ext"; + var inbox = "/inbox"; + var followers = new List + { + new Follower + { + Id = 1, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 9 } } + }, + new Follower + { + Id = 2, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 8 } } + }, + new Follower + { + Id = 3, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 7 } } + } + }; + #endregion + + #region Mocks + var activityPubService = new Mock(MockBehavior.Strict); + activityPubService + .Setup(x => x.PostNewNoteActivity( + It.Is(y => y.id == noteId), + It.Is(y => y == twitterHandle), + It.Is(y => y == tweetId.ToString()), + It.Is(y => y == host), + It.Is(y => y == inbox))) + .ReturnsAsync(HttpStatusCode.Accepted); + + var statusServiceMock = new Mock(MockBehavior.Strict); + statusServiceMock + .Setup(x => x.GetStatus( + It.Is(y => y == twitterHandle), + It.Is(y => y.Id == tweetId))) + .Returns(note); + + var followersDalMock = new Mock(MockBehavior.Strict); + + foreach (var follower in followers) + { + followersDalMock + .Setup(x => x.UpdateFollowerAsync( + It.Is(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId))) + .Returns(Task.CompletedTask); + } + #endregion + + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object); + await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); + + #region Validations + activityPubService.VerifyAll(); + statusServiceMock.VerifyAll(); + followersDalMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ExecuteAsync_MultipleTweets_Test() + { + #region Stubs + var tweetId1 = 10; + var tweetId2 = 11; + var tweetId3 = 12; + var tweets = new List(); + foreach (var tweetId in new[] { tweetId1, tweetId2, tweetId3 }) + { + tweets.Add(new ExtractedTweet + { + Id = tweetId + }); + } + + var twitterHandle = "Test"; + var twitterUserId = 7; + var twitterUser = new SyncTwitterUser + { + Id = twitterUserId, + Acct = twitterHandle + }; + + var host = "domain.ext"; + var inbox = "/inbox"; + var followers = new List + { + new Follower + { + Id = 1, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary {{twitterUserId, 10}} + }, + new Follower + { + Id = 2, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary {{twitterUserId, 8}} + }, + new Follower + { + Id = 3, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary {{twitterUserId, 7}} + } + }; + #endregion + + #region Mocks + var activityPubService = new Mock(MockBehavior.Strict); + foreach (var tweetId in new[] { tweetId2, tweetId3 }) + { + activityPubService + .Setup(x => x.PostNewNoteActivity( + It.Is(y => y.id == tweetId.ToString()), + It.Is(y => y == twitterHandle), + It.Is(y => y == tweetId.ToString()), + It.Is(y => y == host), + It.Is(y => y == inbox))) + .ReturnsAsync(HttpStatusCode.Accepted); + } + + var statusServiceMock = new Mock(MockBehavior.Strict); + foreach (var tweetId in new[] { tweetId2, tweetId3 }) + { + statusServiceMock + .Setup(x => x.GetStatus( + It.Is(y => y == twitterHandle), + It.Is(y => y.Id == tweetId))) + .Returns(new Note { id = tweetId.ToString() }); + } + + var followersDalMock = new Mock(MockBehavior.Strict); + + foreach (var follower in followers) + { + followersDalMock + .Setup(x => x.UpdateFollowerAsync( + It.Is(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId3))) + .Returns(Task.CompletedTask); + } + #endregion + + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object); + await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); + + #region Validations + activityPubService.VerifyAll(); + statusServiceMock.VerifyAll(); + followersDalMock.VerifyAll(); + #endregion + } + + [TestMethod] + [ExpectedException(typeof(Exception))] + public async Task ExecuteAsync_MultipleTweets_Error_Test() + { + #region Stubs + var tweetId1 = 10; + var tweetId2 = 11; + var tweetId3 = 12; + var tweets = new List(); + foreach (var tweetId in new[] { tweetId1, tweetId2, tweetId3 }) + { + tweets.Add(new ExtractedTweet + { + Id = tweetId + }); + } + + var twitterHandle = "Test"; + var twitterUserId = 7; + var twitterUser = new SyncTwitterUser + { + Id = twitterUserId, + Acct = twitterHandle + }; + + var host = "domain.ext"; + var inbox = "/inbox"; + var followers = new List + { + new Follower + { + Id = 1, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary {{twitterUserId, 10}} + }, + new Follower + { + Id = 2, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary {{twitterUserId, 8}} + }, + new Follower + { + Id = 3, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary {{twitterUserId, 7}} + } + }; + #endregion + + #region Mocks + var activityPubService = new Mock(MockBehavior.Strict); + + activityPubService + .Setup(x => x.PostNewNoteActivity( + It.Is(y => y.id == tweetId2.ToString()), + It.Is(y => y == twitterHandle), + It.Is(y => y == tweetId2.ToString()), + It.Is(y => y == host), + It.Is(y => y == inbox))) + .ReturnsAsync(HttpStatusCode.Accepted); + + activityPubService + .Setup(x => x.PostNewNoteActivity( + It.Is(y => y.id == tweetId3.ToString()), + It.Is(y => y == twitterHandle), + It.Is(y => y == tweetId3.ToString()), + It.Is(y => y == host), + It.Is(y => y == inbox))) + .ReturnsAsync(HttpStatusCode.InternalServerError); + + var statusServiceMock = new Mock(MockBehavior.Strict); + foreach (var tweetId in new[] { tweetId2, tweetId3 }) + { + statusServiceMock + .Setup(x => x.GetStatus( + It.Is(y => y == twitterHandle), + It.Is(y => y.Id == tweetId))) + .Returns(new Note { id = tweetId.ToString() }); + } + + var followersDalMock = new Mock(MockBehavior.Strict); + + foreach (var follower in followers) + { + followersDalMock + .Setup(x => x.UpdateFollowerAsync( + It.Is(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId2))) + .Returns(Task.CompletedTask); + } + #endregion + + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object); + + try + { + await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); + } + finally + { + #region Validations + activityPubService.VerifyAll(); + statusServiceMock.VerifyAll(); + followersDalMock.VerifyAll(); + #endregion + } + } + } +} \ No newline at end of file From 6060537d6044668583cce45b43fd9fa84e35f658 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 12 Aug 2020 20:31:28 -0400 Subject: [PATCH 47/67] added SendTweetToInbox Tests --- .../SubTasks/SendTweetsToInboxTask.cs | 3 - .../SubTasks/SendTweetsToInboxTaskTests.cs | 258 +++++++++++++++++- 2 files changed, 256 insertions(+), 5 deletions(-) diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs index 3624f45..eb1fb36 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs @@ -40,9 +40,6 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks .ToList(); var inbox = follower.InboxRoute; - //var inbox = string.IsNullOrWhiteSpace(follower.SharedInboxRoute) - // ? follower.InboxRoute - // : follower.SharedInboxRoute; var syncStatus = fromStatusId; try diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs index 0193442..0596162 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs @@ -1,7 +1,261 @@ -namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks +using System; +using System.Collections.Generic; +using System.Net; +using System.Threading.Tasks; +using BirdsiteLive.ActivityPub.Models; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Domain; +using BirdsiteLive.Pipeline.Processors.SubTasks; +using BirdsiteLive.Twitter.Models; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks { + [TestClass] public class SendTweetsToInboxTaskTests { - + [TestMethod] + public async Task ExecuteAsync_SingleTweet_Test() + { + #region Stubs + var tweetId = 10; + var tweets = new List + { + new ExtractedTweet + { + Id = tweetId, + } + }; + + var noteId = "noteId"; + var note = new Note() + { + id = noteId + }; + + var twitterHandle = "Test"; + var twitterUserId = 7; + var twitterUser = new SyncTwitterUser + { + Id = twitterUserId, + Acct = twitterHandle + }; + + var host = "domain.ext"; + var inbox = "/user/inbox"; + var follower = new Follower + { + Id = 1, + Host = host, + InboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 9 } } + }; + #endregion + + #region Mocks + var activityPubService = new Mock(MockBehavior.Strict); + activityPubService + .Setup(x => x.PostNewNoteActivity( + It.Is(y => y.id == noteId), + It.Is(y => y == twitterHandle), + It.Is(y => y == tweetId.ToString()), + It.Is(y => y == host), + It.Is(y => y == inbox))) + .ReturnsAsync(HttpStatusCode.Accepted); + + var statusServiceMock = new Mock(MockBehavior.Strict); + statusServiceMock + .Setup(x => x.GetStatus( + It.Is(y => y == twitterHandle), + It.Is(y => y.Id == tweetId))) + .Returns(note); + + var followersDalMock = new Mock(MockBehavior.Strict); + followersDalMock + .Setup(x => x.UpdateFollowerAsync( + It.Is(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId))) + .Returns(Task.CompletedTask); + + #endregion + + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object); + await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); + + #region Validations + activityPubService.VerifyAll(); + statusServiceMock.VerifyAll(); + followersDalMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ExecuteAsync_MultipleTweets_Test() + { + #region Stubs + var tweetId1 = 10; + var tweetId2 = 11; + var tweetId3 = 12; + var tweets = new List(); + foreach (var tweetId in new[] { tweetId1, tweetId2, tweetId3 }) + { + tweets.Add(new ExtractedTweet + { + Id = tweetId + }); + } + + var twitterHandle = "Test"; + var twitterUserId = 7; + var twitterUser = new SyncTwitterUser + { + Id = twitterUserId, + Acct = twitterHandle + }; + + var host = "domain.ext"; + var inbox = "/user/inbox"; + var follower = new Follower + { + Id = 1, + Host = host, + InboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 10 } } + }; + #endregion + + #region Mocks + var activityPubService = new Mock(MockBehavior.Strict); + foreach (var tweetId in new[] { tweetId2, tweetId3 }) + { + activityPubService + .Setup(x => x.PostNewNoteActivity( + It.Is(y => y.id == tweetId.ToString()), + It.Is(y => y == twitterHandle), + It.Is(y => y == tweetId.ToString()), + It.Is(y => y == host), + It.Is(y => y == inbox))) + .ReturnsAsync(HttpStatusCode.Accepted); + } + + var statusServiceMock = new Mock(MockBehavior.Strict); + foreach (var tweetId in new[] { tweetId2, tweetId3 }) + { + statusServiceMock + .Setup(x => x.GetStatus( + It.Is(y => y == twitterHandle), + It.Is(y => y.Id == tweetId))) + .Returns(new Note { id = tweetId.ToString() }); + } + + var followersDalMock = new Mock(MockBehavior.Strict); + followersDalMock + .Setup(x => x.UpdateFollowerAsync( + It.Is(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId3))) + .Returns(Task.CompletedTask); + + #endregion + + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object); + await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); + + #region Validations + activityPubService.VerifyAll(); + statusServiceMock.VerifyAll(); + followersDalMock.VerifyAll(); + #endregion + } + + [TestMethod] + [ExpectedException(typeof(Exception))] + public async Task ExecuteAsync_MultipleTweets_Error_Test() + { + #region Stubs + var tweetId1 = 10; + var tweetId2 = 11; + var tweetId3 = 12; + var tweets = new List(); + foreach (var tweetId in new[] { tweetId1, tweetId2, tweetId3 }) + { + tweets.Add(new ExtractedTweet + { + Id = tweetId + }); + } + + var twitterHandle = "Test"; + var twitterUserId = 7; + var twitterUser = new SyncTwitterUser + { + Id = twitterUserId, + Acct = twitterHandle + }; + + var host = "domain.ext"; + var inbox = "/user/inbox"; + var follower = new Follower + { + Id = 1, + Host = host, + InboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 10 } } + }; + #endregion + + #region Mocks + var activityPubService = new Mock(MockBehavior.Strict); + + activityPubService + .Setup(x => x.PostNewNoteActivity( + It.Is(y => y.id == tweetId2.ToString()), + It.Is(y => y == twitterHandle), + It.Is(y => y == tweetId2.ToString()), + It.Is(y => y == host), + It.Is(y => y == inbox))) + .ReturnsAsync(HttpStatusCode.Accepted); + + activityPubService + .Setup(x => x.PostNewNoteActivity( + It.Is(y => y.id == tweetId3.ToString()), + It.Is(y => y == twitterHandle), + It.Is(y => y == tweetId3.ToString()), + It.Is(y => y == host), + It.Is(y => y == inbox))) + .ReturnsAsync(HttpStatusCode.InternalServerError); + + var statusServiceMock = new Mock(MockBehavior.Strict); + foreach (var tweetId in new[] { tweetId2, tweetId3 }) + { + statusServiceMock + .Setup(x => x.GetStatus( + It.Is(y => y == twitterHandle), + It.Is(y => y.Id == tweetId))) + .Returns(new Note { id = tweetId.ToString() }); + } + + var followersDalMock = new Mock(MockBehavior.Strict); + followersDalMock + .Setup(x => x.UpdateFollowerAsync( + It.Is(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId2))) + .Returns(Task.CompletedTask); + + #endregion + + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object); + + try + { + await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); + } + finally + { + #region Validations + activityPubService.VerifyAll(); + statusServiceMock.VerifyAll(); + followersDalMock.VerifyAll(); + #endregion + } + } } } \ No newline at end of file From 5fe1daabd7a199680f19192fb07365db5afe87bc Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 12 Aug 2020 20:55:50 -0400 Subject: [PATCH 48/67] added SendTweetsToFollowersProcessor Tests --- .../SendTweetsToFollowersProcessorTests.cs | 426 ++++++++++++++++++ 1 file changed, 426 insertions(+) create mode 100644 src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs new file mode 100644 index 0000000..034d8d2 --- /dev/null +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs @@ -0,0 +1,426 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Pipeline.Models; +using BirdsiteLive.Pipeline.Processors; +using BirdsiteLive.Pipeline.Processors.SubTasks; +using BirdsiteLive.Twitter.Models; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace BirdsiteLive.Pipeline.Tests.Processors +{ + [TestClass] + public class SendTweetsToFollowersProcessorTests + { + [TestMethod] + public async Task ProcessAsync_SameInstance_SharedInbox_OneTweet_Test() + { + #region Stubs + var tweetId = 1; + var host = "domain.ext"; + var sharedInbox = "/inbox"; + var userId1 = 2; + var userId2 = 3; + var userAcct = "user"; + + var userWithTweets = new UserWithTweetsToSync() + { + Tweets = new [] + { + new ExtractedTweet + { + Id = tweetId + } + }, + User = new SyncTwitterUser + { + Acct = userAcct + }, + Followers = new [] + { + new Follower + { + Id = userId1, + Host = host, + SharedInboxRoute = sharedInbox + }, + new Follower + { + Id = userId2, + Host = host, + SharedInboxRoute = sharedInbox + }, + } + }; + #endregion + + #region Mocks + var sendTweetsToInboxTaskMock = new Mock(MockBehavior.Strict); + + var sendTweetsToSharedInboxTaskMock = new Mock(MockBehavior.Strict); + sendTweetsToSharedInboxTaskMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Length == 1), + It.Is(y => y.Acct == userAcct), + It.Is(y => y == host), + It.Is(y => y.Length == 2))) + .Returns(Task.CompletedTask); + #endregion + + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object); + var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); + + #region Validations + sendTweetsToInboxTaskMock.VerifyAll(); + sendTweetsToSharedInboxTaskMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_MultiInstances_SharedInbox_OneTweet_Test() + { + #region Stubs + var tweetId = 1; + var host1 = "domain1.ext"; + var host2 = "domain2.ext"; + var sharedInbox = "/inbox"; + var userId1 = 2; + var userId2 = 3; + var userAcct = "user"; + + var userWithTweets = new UserWithTweetsToSync() + { + Tweets = new[] + { + new ExtractedTweet + { + Id = tweetId + } + }, + User = new SyncTwitterUser + { + Acct = userAcct + }, + Followers = new[] + { + new Follower + { + Id = userId1, + Host = host1, + SharedInboxRoute = sharedInbox + }, + new Follower + { + Id = userId2, + Host = host2, + SharedInboxRoute = sharedInbox + }, + } + }; + #endregion + + #region Mocks + var sendTweetsToInboxTaskMock = new Mock(MockBehavior.Strict); + + var sendTweetsToSharedInboxTaskMock = new Mock(MockBehavior.Strict); + foreach (var host in new [] { host1, host2}) + { + sendTweetsToSharedInboxTaskMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Length == 1), + It.Is(y => y.Acct == userAcct), + It.Is(y => y == host), + It.Is(y => y.Length == 1))) + .Returns(Task.CompletedTask); + } + #endregion + + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object); + var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); + + #region Validations + sendTweetsToInboxTaskMock.VerifyAll(); + sendTweetsToSharedInboxTaskMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_MultiInstances_SharedInbox_OneTweet_Error_Test() + { + #region Stubs + var tweetId = 1; + var host1 = "domain1.ext"; + var host2 = "domain2.ext"; + var sharedInbox = "/inbox"; + var userId1 = 2; + var userId2 = 3; + var userAcct = "user"; + + var userWithTweets = new UserWithTweetsToSync() + { + Tweets = new[] + { + new ExtractedTweet + { + Id = tweetId + } + }, + User = new SyncTwitterUser + { + Acct = userAcct + }, + Followers = new[] + { + new Follower + { + Id = userId1, + Host = host1, + SharedInboxRoute = sharedInbox + }, + new Follower + { + Id = userId2, + Host = host2, + SharedInboxRoute = sharedInbox + }, + } + }; + #endregion + + #region Mocks + var sendTweetsToInboxTaskMock = new Mock(MockBehavior.Strict); + + var sendTweetsToSharedInboxTaskMock = new Mock(MockBehavior.Strict); + sendTweetsToSharedInboxTaskMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Length == 1), + It.Is(y => y.Acct == userAcct), + It.Is(y => y == host1), + It.Is(y => y.Length == 1))) + .Returns(Task.CompletedTask); + + sendTweetsToSharedInboxTaskMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Length == 1), + It.Is(y => y.Acct == userAcct), + It.Is(y => y == host2), + It.Is(y => y.Length == 1))) + .Throws(new Exception()); + #endregion + + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object); + var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); + + #region Validations + sendTweetsToInboxTaskMock.VerifyAll(); + sendTweetsToSharedInboxTaskMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_SameInstance_Inbox_OneTweet_Test() + { + #region Stubs + var tweetId = 1; + var host = "domain.ext"; + var inbox = "/user/inbox"; + var userId1 = 2; + var userId2 = 3; + var userAcct = "user"; + + var userWithTweets = new UserWithTweetsToSync() + { + Tweets = new[] + { + new ExtractedTweet + { + Id = tweetId + } + }, + User = new SyncTwitterUser + { + Acct = userAcct + }, + Followers = new[] + { + new Follower + { + Id = userId1, + Host = host, + InboxRoute = inbox + }, + new Follower + { + Id = userId2, + Host = host, + InboxRoute = inbox + }, + } + }; + #endregion + + #region Mocks + var sendTweetsToInboxTaskMock = new Mock(MockBehavior.Strict); + foreach (var userId in new[] { userId1, userId2 }) + { + sendTweetsToInboxTaskMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Length == 1), + It.Is(y => y.Id == userId), + It.Is(y => y.Acct == userAcct))) + .Returns(Task.CompletedTask); + } + + var sendTweetsToSharedInboxTaskMock = new Mock(MockBehavior.Strict); + #endregion + + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object); + var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); + + #region Validations + sendTweetsToInboxTaskMock.VerifyAll(); + sendTweetsToSharedInboxTaskMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_MultiInstances_Inbox_OneTweet_Test() + { + #region Stubs + var tweetId = 1; + var host1 = "domain1.ext"; + var host2 = "domain2.ext"; + var inbox = "/user/inbox"; + var userId1 = 2; + var userId2 = 3; + var userAcct = "user"; + + var userWithTweets = new UserWithTweetsToSync() + { + Tweets = new[] + { + new ExtractedTweet + { + Id = tweetId + } + }, + User = new SyncTwitterUser + { + Acct = userAcct + }, + Followers = new[] + { + new Follower + { + Id = userId1, + Host = host1, + InboxRoute = inbox + }, + new Follower + { + Id = userId2, + Host = host2, + InboxRoute = inbox + }, + } + }; + #endregion + + #region Mocks + var sendTweetsToInboxTaskMock = new Mock(MockBehavior.Strict); + foreach (var userId in new[] { userId1, userId2 }) + { + sendTweetsToInboxTaskMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Length == 1), + It.Is(y => y.Id == userId), + It.Is(y => y.Acct == userAcct))) + .Returns(Task.CompletedTask); + } + + var sendTweetsToSharedInboxTaskMock = new Mock(MockBehavior.Strict); + #endregion + + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object); + var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); + + #region Validations + sendTweetsToInboxTaskMock.VerifyAll(); + sendTweetsToSharedInboxTaskMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_MultiInstances_Inbox_OneTweet_Error_Test() + { + #region Stubs + var tweetId = 1; + var host1 = "domain1.ext"; + var host2 = "domain2.ext"; + var inbox = "/user/inbox"; + var userId1 = 2; + var userId2 = 3; + var userAcct = "user"; + + var userWithTweets = new UserWithTweetsToSync() + { + Tweets = new[] + { + new ExtractedTweet + { + Id = tweetId + } + }, + User = new SyncTwitterUser + { + Acct = userAcct + }, + Followers = new[] + { + new Follower + { + Id = userId1, + Host = host1, + InboxRoute = inbox + }, + new Follower + { + Id = userId2, + Host = host2, + InboxRoute = inbox + }, + } + }; + #endregion + + #region Mocks + var sendTweetsToInboxTaskMock = new Mock(MockBehavior.Strict); + sendTweetsToInboxTaskMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Length == 1), + It.Is(y => y.Id == userId1), + It.Is(y => y.Acct == userAcct))) + .Returns(Task.CompletedTask); + + sendTweetsToInboxTaskMock + .Setup(x => x.ExecuteAsync( + It.Is(y => y.Length == 1), + It.Is(y => y.Id == userId2), + It.Is(y => y.Acct == userAcct))) + .Throws(new Exception()); + + var sendTweetsToSharedInboxTaskMock = new Mock(MockBehavior.Strict); + #endregion + + var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object); + var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None); + + #region Validations + sendTweetsToInboxTaskMock.VerifyAll(); + sendTweetsToSharedInboxTaskMock.VerifyAll(); + #endregion + } + } +} \ No newline at end of file From 98d5b2183bb311d243466d3165215c173b2f4d2a Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 12 Aug 2020 21:05:25 -0400 Subject: [PATCH 49/67] added RetrieveFollowersProcessor tests --- .../RetrieveFollowersProcessorTests.cs | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveFollowersProcessorTests.cs diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveFollowersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveFollowersProcessorTests.cs new file mode 100644 index 0000000..98a86bf --- /dev/null +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveFollowersProcessorTests.cs @@ -0,0 +1,79 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Pipeline.Models; +using BirdsiteLive.Pipeline.Processors; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace BirdsiteLive.Pipeline.Tests.Processors +{ + [TestClass] + public class RetrieveFollowersProcessorTests + { + [TestMethod] + public async Task ProcessAsync_Test() + { + #region Stubs + var userId1 = 1; + var userId2 = 2; + + var users = new List + { + new UserWithTweetsToSync + { + User = new SyncTwitterUser + { + Id = userId1 + } + }, + new UserWithTweetsToSync + { + User = new SyncTwitterUser + { + Id = userId2 + } + } + }; + + var followersUser1 = new List + { + new Follower(), + new Follower(), + }; + var followersUser2 = new List + { + new Follower(), + new Follower(), + new Follower(), + }; + #endregion + + #region Mocks + var followersDalMock = new Mock(MockBehavior.Strict); + followersDalMock + .Setup(x => x.GetFollowersAsync(It.Is(y => y == userId1))) + .ReturnsAsync(followersUser1.ToArray()); + + followersDalMock + .Setup(x => x.GetFollowersAsync(It.Is(y => y == userId2))) + .ReturnsAsync(followersUser2.ToArray()); + #endregion + + var processor = new RetrieveFollowersProcessor(followersDalMock.Object); + var result = (await processor.ProcessAsync(users.ToArray(), CancellationToken.None)).ToList(); + + #region Validations + Assert.IsNotNull(result); + Assert.AreEqual(2, result.Count); + Assert.AreEqual(2, result.First(x => x.User.Id == userId1).Followers.Length); + Assert.AreEqual(3, result.First(x => x.User.Id == userId2).Followers.Length); + + followersDalMock.VerifyAll(); + #endregion + } + } +} \ No newline at end of file From 921f73b6d560a1e0a9688e36bb866b0facf5be34 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 9 Sep 2020 19:53:39 -0400 Subject: [PATCH 50/67] added test --- .../RetrieveTweetsProcessorTests.cs | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs new file mode 100644 index 0000000..61ba804 --- /dev/null +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs @@ -0,0 +1,41 @@ +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Pipeline.Processors; +using BirdsiteLive.Twitter; +using Castle.DynamicProxy.Generators.Emitters; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace BirdsiteLive.Pipeline.Tests.Processors +{ + [TestClass] + class RetrieveTweetsProcessorTests + { + [TestMethod] + public async Task ProcessAsync_Test() + { + var users = new List + { + new SyncTwitterUser { Id = 1 }, + new SyncTwitterUser { Id = 2 }, + new SyncTwitterUser { Id = 3 }, + }; + + var twitterServiceMock = new Mock(MockBehavior.Strict); + var twitterUserDalMock = new Mock(MockBehavior.Strict); + + var procesor = new RetrieveTweetsProcessor(twitterServiceMock.Object, twitterUserDalMock.Object); + + var result = await procesor.ProcessAsync(users.ToArray(), CancellationToken.None); + + + + } + + } +} From c7e1a4e5e178d8d01450eddd3941c4173c0cadd0 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 10 Oct 2020 18:35:23 -0400 Subject: [PATCH 51/67] added tests --- ...r.cs => IRetrieveTwitterUsersProcessor.cs} | 0 .../Processors/RetrieveTweetsProcessor.cs | 2 +- ...or.cs => RetrieveTwitterUsersProcessor.cs} | 0 .../RetrieveTweetsProcessorTests.cs | 193 ++++++++++++++++ .../RetrieveTwitterUsersProcessorTests.cs | 93 ++++++++ .../SaveProgressionProcessorTests.cs | 210 ++++++++++++++++++ 6 files changed, 497 insertions(+), 1 deletion(-) rename src/BirdsiteLive.Pipeline/Contracts/{IRetrieveTwitterAccountsProcessor.cs => IRetrieveTwitterUsersProcessor.cs} (100%) rename src/BirdsiteLive.Pipeline/Processors/{RetrieveTwitterAccountsProcessor.cs => RetrieveTwitterUsersProcessor.cs} (100%) create mode 100644 src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs create mode 100644 src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs create mode 100644 src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs diff --git a/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterAccountsProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterUsersProcessor.cs similarity index 100% rename from src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterAccountsProcessor.cs rename to src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterUsersProcessor.cs diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs index dc556ba..68ca0b0 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs @@ -44,7 +44,7 @@ namespace BirdsiteLive.Pipeline.Processors } else if (tweets.Length > 0 && user.LastTweetPostedId == -1) { - var tweetId = tweets.First().Id; + var tweetId = tweets.Last().Id; await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId); } } diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterAccountsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs similarity index 100% rename from src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterAccountsProcessor.cs rename to src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs new file mode 100644 index 0000000..ce29997 --- /dev/null +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs @@ -0,0 +1,193 @@ +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Pipeline.Processors; +using BirdsiteLive.Twitter; +using BirdsiteLive.Twitter.Models; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace BirdsiteLive.Pipeline.Tests.Processors +{ + [TestClass] + public class RetrieveTweetsProcessorTests + { + [TestMethod] + public async Task ProcessAsync_UserNotSync_Test() + { + #region Stubs + var user1 = new SyncTwitterUser + { + Id = 1, + Acct = "acct", + LastTweetPostedId = -1 + }; + + var users = new[] + { + user1 + }; + + var tweets = new[] + { + new ExtractedTweet + { + Id = 47 + } + }; + #endregion + + #region Mocks + var twitterServiceMock = new Mock(MockBehavior.Strict); + twitterServiceMock + .Setup(x => x.GetTimeline( + It.Is(y => y == user1.Acct), + It.Is(y => y == 1), + It.Is(y => y == -1) + )) + .Returns(tweets); + + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.UpdateTwitterUserAsync( + It.Is(y => y == user1.Id), + It.Is(y => y == tweets.Last().Id), + It.Is(y => y == tweets.Last().Id) + )) + .Returns(Task.CompletedTask); + #endregion + + var processor = new RetrieveTweetsProcessor(twitterServiceMock.Object, twitterUserDalMock.Object); + var usersResult = await processor.ProcessAsync(users, CancellationToken.None); + + #region Validations + twitterServiceMock.VerifyAll(); + twitterUserDalMock.VerifyAll(); + + Assert.AreEqual(0, usersResult.Length); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_UserSync_Test() + { + #region Stubs + var user1 = new SyncTwitterUser + { + Id = 1, + Acct = "acct", + LastTweetPostedId = 46, + LastTweetSynchronizedForAllFollowersId = 46 + }; + + var users = new[] + { + user1 + }; + + var tweets = new[] + { + new ExtractedTweet + { + Id = 47 + }, + new ExtractedTweet + { + Id = 48 + }, + new ExtractedTweet + { + Id = 49 + } + }; + #endregion + + #region Mocks + var twitterServiceMock = new Mock(MockBehavior.Strict); + twitterServiceMock + .Setup(x => x.GetTimeline( + It.Is(y => y == user1.Acct), + It.Is(y => y == 200), + It.Is(y => y == user1.LastTweetSynchronizedForAllFollowersId) + )) + .Returns(tweets); + + var twitterUserDalMock = new Mock(MockBehavior.Strict); + #endregion + + var processor = new RetrieveTweetsProcessor(twitterServiceMock.Object, twitterUserDalMock.Object); + var usersResult = await processor.ProcessAsync(users, CancellationToken.None); + + #region Validations + twitterServiceMock.VerifyAll(); + twitterUserDalMock.VerifyAll(); + + Assert.AreEqual(users.Length, usersResult.Length); + Assert.AreEqual(users[0].Acct, usersResult[0].User.Acct); + Assert.AreEqual(tweets.Length, usersResult[0].Tweets.Length); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_UserPartiallySync_Test() + { + #region Stubs + var user1 = new SyncTwitterUser + { + Id = 1, + Acct = "acct", + LastTweetPostedId = 49, + LastTweetSynchronizedForAllFollowersId = 46 + }; + + var users = new[] + { + user1 + }; + + var tweets = new[] + { + new ExtractedTweet + { + Id = 47 + }, + new ExtractedTweet + { + Id = 48 + }, + new ExtractedTweet + { + Id = 49 + } + }; + #endregion + + #region Mocks + var twitterServiceMock = new Mock(MockBehavior.Strict); + twitterServiceMock + .Setup(x => x.GetTimeline( + It.Is(y => y == user1.Acct), + It.Is(y => y == 200), + It.Is(y => y == user1.LastTweetSynchronizedForAllFollowersId) + )) + .Returns(tweets); + + var twitterUserDalMock = new Mock(MockBehavior.Strict); + #endregion + + var processor = new RetrieveTweetsProcessor(twitterServiceMock.Object, twitterUserDalMock.Object); + var usersResult = await processor.ProcessAsync(users, CancellationToken.None); + + #region Validations + twitterServiceMock.VerifyAll(); + twitterUserDalMock.VerifyAll(); + + Assert.AreEqual(users.Length, usersResult.Length); + Assert.AreEqual(users[0].Acct, usersResult[0].User.Acct); + Assert.AreEqual(tweets.Length, usersResult[0].Tweets.Length); + #endregion + } + } +} \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs new file mode 100644 index 0000000..d3f8b08 --- /dev/null +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs @@ -0,0 +1,93 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Pipeline.Processors; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace BirdsiteLive.Pipeline.Tests.Processors +{ + [TestClass] + public class RetrieveTwitterUsersProcessorTests + { + [TestMethod] + public async Task GetTwitterUsersAsync_Test() + { + #region Stubs + var buffer = new BufferBlock(); + var users = new[] + { + new SyncTwitterUser(), + new SyncTwitterUser(), + new SyncTwitterUser(), + }; + #endregion + + #region Mocks + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.GetAllTwitterUsersAsync()) + .ReturnsAsync(users); + #endregion + + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object); + processor.GetTwitterUsersAsync(buffer, CancellationToken.None); + + await Task.Delay(50); + + #region Validations + twitterUserDalMock.VerifyAll(); + Assert.AreEqual(1, buffer.Count); + buffer.TryReceive(out var result); + Assert.AreEqual(3, result.Length); + #endregion + } + + [TestMethod] + public async Task GetTwitterUsersAsync_Exception_Test() + { + #region Stubs + var buffer = new BufferBlock(); + #endregion + + #region Mocks + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.GetAllTwitterUsersAsync()) + .Throws(new Exception()); + #endregion + + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object); + var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None); + + await Task.WhenAny(t, Task.Delay(50)); + + #region Validations + twitterUserDalMock.VerifyAll(); + Assert.AreEqual(0, buffer.Count); + #endregion + } + + + [TestMethod] + [ExpectedException(typeof(OperationCanceledException))] + public async Task GetTwitterUsersAsync_Cancellation_Test() + { + #region Stubs + var buffer = new BufferBlock(); + var canTokenS = new CancellationTokenSource(); + canTokenS.Cancel(); + #endregion + + #region Mocks + var twitterUserDalMock = new Mock(MockBehavior.Strict); + #endregion + + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object); + await processor.GetTwitterUsersAsync(buffer, canTokenS.Token); + } + } +} \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs new file mode 100644 index 0000000..d3880e6 --- /dev/null +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs @@ -0,0 +1,210 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Pipeline.Models; +using BirdsiteLive.Pipeline.Processors; +using BirdsiteLive.Twitter.Models; +using Castle.DynamicProxy.Contributors; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace BirdsiteLive.Pipeline.Tests.Processors +{ + [TestClass] + public class SaveProgressionProcessorTests + { + [TestMethod] + public async Task ProcessAsync_Test() + { + #region Stubs + var user = new SyncTwitterUser + { + Id = 1 + }; + var tweet1 = new ExtractedTweet + { + Id = 36 + }; + var tweet2 = new ExtractedTweet + { + Id = 37 + }; + var follower1 = new Follower + { + FollowingsSyncStatus = new Dictionary + { + {1, 37} + } + }; + + var usersWithTweets = new UserWithTweetsToSync + { + Tweets = new [] + { + tweet1, + tweet2 + }, + Followers = new [] + { + follower1 + }, + User = user + }; + #endregion + + #region Mocks + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.UpdateTwitterUserAsync( + It.Is(y => y == user.Id), + It.Is(y => y == tweet2.Id), + It.Is(y => y == tweet2.Id) + )) + .Returns(Task.CompletedTask); + #endregion + + var processor = new SaveProgressionProcessor(twitterUserDalMock.Object); + await processor.ProcessAsync(usersWithTweets, CancellationToken.None); + + #region Validations + twitterUserDalMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_PartiallySynchronized_Test() + { + #region Stubs + var user = new SyncTwitterUser + { + Id = 1 + }; + var tweet1 = new ExtractedTweet + { + Id = 36 + }; + var tweet2 = new ExtractedTweet + { + Id = 37 + }; + var tweet3 = new ExtractedTweet + { + Id = 38 + }; + var follower1 = new Follower + { + FollowingsSyncStatus = new Dictionary + { + {1, 37} + } + }; + + var usersWithTweets = new UserWithTweetsToSync + { + Tweets = new[] + { + tweet1, + tweet2, + tweet3 + }, + Followers = new[] + { + follower1 + }, + User = user + }; + #endregion + + #region Mocks + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.UpdateTwitterUserAsync( + It.Is(y => y == user.Id), + It.Is(y => y == tweet3.Id), + It.Is(y => y == tweet2.Id) + )) + .Returns(Task.CompletedTask); + #endregion + + var processor = new SaveProgressionProcessor(twitterUserDalMock.Object); + await processor.ProcessAsync(usersWithTweets, CancellationToken.None); + + #region Validations + twitterUserDalMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_PartiallySynchronized_MultiUsers_Test() + { + #region Stubs + var user = new SyncTwitterUser + { + Id = 1 + }; + var tweet1 = new ExtractedTweet + { + Id = 36 + }; + var tweet2 = new ExtractedTweet + { + Id = 37 + }; + var tweet3 = new ExtractedTweet + { + Id = 38 + }; + var follower1 = new Follower + { + FollowingsSyncStatus = new Dictionary + { + {1, 37} + } + }; + var follower2 = new Follower + { + FollowingsSyncStatus = new Dictionary + { + {1, 38} + } + }; + + var usersWithTweets = new UserWithTweetsToSync + { + Tweets = new[] + { + tweet1, + tweet2, + tweet3 + }, + Followers = new[] + { + follower1, + follower2 + }, + User = user + }; + #endregion + + #region Mocks + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.UpdateTwitterUserAsync( + It.Is(y => y == user.Id), + It.Is(y => y == tweet3.Id), + It.Is(y => y == tweet2.Id) + )) + .Returns(Task.CompletedTask); + #endregion + + var processor = new SaveProgressionProcessor(twitterUserDalMock.Object); + await processor.ProcessAsync(usersWithTweets, CancellationToken.None); + + #region Validations + twitterUserDalMock.VerifyAll(); + #endregion + } + } +} \ No newline at end of file From f6145aceb0612a2ca22a1aa8e2cd38ff14158e18 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 10 Oct 2020 19:16:34 -0400 Subject: [PATCH 52/67] clean up --- .../Processors/RetrieveTweetsProcessorTests.cs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs index 9ba771a..8873bf7 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs @@ -8,18 +8,6 @@ using BirdsiteLive.Twitter; using BirdsiteLive.Twitter.Models; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; -using BirdsiteLive.DAL.Contracts; -using BirdsiteLive.DAL.Models; -using BirdsiteLive.Pipeline.Processors; -using BirdsiteLive.Twitter; -using Castle.DynamicProxy.Generators.Emitters; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using Moq; -using System; -using System.Collections.Generic; -using System.Text; -using System.Threading; -using System.Threading.Tasks; namespace BirdsiteLive.Pipeline.Tests.Processors { From b7acb4c907795b5fc232841088616655589a31ae Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 18 Nov 2020 22:48:53 -0500 Subject: [PATCH 53/67] added instance handle in user page --- .../Controllers/UsersController.cs | 20 ++++++++++++++++--- src/BirdsiteLive/Models/DisplayTwitterUser.cs | 13 ++++++++++++ src/BirdsiteLive/Views/Users/Index.cshtml | 7 +++++-- src/BirdsiteLive/wwwroot/css/birdsite.css | 1 + 4 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 src/BirdsiteLive/Models/DisplayTwitterUser.cs diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index 5a98538..3f2d701 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -6,7 +6,9 @@ using System.Net.Mime; using System.Threading; using System.Threading.Tasks; using BirdsiteLive.ActivityPub; +using BirdsiteLive.Common.Settings; using BirdsiteLive.Domain; +using BirdsiteLive.Models; using BirdsiteLive.Twitter; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; @@ -20,13 +22,15 @@ namespace BirdsiteLive.Controllers private readonly ITwitterService _twitterService; private readonly IUserService _userService; private readonly IStatusService _statusService; + private readonly InstanceSettings _instanceSettings; #region Ctor - public UsersController(ITwitterService twitterService, IUserService userService, IStatusService statusService) + public UsersController(ITwitterService twitterService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings) { _twitterService = twitterService; _userService = userService; _statusService = statusService; + _instanceSettings = instanceSettings; } #endregion @@ -45,7 +49,17 @@ namespace BirdsiteLive.Controllers return Content(jsonApUser, "application/activity+json; charset=utf-8"); } - return View(user); + var displayableUser = new DisplayTwitterUser + { + Name = user.Name, + Description = user.Description, + Acct = user.Acct, + Url = user.Url, + ProfileImageUrl = user.ProfileImageUrl, + + InstanceHandle = $"@{user.Acct}@{_instanceSettings.Domain}" + }; + return View(displayableUser); } [Route("/@{id}/{statusId}")] @@ -81,7 +95,7 @@ namespace BirdsiteLive.Controllers using (var reader = new StreamReader(Request.Body)) { var body = await reader.ReadToEndAsync(); - //System.IO.File.WriteAllText($@"C:\apdebug\{Guid.NewGuid()}.json", body); + System.IO.File.WriteAllText($@"C:\apdebug\{Guid.NewGuid()}.json", body); var activity = ApDeserializer.ProcessActivity(body); // Do something diff --git a/src/BirdsiteLive/Models/DisplayTwitterUser.cs b/src/BirdsiteLive/Models/DisplayTwitterUser.cs new file mode 100644 index 0000000..58ba348 --- /dev/null +++ b/src/BirdsiteLive/Models/DisplayTwitterUser.cs @@ -0,0 +1,13 @@ +namespace BirdsiteLive.Models +{ + public class DisplayTwitterUser + { + public string Name { get; set; } + public string Description { get; set; } + public string Acct { get; set; } + public string Url { get; set; } + public string ProfileImageUrl { get; set; } + + public string InstanceHandle { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Views/Users/Index.cshtml b/src/BirdsiteLive/Views/Users/Index.cshtml index 2915493..2f3c727 100644 --- a/src/BirdsiteLive/Views/Users/Index.cshtml +++ b/src/BirdsiteLive/Views/Users/Index.cshtml @@ -1,4 +1,5 @@ -@model BirdsiteLive.Twitter.Models.TwitterUser +@using Tweetinvi.Streams.Model.AccountActivity +@model DisplayTwitterUser @{ ViewData["Title"] = "User"; } @@ -28,7 +29,9 @@
-
+
+

Search this handle to find it in your instance:

+
\ No newline at end of file diff --git a/src/BirdsiteLive/wwwroot/css/birdsite.css b/src/BirdsiteLive/wwwroot/css/birdsite.css index c18719f..5b6023c 100644 --- a/src/BirdsiteLive/wwwroot/css/birdsite.css +++ b/src/BirdsiteLive/wwwroot/css/birdsite.css @@ -53,6 +53,7 @@ .sub-profile { padding: 10px 15px; + min-height: 80px; } /*.sub-profile a { From 01337a63ecf6f38315078f68b4ab2a0aa01797df Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Wed, 18 Nov 2020 22:49:44 -0500 Subject: [PATCH 54/67] dont feed pipeline if no elements --- .../RetrieveTwitterUsersProcessor.cs | 4 ++- .../RetrieveTwitterUsersProcessorTests.cs | 26 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs index dcc9d6b..f8ea2a2 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs @@ -29,7 +29,9 @@ namespace BirdsiteLive.Pipeline.Processors try { var users = await _twitterUserDal.GetAllTwitterUsersAsync(); - await twitterUsersBufferBlock.SendAsync(users, ct); + + if(users.Length > 0) + await twitterUsersBufferBlock.SendAsync(users, ct); } catch (Exception e) { diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs index d3f8b08..b7a2e2b 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs @@ -46,6 +46,32 @@ namespace BirdsiteLive.Pipeline.Tests.Processors #endregion } + [TestMethod] + public async Task GetTwitterUsersAsync_NoUsers_Test() + { + #region Stubs + var buffer = new BufferBlock(); + #endregion + + #region Mocks + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.GetAllTwitterUsersAsync()) + .ReturnsAsync(new SyncTwitterUser[0]); + #endregion + + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object); + processor.GetTwitterUsersAsync(buffer, CancellationToken.None); + + await Task.Delay(50); + + #region Validations + twitterUserDalMock.VerifyAll(); + Assert.AreEqual(0, buffer.Count); + #endregion + } + + [TestMethod] public async Task GetTwitterUsersAsync_Exception_Test() { From f819595aec54d9e1fd9710822ca8de93e44d1ae7 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 20 Nov 2020 18:59:57 -0500 Subject: [PATCH 55/67] fix username --- src/BirdsiteLive.Domain/UserService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index fa2f5aa..f2482d1 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -85,7 +85,7 @@ namespace BirdsiteLive.Domain if (!sigValidation.SignatureIsValidated) return false; // Save Follow in DB - var followerUserName = sigValidation.User.name.ToLowerInvariant(); + var followerUserName = sigValidation.User.preferredUsername.ToLowerInvariant(); var followerHost = sigValidation.User.url.Replace("https://", string.Empty).Split('/').First(); var followerInbox = sigValidation.User.inbox; var followerSharedInbox = sigValidation.User?.endpoints?.sharedInbox; From fbfef2d37b45b1764f5bac1193eef01a4a7ea464 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 20 Nov 2020 19:00:31 -0500 Subject: [PATCH 56/67] fix url --- src/BirdsiteLive.Domain/ActivityPubService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs index 64f5705..7101c23 100644 --- a/src/BirdsiteLive.Domain/ActivityPubService.cs +++ b/src/BirdsiteLive.Domain/ActivityPubService.cs @@ -96,7 +96,7 @@ namespace BirdsiteLive.Domain var httpRequestMessage = new HttpRequestMessage { Method = HttpMethod.Post, - RequestUri = new Uri($"https://{targetHost}/{usedInbox}"), + RequestUri = new Uri($"https://{targetHost}{usedInbox}"), Headers = { {"Host", targetHost}, From a965b013e3278a05e0548f0b22fa06e00ddae540 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Fri, 20 Nov 2020 20:21:44 -0500 Subject: [PATCH 57/67] added digest to call signature --- src/BirdsiteLive.Domain/ActivityPubService.cs | 20 ++++++++++++++++--- src/BirdsiteLive.Domain/CryptoService.cs | 8 ++++---- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs index 7101c23..21ee7fd 100644 --- a/src/BirdsiteLive.Domain/ActivityPubService.cs +++ b/src/BirdsiteLive.Domain/ActivityPubService.cs @@ -2,6 +2,7 @@ using System.Linq; using System.Net; using System.Net.Http; +using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; using BirdsiteLive.ActivityPub; @@ -88,9 +89,10 @@ namespace BirdsiteLive.Domain var date = DateTime.UtcNow.ToUniversalTime(); var httpDate = date.ToString("r"); - var signature = _cryptoService.SignAndGetSignatureHeader(date, actorUrl, targetHost, usedInbox); - + var digest = ComputeSha256Hash(json); + + var signature = _cryptoService.SignAndGetSignatureHeader(date, actorUrl, targetHost, digest, usedInbox); var client = new HttpClient(); var httpRequestMessage = new HttpRequestMessage @@ -101,7 +103,8 @@ namespace BirdsiteLive.Domain { {"Host", targetHost}, {"Date", httpDate}, - {"Signature", signature} + {"Signature", signature}, + {"Digest", $"SHA-256={digest}"} }, Content = new StringContent(json, Encoding.UTF8, "application/ld+json") }; @@ -109,5 +112,16 @@ namespace BirdsiteLive.Domain var response = await client.SendAsync(httpRequestMessage); return response.StatusCode; } + + static string ComputeSha256Hash(string rawData) + { + // Create a SHA256 + using (SHA256 sha256Hash = SHA256.Create()) + { + // ComputeHash - returns byte array + byte[] bytes = sha256Hash.ComputeHash(Encoding.UTF8.GetBytes(rawData)); + return Convert.ToBase64String(bytes); + } + } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/CryptoService.cs b/src/BirdsiteLive.Domain/CryptoService.cs index ed62a59..837922e 100644 --- a/src/BirdsiteLive.Domain/CryptoService.cs +++ b/src/BirdsiteLive.Domain/CryptoService.cs @@ -7,7 +7,7 @@ namespace BirdsiteLive.Domain public interface ICryptoService { string GetUserPem(string id); - string SignAndGetSignatureHeader(DateTime date, string actor, string host, string inbox = null); + string SignAndGetSignatureHeader(DateTime date, string actor, string host, string digest, string inbox); } public class CryptoService : ICryptoService @@ -33,7 +33,7 @@ namespace BirdsiteLive.Domain /// in the form of https://domain.io/actor /// in the form of domain.io /// - public string SignAndGetSignatureHeader(DateTime date, string actor, string targethost, string inbox = null) + public string SignAndGetSignatureHeader(DateTime date, string actor, string targethost, string digest, string inbox) { var usedInbox = "/inbox"; if (!string.IsNullOrWhiteSpace(inbox)) @@ -41,12 +41,12 @@ namespace BirdsiteLive.Domain var httpDate = date.ToString("r"); - var signedString = $"(request-target): post {usedInbox}\nhost: {targethost}\ndate: {httpDate}"; + var signedString = $"(request-target): post {usedInbox}\nhost: {targethost}\ndate: {httpDate}\ndigest: SHA-256={digest}"; var signedStringBytes = Encoding.UTF8.GetBytes(signedString); var signature = _magicKeyFactory.GetMagicKey().Sign(signedStringBytes); var sig64 = Convert.ToBase64String(signature); - var header = "keyId=\"" + actor + "\",headers=\"(request-target) host date\",signature=\"" + sig64 + "\""; + var header = "keyId=\"" + actor + "\",algorithm=\"rsa-sha256\",headers=\"(request-target) host date digest\",signature=\"" + sig64 + "\""; return header; } } From c86847a14608e9bc95136320adbc39cd506ab460 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Sat, 21 Nov 2020 18:54:16 -0500 Subject: [PATCH 58/67] give actor the followers collection --- src/BirdsiteLive.ActivityPub/Models/Actor.cs | 2 ++ .../Models/Followers.cs | 15 +++++++++++++ src/BirdsiteLive.Domain/UserService.cs | 3 ++- .../Controllers/UsersController.cs | 21 +++++++++++++++++-- 4 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 src/BirdsiteLive.ActivityPub/Models/Followers.cs diff --git a/src/BirdsiteLive.ActivityPub/Models/Actor.cs b/src/BirdsiteLive.ActivityPub/Models/Actor.cs index d517dc8..0552f25 100644 --- a/src/BirdsiteLive.ActivityPub/Models/Actor.cs +++ b/src/BirdsiteLive.ActivityPub/Models/Actor.cs @@ -12,11 +12,13 @@ namespace BirdsiteLive.ActivityPub public string[] context { get; set; } = new[] { "https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1" }; public string id { get; set; } public string type { get; set; } + public string followers { get; set; } public string preferredUsername { get; set; } public string name { get; set; } public string summary { get; set; } public string url { get; set; } public string inbox { get; set; } + public bool? discoverable { get; set; } = true; public PublicKey publicKey { get; set; } public Image icon { get; set; } public Image image { get; set; } diff --git a/src/BirdsiteLive.ActivityPub/Models/Followers.cs b/src/BirdsiteLive.ActivityPub/Models/Followers.cs new file mode 100644 index 0000000..85c44d2 --- /dev/null +++ b/src/BirdsiteLive.ActivityPub/Models/Followers.cs @@ -0,0 +1,15 @@ +using BirdsiteLive.ActivityPub.Converters; +using Newtonsoft.Json; + +namespace BirdsiteLive.ActivityPub.Models +{ + public class Followers + { + [JsonProperty("@context")] + [JsonConverter(typeof(ContextArrayConverter))] + public string context { get; set; } = "https://www.w3.org/ns/activitystreams"; + + public string id { get; set; } + public string type { get; set; } = "OrderedCollection"; + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index f2482d1..1e6e8dc 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -48,7 +48,8 @@ namespace BirdsiteLive.Domain var user = new Actor { id = $"https://{_instanceSettings.Domain}/users/{twitterUser.Acct}", - type = "Person", + type = "Service", //Person Service + followers = $"https://{_instanceSettings.Domain}/users/{twitterUser.Acct}/followers", preferredUsername = twitterUser.Acct, name = twitterUser.Name, inbox = $"https://{_instanceSettings.Domain}/users/{twitterUser.Acct}/inbox", diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index 3f2d701..0f6d7e3 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -3,9 +3,11 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Mime; +using System.Runtime.InteropServices.WindowsRuntime; using System.Threading; using System.Threading.Tasks; using BirdsiteLive.ActivityPub; +using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.Common.Settings; using BirdsiteLive.Domain; using BirdsiteLive.Models; @@ -56,7 +58,7 @@ namespace BirdsiteLive.Controllers Acct = user.Acct, Url = user.Url, ProfileImageUrl = user.ProfileImageUrl, - + InstanceHandle = $"@{user.Acct}@{_instanceSettings.Domain}" }; return View(displayableUser); @@ -95,7 +97,7 @@ namespace BirdsiteLive.Controllers using (var reader = new StreamReader(Request.Body)) { var body = await reader.ReadToEndAsync(); - System.IO.File.WriteAllText($@"C:\apdebug\{Guid.NewGuid()}.json", body); + //System.IO.File.WriteAllText($@"C:\apdebug\{Guid.NewGuid()}.json", body); var activity = ApDeserializer.ProcessActivity(body); // Do something @@ -130,6 +132,21 @@ namespace BirdsiteLive.Controllers return Accepted(); } + [Route("/users/{id}/followers")] + [HttpGet] + public async Task Followers(string id) + { + var r = Request.Headers["Accept"].First(); + if (!r.Contains("application/activity+json")) return NotFound(); + + var followers = new Followers + { + id = $"https://{_instanceSettings.Domain}/users/{id}/followers" + }; + var jsonApUser = JsonConvert.SerializeObject(followers); + return Content(jsonApUser, "application/activity+json; charset=utf-8"); + } + private Dictionary RequestHeaders(IHeaderDictionary header) { return header.ToDictionary, string, string>(h => h.Key.ToLowerInvariant(), h => h.Value); From 9ff5707e92ce8c04888f43990026cd83826596e3 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Thu, 3 Dec 2020 02:37:03 -0500 Subject: [PATCH 59/67] added AP call date check --- src/BirdsiteLive.Domain/UserService.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index 1e6e8dc..f1ccc13 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -164,6 +164,14 @@ namespace BirdsiteLive.Domain private async Task ValidateSignature(string actor, string rawSig, string method, string path, string queryString, Dictionary requestHeaders) { + //Check Date Validity + var date = requestHeaders["date"]; + var d = DateTime.Parse(date).ToUniversalTime(); + var now = DateTime.UtcNow; + var delta = Math.Abs((d - now).TotalSeconds); + if (delta > 30) return new SignatureValidationResult { SignatureIsValidated = false }; + + //Check Signature var signatures = rawSig.Split(','); var signature_header = new Dictionary(); foreach (var signature in signatures) From 5d86ebf61873aa1cf47c8512e06d796e60d284b8 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Mon, 28 Dec 2020 00:17:06 -0500 Subject: [PATCH 60/67] added version display, fix #15 --- src/BirdsiteLive/BirdsiteLive.csproj | 1 + src/BirdsiteLive/Controllers/WellKnownController.cs | 6 ++++-- src/BirdsiteLive/Views/Shared/_Layout.cshtml | 13 ++++++++----- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index 2195422..8b92b7c 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -4,6 +4,7 @@ netcoreapp3.1 d21486de-a812-47eb-a419-05682bb68856 Linux + 0.1.0 diff --git a/src/BirdsiteLive/Controllers/WellKnownController.cs b/src/BirdsiteLive/Controllers/WellKnownController.cs index 30a6f22..9ac3bb3 100644 --- a/src/BirdsiteLive/Controllers/WellKnownController.cs +++ b/src/BirdsiteLive/Controllers/WellKnownController.cs @@ -50,6 +50,8 @@ namespace BirdsiteLive.Controllers [Route("/nodeinfo/{id}.json")] public IActionResult NodeInfo(string id) { + var version = System.Reflection.Assembly.GetEntryAssembly().GetName().Version.ToString(3); + if (id == "2.0") { var nodeInfo = new NodeInfoV20 @@ -66,7 +68,7 @@ namespace BirdsiteLive.Controllers software = new Software() { name = "birdsitelive", - version = "0.1.0" + version = version }, protocols = new[] { @@ -101,7 +103,7 @@ namespace BirdsiteLive.Controllers software = new SoftwareV21() { name = "birdsitelive", - version = "0.1.0", + version = version, repository = "https://github.com/NicolasConstant/BirdsiteLive" }, protocols = new[] diff --git a/src/BirdsiteLive/Views/Shared/_Layout.cshtml b/src/BirdsiteLive/Views/Shared/_Layout.cshtml index 633fd0d..5275267 100644 --- a/src/BirdsiteLive/Views/Shared/_Layout.cshtml +++ b/src/BirdsiteLive/Views/Shared/_Layout.cshtml @@ -36,11 +36,14 @@ - +
+
+ + Github @*Privacy*@ + + BirdsiteLIVE @System.Reflection.Assembly.GetEntryAssembly().GetName().Version.ToString(3) +
+
From 20a05e9e9f202262d375045bbfc85fae458b4bba Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Mon, 28 Dec 2020 00:43:02 -0500 Subject: [PATCH 61/67] validate digest, fix #13 --- src/BirdsiteLive.Domain/ActivityPubService.cs | 13 ++---------- src/BirdsiteLive.Domain/CryptoService.cs | 13 ++++++++++++ src/BirdsiteLive.Domain/UserService.cs | 20 ++++++++++++------- .../Controllers/UsersController.cs | 4 ++-- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs index 21ee7fd..bbabf07 100644 --- a/src/BirdsiteLive.Domain/ActivityPubService.cs +++ b/src/BirdsiteLive.Domain/ActivityPubService.cs @@ -90,7 +90,7 @@ namespace BirdsiteLive.Domain var date = DateTime.UtcNow.ToUniversalTime(); var httpDate = date.ToString("r"); - var digest = ComputeSha256Hash(json); + var digest = _cryptoService.ComputeSha256Hash(json); var signature = _cryptoService.SignAndGetSignatureHeader(date, actorUrl, targetHost, digest, usedInbox); @@ -113,15 +113,6 @@ namespace BirdsiteLive.Domain return response.StatusCode; } - static string ComputeSha256Hash(string rawData) - { - // Create a SHA256 - using (SHA256 sha256Hash = SHA256.Create()) - { - // ComputeHash - returns byte array - byte[] bytes = sha256Hash.ComputeHash(Encoding.UTF8.GetBytes(rawData)); - return Convert.ToBase64String(bytes); - } - } + } } \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/CryptoService.cs b/src/BirdsiteLive.Domain/CryptoService.cs index 837922e..01e7d63 100644 --- a/src/BirdsiteLive.Domain/CryptoService.cs +++ b/src/BirdsiteLive.Domain/CryptoService.cs @@ -1,4 +1,5 @@ using System; +using System.Security.Cryptography; using System.Text; using BirdsiteLive.Domain.Factories; @@ -8,6 +9,7 @@ namespace BirdsiteLive.Domain { string GetUserPem(string id); string SignAndGetSignatureHeader(DateTime date, string actor, string host, string digest, string inbox); + string ComputeSha256Hash(string data); } public class CryptoService : ICryptoService @@ -49,5 +51,16 @@ namespace BirdsiteLive.Domain var header = "keyId=\"" + actor + "\",algorithm=\"rsa-sha256\",headers=\"(request-target) host date digest\",signature=\"" + sig64 + "\""; return header; } + + public string ComputeSha256Hash(string data) + { + // Create a SHA256 + using (SHA256 sha256Hash = SHA256.Create()) + { + // ComputeHash - returns byte array + byte[] bytes = sha256Hash.ComputeHash(Encoding.UTF8.GetBytes(data)); + return Convert.ToBase64String(bytes); + } + } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index f1ccc13..221ff4f 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -18,8 +18,8 @@ namespace BirdsiteLive.Domain public interface IUserService { Actor GetUser(TwitterUser twitterUser); - Task FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityFollow activity); - Task UndoFollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityUndoFollow activity); + Task FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityFollow activity, string body); + Task UndoFollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityUndoFollow activity, string body); } public class UserService : IUserService @@ -79,10 +79,10 @@ namespace BirdsiteLive.Domain return user; } - public async Task FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityFollow activity) + public async Task FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityFollow activity, string body) { // Validate - var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders); + var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body); if (!sigValidation.SignatureIsValidated) return false; // Save Follow in DB @@ -130,10 +130,10 @@ namespace BirdsiteLive.Domain } public async Task UndoFollowRequestedAsync(string signature, string method, string path, string queryString, - Dictionary requestHeaders, ActivityUndoFollow activity) + Dictionary requestHeaders, ActivityUndoFollow activity, string body) { // Validate - var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders); + var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body); if (!sigValidation.SignatureIsValidated) return false; // Save Follow in DB @@ -162,7 +162,7 @@ namespace BirdsiteLive.Domain return result == HttpStatusCode.Accepted; } - private async Task ValidateSignature(string actor, string rawSig, string method, string path, string queryString, Dictionary requestHeaders) + private async Task ValidateSignature(string actor, string rawSig, string method, string path, string queryString, Dictionary requestHeaders, string body) { //Check Date Validity var date = requestHeaders["date"]; @@ -171,6 +171,12 @@ namespace BirdsiteLive.Domain var delta = Math.Abs((d - now).TotalSeconds); if (delta > 30) return new SignatureValidationResult { SignatureIsValidated = false }; + //Check Digest + var digest = requestHeaders["digest"]; + var digestHash = digest.Split(new [] {"SHA-256="},StringSplitOptions.RemoveEmptyEntries).LastOrDefault(); + var calculatedDigestHash = _cryptoService.ComputeSha256Hash(body); + if (digestHash != calculatedDigestHash) return new SignatureValidationResult { SignatureIsValidated = false }; + //Check Signature var signatures = rawSig.Split(','); var signature_header = new Dictionary(); diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index 0f6d7e3..dd1b081 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -111,7 +111,7 @@ namespace BirdsiteLive.Controllers case "Follow": { var succeeded = await _userService.FollowRequestedAsync(signature, r.Method, r.Path, - r.QueryString.ToString(), RequestHeaders(r.Headers), activity as ActivityFollow); + r.QueryString.ToString(), RequestHeaders(r.Headers), activity as ActivityFollow, body); if (succeeded) return Accepted(); else return Unauthorized(); } @@ -119,7 +119,7 @@ namespace BirdsiteLive.Controllers if (activity is ActivityUndoFollow) { var succeeded = await _userService.UndoFollowRequestedAsync(signature, r.Method, r.Path, - r.QueryString.ToString(), RequestHeaders(r.Headers), activity as ActivityUndoFollow); + r.QueryString.ToString(), RequestHeaders(r.Headers), activity as ActivityUndoFollow, body); if (succeeded) return Accepted(); else return Unauthorized(); } From f585197e90d913dab4b7d50446c9f6c47933ab32 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Mon, 28 Dec 2020 01:33:09 -0500 Subject: [PATCH 62/67] created Dockerfile #11 --- src/BirdsiteLive/Dockerfile => Dockerfile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) rename src/BirdsiteLive/Dockerfile => Dockerfile (50%) diff --git a/src/BirdsiteLive/Dockerfile b/Dockerfile similarity index 50% rename from src/BirdsiteLive/Dockerfile rename to Dockerfile index 7945589..5601731 100644 --- a/src/BirdsiteLive/Dockerfile +++ b/Dockerfile @@ -6,15 +6,16 @@ EXPOSE 80 EXPOSE 443 FROM mcr.microsoft.com/dotnet/core/sdk:3.1-buster AS build -WORKDIR /src -COPY ["BirdsiteLive/BirdsiteLive.csproj", "BirdsiteLive/"] -RUN dotnet restore "BirdsiteLive/BirdsiteLive.csproj" +# WORKDIR /src +# COPY ["/src/BirdsiteLive/BirdsiteLive.csproj", "BirdsiteLive/"] COPY . . -WORKDIR "/src/BirdsiteLive" -RUN dotnet build "BirdsiteLive.csproj" -c Release -o /app/build +RUN dotnet restore "/src/BirdsiteLive/BirdsiteLive.csproj" +# COPY . . +# WORKDIR /BirdsiteLive +RUN dotnet build "/src/BirdsiteLive/BirdsiteLive.csproj" -c Release -o /app/build FROM build AS publish -RUN dotnet publish "BirdsiteLive.csproj" -c Release -o /app/publish +RUN dotnet publish "/src/BirdsiteLive/BirdsiteLive.csproj" -c Release -o /app/publish FROM base AS final WORKDIR /app From 56d3de862e89906fc0c207173a243a5398ad6fdb Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Mon, 28 Dec 2020 01:34:24 -0500 Subject: [PATCH 63/67] clean up --- Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5601731..0309a90 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,12 +6,8 @@ EXPOSE 80 EXPOSE 443 FROM mcr.microsoft.com/dotnet/core/sdk:3.1-buster AS build -# WORKDIR /src -# COPY ["/src/BirdsiteLive/BirdsiteLive.csproj", "BirdsiteLive/"] COPY . . RUN dotnet restore "/src/BirdsiteLive/BirdsiteLive.csproj" -# COPY . . -# WORKDIR /BirdsiteLive RUN dotnet build "/src/BirdsiteLive/BirdsiteLive.csproj" -c Release -o /app/build FROM build AS publish From d95debeff9e696a60e2f94a8a926de27fe8168a2 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Mon, 28 Dec 2020 01:36:30 -0500 Subject: [PATCH 64/67] only copy src folder --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0309a90..0dcccda 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ EXPOSE 80 EXPOSE 443 FROM mcr.microsoft.com/dotnet/core/sdk:3.1-buster AS build -COPY . . +COPY ./src/ ./src/ RUN dotnet restore "/src/BirdsiteLive/BirdsiteLive.csproj" RUN dotnet build "/src/BirdsiteLive/BirdsiteLive.csproj" -c Release -o /app/build From 790766f7532e42e411dfd3eaf0b2f135c81ff2e7 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Mon, 28 Dec 2020 02:19:11 -0500 Subject: [PATCH 65/67] better settings, ref #12 --- .../Settings/DbSettings.cs | 11 ++++++++ .../Settings/InstanceSettings.cs | 1 - src/BirdsiteLive.Common/Structs/DbTypes.cs | 7 +++++ src/BirdsiteLive/Startup.cs | 28 +++++++++++++------ src/BirdsiteLive/appsettings.json | 10 +++++-- 5 files changed, 46 insertions(+), 11 deletions(-) create mode 100644 src/BirdsiteLive.Common/Settings/DbSettings.cs create mode 100644 src/BirdsiteLive.Common/Structs/DbTypes.cs diff --git a/src/BirdsiteLive.Common/Settings/DbSettings.cs b/src/BirdsiteLive.Common/Settings/DbSettings.cs new file mode 100644 index 0000000..b70fba1 --- /dev/null +++ b/src/BirdsiteLive.Common/Settings/DbSettings.cs @@ -0,0 +1,11 @@ +namespace BirdsiteLive.Common.Settings +{ + public class DbSettings + { + public string Type { get; set; } + public string Host { get; set; } + public string Name { get; set; } + public string User { get; set; } + public string Password { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs index ce200d9..ba2a517 100644 --- a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs +++ b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs @@ -4,6 +4,5 @@ { public string Domain { get; set; } public string AdminEmail { get; set; } - public string PostgresConnString { get; set; } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Common/Structs/DbTypes.cs b/src/BirdsiteLive.Common/Structs/DbTypes.cs new file mode 100644 index 0000000..767f0f3 --- /dev/null +++ b/src/BirdsiteLive.Common/Structs/DbTypes.cs @@ -0,0 +1,7 @@ +namespace BirdsiteLive.Common.Structs +{ + public struct DbTypes + { + public static string Postgres = "postgres"; + } +} \ No newline at end of file diff --git a/src/BirdsiteLive/Startup.cs b/src/BirdsiteLive/Startup.cs index e31945f..c2d7cb0 100644 --- a/src/BirdsiteLive/Startup.cs +++ b/src/BirdsiteLive/Startup.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using BirdsiteLive.Common.Settings; +using BirdsiteLive.Common.Structs; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Postgres.DataAccessLayers; using BirdsiteLive.DAL.Postgres.Settings; @@ -51,15 +52,26 @@ namespace BirdsiteLive var instanceSettings = Configuration.GetSection("Instance").Get(); services.For().Use(x => instanceSettings); - var postgresSettings = new PostgresSettings - { - ConnString = instanceSettings.PostgresConnString - }; - services.For().Use(x => postgresSettings); + var dbSettings = Configuration.GetSection("Db").Get(); + services.For().Use(x => dbSettings); - services.For().Use().Singleton(); - services.For().Use().Singleton(); - services.For().Use().Singleton(); + if (string.Equals(dbSettings.Type, DbTypes.Postgres, StringComparison.OrdinalIgnoreCase)) + { + var connString = $"Host={dbSettings.Host};Username={dbSettings.User};Password={dbSettings.Password};Database={dbSettings.Name}"; + var postgresSettings = new PostgresSettings + { + ConnString = connString + }; + services.For().Use(x => postgresSettings); + + services.For().Use().Singleton(); + services.For().Use().Singleton(); + services.For().Use().Singleton(); + } + else + { + throw new NotImplementedException($"{dbSettings.Type} is not supported"); + } services.Scan(_ => { diff --git a/src/BirdsiteLive/appsettings.json b/src/BirdsiteLive/appsettings.json index 08c587a..4e6acb9 100644 --- a/src/BirdsiteLive/appsettings.json +++ b/src/BirdsiteLive/appsettings.json @@ -9,8 +9,14 @@ "AllowedHosts": "*", "Instance": { "Domain": "domain.name", - "AdminEmail": "me@domain.name", - "PostgresConnString": "Host=127.0.0.1;Username=username;Password=password;Database=mydb" + "AdminEmail": "me@domain.name" + }, + "Db": { + "Type": "postgres", + "Host": "127.0.0.1", + "Name": "mydb", + "User": "username", + "Password": "password" }, "Twitter": { "ConsumerKey": "twitter.api.key", From aaad00c9bf4be0810f2ca5d077a1f256143b6f91 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Mon, 28 Dec 2020 16:42:30 -0500 Subject: [PATCH 66/67] settings clean up --- src/BirdsiteLive.Common/Settings/TwitterSettings.cs | 2 -- src/BirdsiteLive/appsettings.json | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/BirdsiteLive.Common/Settings/TwitterSettings.cs b/src/BirdsiteLive.Common/Settings/TwitterSettings.cs index d0970f2..3e9095a 100644 --- a/src/BirdsiteLive.Common/Settings/TwitterSettings.cs +++ b/src/BirdsiteLive.Common/Settings/TwitterSettings.cs @@ -4,7 +4,5 @@ { public string ConsumerKey { get; set; } public string ConsumerSecret { get; set; } - public string AccessToken { get; set; } - public string AccessTokenSecret { get; set; } } } \ No newline at end of file diff --git a/src/BirdsiteLive/appsettings.json b/src/BirdsiteLive/appsettings.json index 4e6acb9..88712f8 100644 --- a/src/BirdsiteLive/appsettings.json +++ b/src/BirdsiteLive/appsettings.json @@ -20,8 +20,6 @@ }, "Twitter": { "ConsumerKey": "twitter.api.key", - "ConsumerSecret": "twitter.api.key", - "AccessToken": "twitter.api.key", - "AccessTokenSecret": "twitter.api.key" + "ConsumerSecret": "twitter.api.key" } } From 037d7050b364dac394d8de08353f5129484c9a57 Mon Sep 17 00:00:00 2001 From: Nicolas Constant Date: Tue, 29 Dec 2020 00:44:59 -0500 Subject: [PATCH 67/67] created docker-compose --- docker-compose.yml | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..77dee53 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,39 @@ +version: "3" + +networks: + birdsitelivenetwork: + external: false + +services: + server: + image: nicolasconstant/birdsitelive:latest + restart: always + container_name: birdsitelive + environment: + - Instance:Domain=domain.name + - Instance:AdminEmail=name@domain.ext + - Db:Type=postgres + - Db:Host=db + - Db:Name=birdsitelive + - Db:User=birdsitelive + - Db:Password=birdsitelive + - Twitter:ConsumerKey=twitter.api.key + - Twitter:ConsumerSecret=twitter.api.key + networks: + - birdsitelivenetwork + ports: + - "5000:80" + depends_on: + - db + + db: + image: postgres:9.6 + restart: always + environment: + - POSTGRES_USER=birdsitelive + - POSTGRES_PASSWORD=birdsitelive + - POSTGRES_DB=birdsitelive + networks: + - birdsitelivenetwork + volumes: + - ./postgres:/var/lib/postgresql/data \ No newline at end of file