sam/BirdsiteLIVE
sam/BirdsiteLIVE
Archived
1
0
Fork 0
Code Issues 1 Pull requests 1 Projects Releases Packages 1 Wiki Activity

db structure v3

Browse source
This commit is contained in:
Vincent Cloutier 2023-04-28 12:14:47 -04:00
parent ad79d183b4
commit 06bb1013ed
11 changed files with 72 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs
View file

@ -61,25 +61,25 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
{
// skip the first time to avoid sending backlog of tweet
var tweetId = tweets.Last().Id;
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now);
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, user.FetchingErrorCount, now);
}
else if (tweets.Length > 0 && user.LastTweetPostedId != -1)
{
userWtData.Tweets = tweets;
usersWtTweets.Add(userWtData);
var tweetId = tweets.Last().Id;
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now);
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, user.FetchingErrorCount, now);
}
else
{
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now);
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.FetchingErrorCount, now);
}
}
catch(Exception e)
{
_logger.LogError(e.Message);
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now);
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.FetchingErrorCount, now);
}
});
todo.Add(t);
@ -104,7 +104,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
if (user.LastTweetPostedId == -1)
tweets = await _twitterTweetsService.GetTimelineAsync(user.Acct);
else
tweets = await _twitterTweetsService.GetTimelineAsync(user.Acct, user.LastTweetSynchronizedForAllFollowersId);
tweets = await _twitterTweetsService.GetTimelineAsync(user.Acct, user.LastTweetPostedId);
}
catch (Exception e)
{

52
src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs
View file

@ -23,7 +23,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
public class DbInitializerPostgresDal : PostgresBase, IDbInitializerDal
{
private readonly PostgresTools _tools;
private readonly Version _currentVersion = new Version(2, 5);
private readonly Version _currentVersion = new Version(3, 0);
private const string DbVersionType = "db-version";
#region Ctor
@ -136,7 +136,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
new Tuple<Version, Version>(new Version(2,1), new Version(2,2)),
new Tuple<Version, Version>(new Version(2,2), new Version(2,3)),
new Tuple<Version, Version>(new Version(2,3), new Version(2,4)),
new Tuple<Version, Version>(new Version(2,4), new Version(2,5))
new Tuple<Version, Version>(new Version(2,4), new Version(2,5)),
new Tuple<Version, Version>(new Version(2,5), new Version(3,0))
};
}
@ -179,6 +180,48 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
await _tools.ExecuteRequestAsync(alterTwitterUserId);
}
else if (from == new Version(2, 5) && to == new Version(3, 0))
{
var dropFollowingSyncStatus = $@"ALTER TABLE {_settings.FollowersTableName} DROP COLUMN followingssyncstatus";
await _tools.ExecuteRequestAsync(dropFollowingSyncStatus);
var dropLastTweet = $@"ALTER TABLE {_settings.TwitterUserTableName} DROP COLUMN lasttweetsynchronizedforallfollowersid";
await _tools.ExecuteRequestAsync(dropLastTweet);
var addFediverseEquivalent = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD fediverseaccount text";
await _tools.ExecuteRequestAsync(addFediverseEquivalent);
var createWorkers = $@"CREATE TABLE {_settings.WorkersTableName}
(
id BIGINT PRIMARY KEY,
rangeStart INTEGER,
rangeEnd INTEGER,
lastSeen TIMESTAMP (2) WITHOUT TIME ZONE,
name text
);";
await _tools.ExecuteRequestAsync(createWorkers);
var createWorkerInstance = $@" INSERT INTO {_settings.WorkersTableName} (id,rangeStart,rangeEnd) VALUES(0,0, 100) ";
await _tools.ExecuteRequestAsync(createWorkerInstance);
var createInstagram = $@"CREATE TABLE {_settings.InstagramUserTableName}
(
id SERIAL PRIMARY KEY,
acct VARCHAR(20) UNIQUE,
data JSONB
);";
await _tools.ExecuteRequestAsync(createInstagram);
var createInstagramPost = $@"CREATE TABLE {_settings.CachedInstaPostsTableName}
(
id VARCHAR(30) PRIMARY KEY,
acct VARCHAR(20) UNIQUE,
data JSONB
);";
await _tools.ExecuteRequestAsync(createInstagramPost);
}
else
{
throw new NotImplementedException();
@ -207,7 +250,10 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
$@"DROP TABLE {_settings.DbVersionTableName};",
$@"DROP TABLE {_settings.TwitterUserTableName};",
$@"DROP TABLE {_settings.FollowersTableName};",
$@"DROP TABLE {_settings.CachedTweetsTableName};"
$@"DROP TABLE {_settings.CachedTweetsTableName};",
$@"DROP TABLE {_settings.InstagramUserTableName};",
$@"DROP TABLE {_settings.CachedInstaPostsTableName};",
$@"DROP TABLE {_settings.WorkersTableName};"
};
foreach (var r in dropsRequests)

1
src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs
View file

@ -105,7 +105,6 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
var followers = new List<Follower>();
while (await reader.ReadAsync())
{
string syncStatusString = reader["followingsSyncStatus"] as string;
followers.Add(new Follower
{
Id = reader["id"] as int? ?? default,

15
src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs
View file

@ -27,8 +27,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
using (var dbConnection = Connection)
{
await dbConnection.ExecuteAsync(
$"INSERT INTO {_settings.TwitterUserTableName} (acct,lastTweetPostedId,lastTweetSynchronizedForAllFollowersId) VALUES(@acct,@lastTweetPostedId,@lastTweetSynchronizedForAllFollowersId)",
new { acct, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = lastTweetPostedId });
$"INSERT INTO {_settings.TwitterUserTableName} (acct,lastTweetPostedId) VALUES(@acct,@lastTweetPostedId)",
new { acct, lastTweetPostedId });
}
}
@ -53,7 +53,6 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
Acct = reader["acct"] as string,
TwitterUserId = reader["twitterUserId"] as long? ?? default,
LastTweetPostedId = reader["lastTweetPostedId"] as long? ?? default,
LastTweetSynchronizedForAllFollowersId = reader["lastTweetSynchronizedForAllFollowersId"] as long? ?? default,
LastSync = reader["lastSync"] as DateTime? ?? default,
FetchingErrorCount = reader["fetchingErrorCount"] as int? ?? default,
};
@ -79,7 +78,6 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
Acct = reader["acct"] as string,
TwitterUserId = reader["twitterUserId"] as long? ?? default,
LastTweetPostedId = reader["lastTweetPostedId"] as long? ?? default,
LastTweetSynchronizedForAllFollowersId = reader["lastTweetSynchronizedForAllFollowersId"] as long? ?? default,
LastSync = reader["lastSync"] as DateTime? ?? default,
FetchingErrorCount = reader["fetchingErrorCount"] as int? ?? default,
};
@ -143,7 +141,6 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
Acct = reader["acct"] as string,
TwitterUserId = reader["twitterUserId"] as long? ?? default,
LastTweetPostedId = reader["lastTweetPostedId"] as long? ?? default,
LastTweetSynchronizedForAllFollowersId = reader["lastTweetSynchronizedForAllFollowersId"] as long? ?? default,
LastSync = reader["lastSync"] as DateTime? ?? default,
FetchingErrorCount = reader["fetchingErrorCount"] as int? ?? default,
}
@ -189,21 +186,19 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
await command.ExecuteNonQueryAsync();
}
public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync)
public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, int fetchingErrorCount, DateTime lastSync)
{
if(id == default) throw new ArgumentException("id");
if(lastTweetPostedId == default) throw new ArgumentException("lastTweetPostedId");
if(lastTweetSynchronizedForAllFollowersId == default) throw new ArgumentException("lastTweetSynchronizedForAllFollowersId");
if(lastSync == default) throw new ArgumentException("lastSync");
var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = $1, lastTweetSynchronizedForAllFollowersId = $2, fetchingErrorCount = $3, lastSync = $4 WHERE id = $5";
var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = $1, fetchingErrorCount = $2, lastSync = $3 WHERE id = $4";
await using var connection = DataSource.CreateConnection();
await connection.OpenAsync();
await using var command = new NpgsqlCommand(query, connection) {
Parameters = {
new() { Value = lastTweetPostedId},
new() { Value = lastTweetSynchronizedForAllFollowersId},
new() { Value = fetchingErrorCount},
new() { Value = lastSync.ToUniversalTime()},
new() { Value = id},
@ -215,7 +210,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
public async Task UpdateTwitterUserAsync(SyncTwitterUser user)
{
await UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, user.LastSync);
await UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.FetchingErrorCount, user.LastSync);
}
public async Task DeleteTwitterUserAsync(string acct)

3
src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Settings/PostgresSettings.cs
View file

@ -6,6 +6,9 @@
public string DbVersionTableName { get; set; } = "db_version";
public string TwitterUserTableName { get; set; } = "twitter_users";
public string InstagramUserTableName { get; set; } = "instagram_users";
public string WorkersTableName { get; set; } = "workers";
public string CachedInstaPostsTableName { get; set; } = "cached_insta_posts";
public string FollowersTableName { get; set; } = "followers";
public string CachedTweetsTableName { get; set; } = "cached_tweets";
}

2
src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs
View file

@ -12,7 +12,7 @@ namespace BirdsiteLive.DAL.Contracts
Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber, int nStart, int nEnd, int m);
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber);
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync();
Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync);
Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, int fetchingErrorCount, DateTime lastSync);
Task UpdateTwitterUserIdAsync(string username, long twitterUserId);
Task UpdateTwitterUserAsync(SyncTwitterUser user);
Task DeleteTwitterUserAsync(string acct);

1
src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs
View file

@ -9,7 +9,6 @@ namespace BirdsiteLive.DAL.Models
public string Acct { get; set; }
public long LastTweetPostedId { get; set; }
public long LastTweetSynchronizedForAllFollowersId { get; set; }
public DateTime LastSync { get; set; }

3
src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/Base/PostgresTestingBase.cs
View file

@ -19,6 +19,9 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers.Base
CachedTweetsTableName = "CachedTweetsTableName" + RandomGenerator.GetString(4),
FollowersTableName = "FollowersTableName" + RandomGenerator.GetString(4),
TwitterUserTableName = "TwitterUserTableName" + RandomGenerator.GetString(4),
InstagramUserTableName = "InstagramUserTableName" + RandomGenerator.GetString(4),
CachedInstaPostsTableName = "CachedInstaPosts" + RandomGenerator.GetString(4),
WorkersTableName = "workers" + RandomGenerator.GetString(4),
};
_tools = new PostgresTools(_settings);
}

29
src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs
View file

@ -48,7 +48,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
Assert.AreEqual(acct, result.Acct);
Assert.AreEqual(lastTweetId, result.LastTweetPostedId);
Assert.AreEqual(lastTweetId, result.LastTweetSynchronizedForAllFollowersId);
Assert.AreEqual(0, result.FetchingErrorCount);
Assert.IsTrue(result.Id > 0);
}
@ -67,7 +66,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
Assert.AreEqual(acct, resultById.Acct);
Assert.AreEqual(lastTweetId, resultById.LastTweetPostedId);
Assert.AreEqual(lastTweetId, resultById.LastTweetSynchronizedForAllFollowersId);
Assert.AreEqual(result.Id, resultById.Id);
}
@ -87,13 +85,12 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
var updatedLastSyncId = 1550L;
var now = DateTime.Now;
var errors = 15;
await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now);
await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, errors, now);
result = await dal.GetTwitterUserAsync(acct);
Assert.AreEqual(acct, result.Acct);
Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId);
Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId);
Assert.AreEqual(errors, result.FetchingErrorCount);
Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100);
}
@ -116,7 +113,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
var errors = 15;
result.LastTweetPostedId = updatedLastTweetId;
result.LastTweetSynchronizedForAllFollowersId = updatedLastSyncId;
result.FetchingErrorCount = errors;
result.LastSync = now;
await dal.UpdateTwitterUserAsync(result);
@ -125,7 +121,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
Assert.AreEqual(acct, result.Acct);
Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId);
Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId);
Assert.AreEqual(errors, result.FetchingErrorCount);
Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100);
}
@ -148,7 +143,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
var errors = 32768;
result.LastTweetPostedId = updatedLastTweetId;
result.LastTweetSynchronizedForAllFollowersId = updatedLastSyncId;
result.FetchingErrorCount = errors;
result.LastSync = now;
await dal.UpdateTwitterUserAsync(result);
@ -157,7 +151,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
Assert.AreEqual(acct, result.Acct);
Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId);
Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId);
Assert.AreEqual(errors, result.FetchingErrorCount);
Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100);
}
@ -167,7 +160,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
public async Task Update_NoId()
{
var dal = new TwitterUserPostgresDal(_settings);
await dal.UpdateTwitterUserAsync(default, default, default, default, DateTime.UtcNow);
await dal.UpdateTwitterUserAsync(default, default, default, DateTime.UtcNow);
}
[TestMethod]
@ -175,23 +168,16 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
public async Task Update_NoLastTweetPostedId()
{
var dal = new TwitterUserPostgresDal(_settings);
await dal.UpdateTwitterUserAsync(12, default, default, default, DateTime.UtcNow);
await dal.UpdateTwitterUserAsync(12, default, default, DateTime.UtcNow);
}
[TestMethod]
[ExpectedException(typeof(ArgumentException))]
public async Task Update_NoLastTweetSynchronizedForAllFollowersId()
{
var dal = new TwitterUserPostgresDal(_settings);
await dal.UpdateTwitterUserAsync(12, 9556, default, default, DateTime.UtcNow);
}
[TestMethod]
[ExpectedException(typeof(ArgumentException))]
public async Task Update_NoLastSync()
{
var dal = new TwitterUserPostgresDal(_settings);
await dal.UpdateTwitterUserAsync(12, 9556, 65, default, default);
await dal.UpdateTwitterUserAsync(12, 9556, 65, default);
}
[TestMethod]
@ -261,7 +247,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
Assert.IsFalse(result[0].Id == default);
Assert.IsFalse(result[0].Acct == default);
Assert.IsFalse(result[0].LastTweetPostedId == default);
Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
}
[TestMethod]
@ -318,7 +303,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
{
var user = allUsers[i];
var date = i % 2 == 0 ? oldest : newest;
await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date);
await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, 0, date);
}
var result = await dal.GetAllTwitterUsersAsync(10);
@ -326,7 +311,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
Assert.IsFalse(result[0].Id == default);
Assert.IsFalse(result[0].Acct == default);
Assert.IsFalse(result[0].LastTweetPostedId == default);
Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
foreach (var acc in result)
Assert.IsTrue(Math.Abs((acc.LastSync - oldest.ToUniversalTime()).TotalMilliseconds) < 1000);
@ -349,7 +333,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
Assert.IsFalse(result[0].Id == default);
Assert.IsFalse(result[0].Acct == default);
Assert.IsFalse(result[0].LastTweetPostedId == default);
Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
}
[TestMethod]
@ -382,7 +365,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
if (i == 0 || i == 2 || i == 3)
{
var t = await dal.GetTwitterUserAsync(acct);
await dal.UpdateTwitterUserAsync(t.Id ,1L,2L, 50+i*2, DateTime.Now);
await dal.UpdateTwitterUserAsync(t.Id ,1L, 50+i*2, DateTime.Now);
}
}

2
src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessFollowUserTests.cs
View file

@ -37,7 +37,6 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
Id = 2,
Acct = twitterName,
LastTweetPostedId = -1,
LastTweetSynchronizedForAllFollowersId = -1
};
#endregion
@ -112,7 +111,6 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
Id = 2,
Acct = twitterName,
LastTweetPostedId = -1,
LastTweetSynchronizedForAllFollowersId = -1
};
#endregion

2
src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessUnfollowUserTests.cs
View file

@ -97,7 +97,6 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
Id = 2,
Acct = twitterName,
LastTweetPostedId = 460,
LastTweetSynchronizedForAllFollowersId = 460
};
var followerList = new List<Follower>
@ -159,7 +158,6 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
Id = 2,
Acct = twitterName,
LastTweetPostedId = 460,
LastTweetSynchronizedForAllFollowersId = 460
};
var followerList = new List<Follower>();