From 5d521ad1992862de307bedeeda2d589885c8c523 Mon Sep 17 00:00:00 2001 From: Dull Bananas Date: Mon, 14 Jul 2025 17:55:05 -0700 Subject: [PATCH 1/3] add is_counted columns --- crates/api/api_utils/src/utils.rs | 1 + crates/db_schema/src/impls/comment.rs | 1 + crates/db_schema/src/impls/post.rs | 1 + crates/db_schema/src/source/comment.rs | 2 ++ crates/db_schema/src/source/comment_report.rs | 2 ++ .../db_schema/src/source/community_report.rs | 2 ++ crates/db_schema/src/source/post.rs | 4 +++ crates/db_schema/src/source/post_report.rs | 2 ++ crates/db_schema/src/utils/queries.rs | 1 + crates/db_schema_file/src/schema.rs | 8 ++++++ .../2025-07-14-215605_scalable_count/down.sql | 1 + .../2025-07-14-215605_scalable_count/up.sql | 27 +++++++++++++++++++ 12 files changed, 52 insertions(+) create mode 100644 migrations/2025-07-14-215605_scalable_count/down.sql create mode 100644 migrations/2025-07-14-215605_scalable_count/up.sql diff --git a/crates/api/api_utils/src/utils.rs b/crates/api/api_utils/src/utils.rs index a29e8527db..1836efcdb7 100644 --- a/crates/api/api_utils/src/utils.rs +++ b/crates/api/api_utils/src/utils.rs @@ -1138,6 +1138,7 @@ mod tests { report_count: 0, unresolved_report_count: 0, federation_pending: false, + is_counted: todo!(), }; assert!(check_comment_depth(&comment).is_ok()); comment.path = Ltree("0.123.456".to_string()); diff --git a/crates/db_schema/src/impls/comment.rs b/crates/db_schema/src/impls/comment.rs index 3dbb677834..af931bc700 100644 --- a/crates/db_schema/src/impls/comment.rs +++ b/crates/db_schema/src/impls/comment.rs @@ -452,6 +452,7 @@ mod tests { report_count: 0, unresolved_report_count: 0, federation_pending: false, + is_counted: todo!(), }; let child_comment_form = CommentInsertForm::new( diff --git a/crates/db_schema/src/impls/post.rs b/crates/db_schema/src/impls/post.rs index 9ca9863386..729e9578eb 100644 --- a/crates/db_schema/src/impls/post.rs +++ b/crates/db_schema/src/impls/post.rs @@ -676,6 +676,7 @@ mod tests { scaled_rank: RANK_DEFAULT, unresolved_report_count: 0, federation_pending: false, + is_counted: todo!(), }; // Post Like diff --git a/crates/db_schema/src/source/comment.rs b/crates/db_schema/src/source/comment.rs index 6337fa65d1..5e6957a0af 100644 --- a/crates/db_schema/src/source/comment.rs +++ b/crates/db_schema/src/source/comment.rs @@ -63,6 +63,8 @@ pub struct Comment { /// If a local user comments in a remote community, the comment is hidden until it is confirmed /// accepted by the community (by receiving it back via federation). pub federation_pending: bool, + #[serde(skip)] + pub is_counted: bool, } #[derive(Debug, Clone, derive_new::new)] diff --git a/crates/db_schema/src/source/comment_report.rs b/crates/db_schema/src/source/comment_report.rs index de000cc6a5..48bc8a1323 100644 --- a/crates/db_schema/src/source/comment_report.rs +++ b/crates/db_schema/src/source/comment_report.rs @@ -28,6 +28,8 @@ pub struct CommentReport { pub published_at: DateTime, pub updated_at: Option>, pub violates_instance_rules: bool, + #[serde(skip)] + pub is_counted: bool, } #[derive(Clone)] diff --git a/crates/db_schema/src/source/community_report.rs b/crates/db_schema/src/source/community_report.rs index 5f4844ce17..9e7ed39983 100644 --- a/crates/db_schema/src/source/community_report.rs +++ b/crates/db_schema/src/source/community_report.rs @@ -35,6 +35,8 @@ pub struct CommunityReport { pub resolver_id: Option, pub published_at: DateTime, pub updated_at: Option>, + #[serde(skip)] + pub is_counted: bool, } #[derive(Clone)] diff --git a/crates/db_schema/src/source/post.rs b/crates/db_schema/src/source/post.rs index d4bceb1198..16a8339253 100644 --- a/crates/db_schema/src/source/post.rs +++ b/crates/db_schema/src/source/post.rs @@ -84,6 +84,8 @@ pub struct Post { /// If a local user posts in a remote community, the comment is hidden until it is confirmed /// accepted by the community (by receiving it back via federation). pub federation_pending: bool, + #[serde(skip)] + pub is_counted: bool, } // TODO: FromBytes, ToBytes are only needed to develop wasm plugin, could be behind feature flag @@ -201,6 +203,8 @@ pub struct PostActions { pub like_score: Option, /// When the post was hidden. pub hidden_at: Option>, + #[serde(skip)] + pub like_is_counted: Option, } #[derive(Clone, derive_new::new)] diff --git a/crates/db_schema/src/source/post_report.rs b/crates/db_schema/src/source/post_report.rs index 6ddd1cbe04..8f73d5d340 100644 --- a/crates/db_schema/src/source/post_report.rs +++ b/crates/db_schema/src/source/post_report.rs @@ -33,6 +33,8 @@ pub struct PostReport { pub published_at: DateTime, pub updated_at: Option>, pub violates_instance_rules: bool, + #[serde(skip)] + pub is_counted: bool, } #[derive(Clone, Default)] diff --git a/crates/db_schema/src/utils/queries.rs b/crates/db_schema/src/utils/queries.rs index 1c3aebc82a..21ee5113b1 100644 --- a/crates/db_schema/src/utils/queries.rs +++ b/crates/db_schema/src/utils/queries.rs @@ -232,6 +232,7 @@ pub fn comment_select_remove_deletes() -> _ { comment::report_count, comment::unresolved_report_count, comment::federation_pending, + comment::is_counted, ) } diff --git a/crates/db_schema_file/src/schema.rs b/crates/db_schema_file/src/schema.rs index 2039690f55..db4ce326a6 100644 --- a/crates/db_schema_file/src/schema.rs +++ b/crates/db_schema_file/src/schema.rs @@ -143,6 +143,7 @@ diesel::table! { report_count -> Int2, unresolved_report_count -> Int2, federation_pending -> Bool, + is_counted -> Bool, } } @@ -153,6 +154,7 @@ diesel::table! { like_score -> Nullable, liked_at -> Nullable, saved_at -> Nullable, + like_is_counted -> Nullable, } } @@ -178,6 +180,7 @@ diesel::table! { published_at -> Timestamptz, updated_at -> Nullable, violates_instance_rules -> Bool, + is_counted -> Bool, } } @@ -249,6 +252,7 @@ diesel::table! { became_moderator_at -> Nullable, received_ban_at -> Nullable, ban_expires_at -> Nullable, + follow_is_counted -> Nullable, } } @@ -275,6 +279,7 @@ diesel::table! { resolver_id -> Nullable, published_at -> Timestamptz, updated_at -> Nullable, + is_counted -> Bool, } } @@ -930,6 +935,7 @@ diesel::table! { report_count -> Int2, unresolved_report_count -> Int2, federation_pending -> Bool, + is_counted -> Bool, } } @@ -944,6 +950,7 @@ diesel::table! { liked_at -> Nullable, like_score -> Nullable, hidden_at -> Nullable, + like_is_counted -> Nullable, } } @@ -962,6 +969,7 @@ diesel::table! { published_at -> Timestamptz, updated_at -> Nullable, violates_instance_rules -> Bool, + is_counted -> Bool, } } diff --git a/migrations/2025-07-14-215605_scalable_count/down.sql b/migrations/2025-07-14-215605_scalable_count/down.sql new file mode 100644 index 0000000000..d9a93fe9a1 --- /dev/null +++ b/migrations/2025-07-14-215605_scalable_count/down.sql @@ -0,0 +1 @@ +-- This file should undo anything in `up.sql` diff --git a/migrations/2025-07-14-215605_scalable_count/up.sql b/migrations/2025-07-14-215605_scalable_count/up.sql new file mode 100644 index 0000000000..6cd21579d6 --- /dev/null +++ b/migrations/2025-07-14-215605_scalable_count/up.sql @@ -0,0 +1,27 @@ +ALTER TABLE comment + ADD COLUMN is_counted boolean NOT NULL DEFAULT FALSE; + +ALTER TABLE comment_actions + ADD COLUMN like_is_counted boolean, + ADD CONSTRAINT comment_actions_check_like_is_counted CHECK ((liked_at IS NULL) = (like_is_counted IS NULL)); + +ALTER TABLE comment_report + ADD COLUMN is_counted boolean NOT NULL DEFAULT FALSE; + +ALTER TABLE community_actions + ADD COLUMN follow_is_counted boolean, + ADD CONSTRAINT community_actions_check_follow_is_counted CHECK ((followed_at IS NULL) = (follow_is_counted IS NULL)); + +ALTER TABLE community_report + ADD COLUMN is_counted boolean NOT NULL DEFAULT FALSE; + +ALTER TABLE post + ADD COLUMN is_counted boolean NOT NULL DEFAULT FALSE; + +ALTER TABLE post_actions + ADD COLUMN like_is_counted boolean, + ADD CONSTRAINT post_actions_check_like_is_counted CHECK ((liked_at IS NULL) = (like_is_counted IS NULL)); + +ALTER TABLE post_report + ADD COLUMN is_counted boolean NOT NULL DEFAULT FALSE; + From b44bfe02609d1b5ac322ab7dc754d0b33033ddd1 Mon Sep 17 00:00:00 2001 From: Dull Bananas Date: Wed, 16 Jul 2025 23:00:32 -0700 Subject: [PATCH 2/3] add stuff to scheduled_tasks.rs, not including the inner queries for update --- crates/routes/src/utils/scheduled_tasks.rs | 59 +++++++++++++++++++++- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/crates/routes/src/utils/scheduled_tasks.rs b/crates/routes/src/utils/scheduled_tasks.rs index ac4b4bc9b6..5a33343b66 100644 --- a/crates/routes/src/utils/scheduled_tasks.rs +++ b/crates/routes/src/utils/scheduled_tasks.rs @@ -13,7 +13,7 @@ use diesel::{ QueryDsl, QueryableByName, }; -use diesel_async::{AsyncPgConnection, RunQueryDsl}; +use diesel_async::{AsyncPgConnection, RunQueryDsl, SimpleAsyncConnection}; use diesel_uplete::uplete; use lemmy_api_utils::{ context::LemmyContext, @@ -49,7 +49,8 @@ use lemmy_db_schema_file::schema::{ use lemmy_db_views_site::SiteView; use lemmy_utils::error::{LemmyErrorType, LemmyResult}; use reqwest_middleware::ClientWithMiddleware; -use std::time::Duration; +use std::{convert::Infallible, time::Duration}; +use tokio::sync::Notify; use tracing::{info, warn}; /// Schedules various cleanup tasks for lemmy in a background thread @@ -128,6 +129,29 @@ pub async fn setup(context: Data) -> LemmyResult<()> { } }); + let context_1 = context.clone(); + // Update counters. + // + // This handles high-frequency changes more efficiently than triggers would. For example, if a + // counter update begins, and two posts are created during that update, then the next counter + // update will increment counters by 2. With triggers, the first post's insert transaction would + // include incrementing the counter by 1, and the second post's insert transaction would include + // waiting for the first post's insert transaction to finish before incrementing the counter by 1. + // + // To prevent users from sensing imperfectness or being misled because of delayed counter updates, + // queries use the `count` aggregate function to count items that have not yet been handled by a + // counter update. To minimize the chance that this involves scanning anything other than an empty + // index, the delay of a counter update is minimized by responding to notifications (sent by + // triggers) instead of schedule intervals. + tokio::spawn(async move { + loop { + match counter_update_loop(&mut context_1.pool()).await { + Ok(never) => match never {}, + Err(e) => warn!("Counter update loop unexpectedly stopped and will restart: {e}"), + } + } + }); + // Manually run the scheduler in an event loop loop { scheduler.run_pending().await; @@ -561,6 +585,37 @@ async fn build_update_instance_form( Some(instance_form) } +async fn counter_update_loop(pool: &mut DbPool<'_>) -> LemmyResult { + let mut conn = get_conn(pool).await?; + conn.batch_execute("LISTEN need_counter_update").await?; + let mut stream = conn.notification_stream(); + + let wakeup = Notify::new(); + + // To handle crashes and prevent race conditions, unconditionally update counters after the + // `LISTEN` statement. + wakeup.notify_one(); + + let listen_to_db = async { + loop { + let _: PgNotification = stream + .next + .await + .context("unexpected end of notification stream")??; + wakeup.notify_one(); + } + }; + + let update_when_woke = async { + loop { + wakeup.notified().await; + // TODO: update counters here + } + }; + + tokio::select! {result = listen_to_db => result, result = update_when_woke => result} +} + #[cfg(test)] mod tests { From be167b8c629241665b4e094f4f14adab70188066 Mon Sep 17 00:00:00 2001 From: Dull Bananas Date: Sat, 19 Jul 2025 21:55:46 -0700 Subject: [PATCH 3/3] note to self --- crates/routes/src/utils/scheduled_tasks.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/routes/src/utils/scheduled_tasks.rs b/crates/routes/src/utils/scheduled_tasks.rs index 5a33343b66..5d67470b1b 100644 --- a/crates/routes/src/utils/scheduled_tasks.rs +++ b/crates/routes/src/utils/scheduled_tasks.rs @@ -609,7 +609,7 @@ async fn counter_update_loop(pool: &mut DbPool<'_>) -> LemmyResult { let update_when_woke = async { loop { wakeup.notified().await; - // TODO: update counters here + // TODO: update counters here (there must be an additional filter to prevent integer overflow errors from being thrown) } };