From e3fb78e07a2daa9fa1f27d7551a95d8cae38ea6f Mon Sep 17 00:00:00 2001 From: malsadev Date: Sun, 8 Mar 2026 17:38:26 -0400 Subject: [PATCH 01/12] Add additional_stats columns to local_site --- crates/db_schema/src/source/local_site.rs | 10 +++++++ crates/db_schema_file/src/schema.rs | 10 +++++++ .../down.sql | 30 +++++++++++++++++++ .../up.sql | 30 +++++++++++++++++++ 4 files changed, 80 insertions(+) create mode 100644 migrations/2026-03-08-173221-0000_additional-statistics/down.sql create mode 100644 migrations/2026-03-08-173221-0000_additional-statistics/up.sql diff --git a/crates/db_schema/src/source/local_site.rs b/crates/db_schema/src/source/local_site.rs index 5b945c465e..84f8a3566a 100644 --- a/crates/db_schema/src/source/local_site.rs +++ b/crates/db_schema/src/source/local_site.rs @@ -95,6 +95,16 @@ pub struct LocalSite { #[serde(skip)] pub system_account: PersonId, pub default_items_per_page: i32, + pub linked_instances: Option, + pub total_posts: Option, + pub total_comments: Option, + pub total_users: Option, + pub total_communities: Option, + pub user_retention_percent: Option, + pub local_post_english_percent: Option, + pub ban_rate: Option, + pub accepted_signups_rate: Option, + pub failed_signups_rate: Option, } #[derive(Clone, derive_new::new)] diff --git a/crates/db_schema_file/src/schema.rs b/crates/db_schema_file/src/schema.rs index 27dea21895..5230203410 100644 --- a/crates/db_schema_file/src/schema.rs +++ b/crates/db_schema_file/src/schema.rs @@ -413,6 +413,16 @@ diesel::table! { suggested_multi_community_id -> Nullable, system_account -> Int4, default_items_per_page -> Int4, + linked_instances -> Nullable, + total_posts -> Nullable, + total_comments -> Nullable, + total_users -> Nullable, + total_communities -> Nullable, + user_retention_percent -> Nullable, + local_post_english_percent -> Nullable, + ban_rate -> Nullable, + accepted_signups_rate -> Nullable, + failed_signups_rate -> Nullable, } } diff --git a/migrations/2026-03-08-173221-0000_additional-statistics/down.sql b/migrations/2026-03-08-173221-0000_additional-statistics/down.sql new file mode 100644 index 0000000000..4ec77f2b70 --- /dev/null +++ b/migrations/2026-03-08-173221-0000_additional-statistics/down.sql @@ -0,0 +1,30 @@ +-- This file should undo anything in `up.sql` +ALTER TABLE local_site + DROP COLUMN linked_instances; + +ALTER TABLE local_site + DROP COLUMN total_posts; + +ALTER TABLE local_site + DROP COLUMN total_comments; + +ALTER TABLE local_site + DROP COLUMN total_users; + +ALTER TABLE local_site + DROP COLUMN total_communities; + +ALTER TABLE local_site + DROP COLUMN user_retention_percent; + +ALTER TABLE local_site + DROP COLUMN local_post_english_percent; + +ALTER TABLE local_site + DROP COLUMN ban_rate; + +ALTER TABLE local_site + DROP COLUMN accepted_signups_rate; + +ALTER TABLE local_site + DROP COLUMN failed_signups_rate; diff --git a/migrations/2026-03-08-173221-0000_additional-statistics/up.sql b/migrations/2026-03-08-173221-0000_additional-statistics/up.sql new file mode 100644 index 0000000000..c9a8fee85c --- /dev/null +++ b/migrations/2026-03-08-173221-0000_additional-statistics/up.sql @@ -0,0 +1,30 @@ +-- Your SQL goes here +ALTER TABLE local_site + ADD COLUMN linked_instances integer; + +ALTER TABLE local_site + ADD COLUMN total_posts integer; + +ALTER TABLE local_site + ADD COLUMN total_comments integer; + +ALTER TABLE local_site + ADD COLUMN total_users integer; + +ALTER TABLE local_site + ADD COLUMN total_communities integer; + +ALTER TABLE local_site + ADD COLUMN user_retention_percent integer; + +ALTER TABLE local_site + ADD COLUMN local_post_english_percent integer; + +ALTER TABLE local_site + ADD COLUMN ban_rate integer; + +ALTER TABLE local_site + ADD COLUMN accepted_signups_rate integer; + +ALTER TABLE local_site + ADD COLUMN failed_signups_rate integer; From 887ca0a5bf64fbcc5630aa464967b4eecc107c16 Mon Sep 17 00:00:00 2001 From: malsadev Date: Sun, 8 Mar 2026 17:51:01 -0400 Subject: [PATCH 02/12] Fix SQL formatting --- .../down.sql | 21 ++++++++++--------- .../up.sql | 21 ++++++++++--------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/migrations/2026-03-08-173221-0000_additional-statistics/down.sql b/migrations/2026-03-08-173221-0000_additional-statistics/down.sql index 4ec77f2b70..e9de18727f 100644 --- a/migrations/2026-03-08-173221-0000_additional-statistics/down.sql +++ b/migrations/2026-03-08-173221-0000_additional-statistics/down.sql @@ -1,30 +1,31 @@ -- This file should undo anything in `up.sql` ALTER TABLE local_site - DROP COLUMN linked_instances; + DROP COLUMN linked_instances; ALTER TABLE local_site - DROP COLUMN total_posts; + DROP COLUMN total_posts; ALTER TABLE local_site - DROP COLUMN total_comments; + DROP COLUMN total_comments; ALTER TABLE local_site - DROP COLUMN total_users; + DROP COLUMN total_users; ALTER TABLE local_site - DROP COLUMN total_communities; + DROP COLUMN total_communities; ALTER TABLE local_site - DROP COLUMN user_retention_percent; + DROP COLUMN user_retention_percent; ALTER TABLE local_site - DROP COLUMN local_post_english_percent; + DROP COLUMN local_post_english_percent; ALTER TABLE local_site - DROP COLUMN ban_rate; + DROP COLUMN ban_rate; ALTER TABLE local_site - DROP COLUMN accepted_signups_rate; + DROP COLUMN accepted_signups_rate; ALTER TABLE local_site - DROP COLUMN failed_signups_rate; + DROP COLUMN failed_signups_rate; + diff --git a/migrations/2026-03-08-173221-0000_additional-statistics/up.sql b/migrations/2026-03-08-173221-0000_additional-statistics/up.sql index c9a8fee85c..896e582074 100644 --- a/migrations/2026-03-08-173221-0000_additional-statistics/up.sql +++ b/migrations/2026-03-08-173221-0000_additional-statistics/up.sql @@ -1,30 +1,31 @@ -- Your SQL goes here ALTER TABLE local_site - ADD COLUMN linked_instances integer; + ADD COLUMN linked_instances integer; ALTER TABLE local_site - ADD COLUMN total_posts integer; + ADD COLUMN total_posts integer; ALTER TABLE local_site - ADD COLUMN total_comments integer; + ADD COLUMN total_comments integer; ALTER TABLE local_site - ADD COLUMN total_users integer; + ADD COLUMN total_users integer; ALTER TABLE local_site - ADD COLUMN total_communities integer; + ADD COLUMN total_communities integer; ALTER TABLE local_site - ADD COLUMN user_retention_percent integer; + ADD COLUMN user_retention_percent integer; ALTER TABLE local_site - ADD COLUMN local_post_english_percent integer; + ADD COLUMN local_post_english_percent integer; ALTER TABLE local_site - ADD COLUMN ban_rate integer; + ADD COLUMN ban_rate integer; ALTER TABLE local_site - ADD COLUMN accepted_signups_rate integer; + ADD COLUMN accepted_signups_rate integer; ALTER TABLE local_site - ADD COLUMN failed_signups_rate integer; + ADD COLUMN failed_signups_rate integer; + From da3ad084149c4836c66f173f461a5709699488d6 Mon Sep 17 00:00:00 2001 From: malsadev Date: Sun, 8 Mar 2026 23:39:00 -0400 Subject: [PATCH 03/12] Update linked instance count scheduled task (WIP) --- crates/routes/src/utils/scheduled_tasks.rs | 59 +++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/crates/routes/src/utils/scheduled_tasks.rs b/crates/routes/src/utils/scheduled_tasks.rs index 007c80b805..fb4d5b6478 100644 --- a/crates/routes/src/utils/scheduled_tasks.rs +++ b/crates/routes/src/utils/scheduled_tasks.rs @@ -9,7 +9,7 @@ use diesel::{ QueryDsl, QueryableByName, SelectableHelper, - dsl::{IntervalDsl, count, exists, not, update}, + dsl::{IntervalDsl, count, count_star, exists, not, update}, query_builder::AsQuery, sql_query, sql_types::{BigInt, Integer, Timestamptz}, @@ -34,7 +34,9 @@ use lemmy_db_schema_file::schema::{ comment, community, community_actions, + federation_allowlist, federation_blocklist, + federation_queue_state, instance, instance_actions, local_site, @@ -125,6 +127,10 @@ pub async fn setup(context: Data) -> LemmyResult<()> { .await .inspect_err(|e| warn!("Failed to update local user count: {e}")) .ok(); + update_stats(&mut context.pool()) + .await + .inspect_err(|e| warn!("Failed to update stats: {e}")) + .ok(); overwrite_deleted_posts_and_comments(&mut context.pool()) .await .inspect_err(|e| warn!("Failed to overwrite deleted posts/comments: {e}")) @@ -518,6 +524,33 @@ async fn update_local_user_count(pool: &mut DbPool<'_>) -> LemmyResult<()> { Ok(()) } +async fn update_stats(pool: &mut DbPool<'_>) -> LemmyResult<()> { + info!("Updating the linked instance count ..."); + + let conn = &mut get_conn(pool).await?; + let linked_instance_count = instance::table + // omit instance representing the local site + .left_join(site::table.left_join(local_site::table)) + .filter(local_site::id.is_null()) + .left_join(federation_blocklist::table) + .left_join(federation_allowlist::table) + .left_join(federation_queue_state::table) + .filter(federation_blocklist::instance_id.is_null()) + .select(count_star()) + .first::(conn) + .await + .map(i32::try_from)??; + + update(local_site::table) + .set(local_site::linked_instances.eq(linked_instance_count)) + .execute(conn) + .await?; + + info!("Done."); + + Ok(()) +} + /// Set banned to false after ban expires async fn update_banned_when_expired(pool: &mut DbPool<'_>) -> LemmyResult<()> { info!("Updating banned column if it expires ..."); @@ -790,4 +823,28 @@ mod tests { data.delete(pool).await?; Ok(()) } + + #[tokio::test] + #[serial] + async fn test_update_stats() -> LemmyResult<()> { + let context = LemmyContext::init_test_context().await; + let pool = &mut context.pool(); + // Setup local site + let data = TestData::create(pool).await?; + + // insert test data + let _instance0 = Instance::read_or_create(pool, "example0.com").await?; + let _instance1 = Instance::read_or_create(pool, "example1.com").await?; + + // run the query + update_stats(pool).await?; + let local_site_after = SiteView::read_local(pool).await?.local_site; + dbg!(&local_site_after); + + assert_eq!(2, local_site_after.linked_instances.unwrap()); + + data.delete(pool).await?; + Instance::delete_all(pool).await?; + Ok(()) + } } From 8a657e05015c65b6b6df532a097ef898a3f4ae1f Mon Sep 17 00:00:00 2001 From: malsadev Date: Tue, 10 Mar 2026 15:32:22 -0400 Subject: [PATCH 04/12] Rename local_site aggregate columns and make statistics non-nullable - Rename `users/posts/comments/communities` to `local_users/local_posts/local_comments/local_communities` to distinguish from the new total_* columns - Add `NOT NULL DEFAULT 0` to all new statistics columns (linked_instances, total_*, rates) instead of nullable - Update triggers.sql, nodeinfo.rs, convert.rs, and tests to use new column names --- crates/api/routes_v3/src/convert.rs | 8 +++--- crates/db_schema/src/impls/local_site.rs | 18 ++++++------ crates/db_schema/src/source/local_site.rs | 28 +++++++++---------- crates/db_schema_file/src/schema.rs | 28 +++++++++---------- .../replaceable_schema/triggers.sql | 6 ++-- crates/routes/src/nodeinfo.rs | 6 ++-- .../down.sql | 7 +++++ .../up.sql | 27 +++++++++++------- 8 files changed, 71 insertions(+), 57 deletions(-) diff --git a/crates/api/routes_v3/src/convert.rs b/crates/api/routes_v3/src/convert.rs index 6b56fd5b91..66a4c4620c 100644 --- a/crates/api/routes_v3/src/convert.rs +++ b/crates/api/routes_v3/src/convert.rs @@ -552,10 +552,10 @@ pub(crate) fn convert_site_view(site_view: SiteView) -> SiteViewV3 { let counts = SiteAggregates { site_id: SiteIdV3(site.id.0), - users: local_site.users.into(), - posts: local_site.posts.into(), - comments: local_site.comments.into(), - communities: local_site.communities.into(), + users: local_site.local_users.into(), + posts: local_site.local_posts.into(), + comments: local_site.local_comments.into(), + communities: local_site.local_communities.into(), users_active_day: local_site.users_active_day.into(), users_active_week: local_site.users_active_week.into(), users_active_month: local_site.users_active_month.into(), diff --git a/crates/db_schema/src/impls/local_site.rs b/crates/db_schema/src/impls/local_site.rs index 6254a6902b..4945b2f8a8 100644 --- a/crates/db_schema/src/impls/local_site.rs +++ b/crates/db_schema/src/impls/local_site.rs @@ -123,15 +123,15 @@ mod tests { // TODO: this is unstable, sometimes it returns 0 users, sometimes 1 //assert_eq!(0, site_aggregates_before_delete.users); - assert_eq!(1, site_aggregates_before_delete.communities); - assert_eq!(2, site_aggregates_before_delete.posts); - assert_eq!(2, site_aggregates_before_delete.comments); + assert_eq!(1, site_aggregates_before_delete.local_communities); + assert_eq!(2, site_aggregates_before_delete.local_posts); + assert_eq!(2, site_aggregates_before_delete.local_comments); // Try a post delete Post::delete(pool, inserted_post.id).await?; let site_aggregates_after_post_delete = read_local_site(pool).await?; - assert_eq!(1, site_aggregates_after_post_delete.posts); - assert_eq!(0, site_aggregates_after_post_delete.comments); + assert_eq!(1, site_aggregates_after_post_delete.local_posts); + assert_eq!(0, site_aggregates_after_post_delete.local_comments); // This shouuld delete all the associated rows, and fire triggers let person_num_deleted = Person::delete(pool, inserted_person.id).await?; @@ -163,7 +163,7 @@ mod tests { let (data, inserted_person, inserted_community) = prepare_site_with_community(pool).await?; let site_aggregates_before = read_local_site(pool).await?; - assert_eq!(1, site_aggregates_before.communities); + assert_eq!(1, site_aggregates_before.local_communities); Community::update( pool, @@ -176,7 +176,7 @@ mod tests { .await?; let site_aggregates_after_delete = read_local_site(pool).await?; - assert_eq!(0, site_aggregates_after_delete.communities); + assert_eq!(0, site_aggregates_after_delete.local_communities); Community::update( pool, @@ -199,7 +199,7 @@ mod tests { .await?; let site_aggregates_after_remove = read_local_site(pool).await?; - assert_eq!(0, site_aggregates_after_remove.communities); + assert_eq!(0, site_aggregates_after_remove.local_communities); Community::update( pool, @@ -212,7 +212,7 @@ mod tests { .await?; let site_aggregates_after_remove_delete = read_local_site(pool).await?; - assert_eq!(0, site_aggregates_after_remove_delete.communities); + assert_eq!(0, site_aggregates_after_remove_delete.local_communities); Community::delete(pool, inserted_community.id).await?; Person::delete(pool, inserted_person.id).await?; diff --git a/crates/db_schema/src/source/local_site.rs b/crates/db_schema/src/source/local_site.rs index b3f468630a..b2f3e8b882 100644 --- a/crates/db_schema/src/source/local_site.rs +++ b/crates/db_schema/src/source/local_site.rs @@ -78,10 +78,10 @@ pub struct LocalSite { pub default_post_time_range_seconds: Option, /// Block NSFW content being created pub disallow_nsfw_content: bool, - pub users: i32, - pub posts: i32, - pub comments: i32, - pub communities: i32, + pub local_users: i32, + pub local_posts: i32, + pub local_comments: i32, + pub local_communities: i32, /// The number of users with any activity in the last day. pub users_active_day: i32, /// The number of users with any activity in the last week. @@ -113,16 +113,16 @@ pub struct LocalSite { /// This affects post and comment images, but not avatars and banners. pub image_allow_video_uploads: bool, pub image_upload_disabled: bool, - pub linked_instances: Option, - pub total_posts: Option, - pub total_comments: Option, - pub total_users: Option, - pub total_communities: Option, - pub user_retention_percent: Option, - pub local_post_english_percent: Option, - pub ban_rate: Option, - pub accepted_signups_rate: Option, - pub failed_signups_rate: Option, + pub linked_instances: i32, + pub total_posts: i32, + pub total_comments: i32, + pub total_users: i32, + pub total_communities: i32, + pub user_retention_percent: i32, + pub local_post_english_percent: i32, + pub ban_rate: i32, + pub accepted_signups_rate: i32, + pub failed_signups_rate: i32, } #[derive(Clone, derive_new::new)] diff --git a/crates/db_schema_file/src/schema.rs b/crates/db_schema_file/src/schema.rs index 732e8524a9..7920f44a3b 100644 --- a/crates/db_schema_file/src/schema.rs +++ b/crates/db_schema_file/src/schema.rs @@ -406,10 +406,10 @@ diesel::table! { comment_downvotes -> FederationModeEnum, default_post_time_range_seconds -> Nullable, disallow_nsfw_content -> Bool, - users -> Int4, - posts -> Int4, - comments -> Int4, - communities -> Int4, + local_users -> Int4, + local_posts -> Int4, + local_comments -> Int4, + local_communities -> Int4, users_active_day -> Int4, users_active_week -> Int4, users_active_month -> Int4, @@ -427,16 +427,16 @@ diesel::table! { image_max_upload_size -> Int4, image_allow_video_uploads -> Bool, image_upload_disabled -> Bool, - linked_instances -> Nullable, - total_posts -> Nullable, - total_comments -> Nullable, - total_users -> Nullable, - total_communities -> Nullable, - user_retention_percent -> Nullable, - local_post_english_percent -> Nullable, - ban_rate -> Nullable, - accepted_signups_rate -> Nullable, - failed_signups_rate -> Nullable, + linked_instances -> Int4, + total_posts -> Int4, + total_comments -> Int4, + total_users -> Int4, + total_communities -> Int4, + user_retention_percent -> Int4, + local_post_english_percent -> Int4, + ban_rate -> Int4, + accepted_signups_rate -> Int4, + failed_signups_rate -> Int4, } } diff --git a/crates/diesel_utils/replaceable_schema/triggers.sql b/crates/diesel_utils/replaceable_schema/triggers.sql index 6bcda13efb..ebaa1ed690 100644 --- a/crates/diesel_utils/replaceable_schema/triggers.sql +++ b/crates/diesel_utils/replaceable_schema/triggers.sql @@ -169,7 +169,7 @@ WHERE UPDATE local_site AS a SET - comments = a.comments + diff.comments + local_comments = a.local_comments + diff.comments FROM ( SELECT coalesce(sum(count_diff), 0) AS comments @@ -223,7 +223,7 @@ WHERE UPDATE local_site AS a SET - posts = a.posts + diff.posts + local_posts = a.local_posts + diff.posts FROM ( SELECT coalesce(sum(count_diff), 0) AS posts @@ -242,7 +242,7 @@ BEGIN UPDATE local_site AS a SET - communities = a.communities + diff.communities + local_communities = a.local_communities + diff.communities FROM ( SELECT coalesce(sum(count_diff), 0) AS communities diff --git a/crates/routes/src/nodeinfo.rs b/crates/routes/src/nodeinfo.rs index 71e3fa7723..e6243942d3 100644 --- a/crates/routes/src/nodeinfo.rs +++ b/crates/routes/src/nodeinfo.rs @@ -59,12 +59,12 @@ async fn node_info(context: web::Data) -> Result Date: Wed, 11 Mar 2026 14:51:47 -0400 Subject: [PATCH 05/12] Fix SQL formatting --- migrations/2026-03-08-173221-0000_additional-statistics/down.sql | 1 + migrations/2026-03-08-173221-0000_additional-statistics/up.sql | 1 + 2 files changed, 2 insertions(+) diff --git a/migrations/2026-03-08-173221-0000_additional-statistics/down.sql b/migrations/2026-03-08-173221-0000_additional-statistics/down.sql index fce44e9017..a12fb7008d 100644 --- a/migrations/2026-03-08-173221-0000_additional-statistics/down.sql +++ b/migrations/2026-03-08-173221-0000_additional-statistics/down.sql @@ -36,3 +36,4 @@ ALTER TABLE local_site RENAME local_comments TO comments; ALTER TABLE local_site RENAME local_users TO users; ALTER TABLE local_site RENAME local_communities TO communities; + diff --git a/migrations/2026-03-08-173221-0000_additional-statistics/up.sql b/migrations/2026-03-08-173221-0000_additional-statistics/up.sql index 840caf26c0..d68ad2bffe 100644 --- a/migrations/2026-03-08-173221-0000_additional-statistics/up.sql +++ b/migrations/2026-03-08-173221-0000_additional-statistics/up.sql @@ -36,3 +36,4 @@ ALTER TABLE local_site RENAME comments TO local_comments; ALTER TABLE local_site RENAME users TO local_users; ALTER TABLE local_site RENAME communities TO local_communities; + From 1b148a5e8e423181fd768de836cd6f79bce63b0b Mon Sep 17 00:00:00 2001 From: malsadev Date: Thu, 12 Mar 2026 12:09:32 -0400 Subject: [PATCH 06/12] Add language_percentage_usage column --- .../2026-03-08-173221-0000_additional-statistics/down.sql | 3 +++ .../2026-03-08-173221-0000_additional-statistics/up.sql | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/migrations/2026-03-08-173221-0000_additional-statistics/down.sql b/migrations/2026-03-08-173221-0000_additional-statistics/down.sql index a12fb7008d..d751940e86 100644 --- a/migrations/2026-03-08-173221-0000_additional-statistics/down.sql +++ b/migrations/2026-03-08-173221-0000_additional-statistics/down.sql @@ -29,6 +29,9 @@ ALTER TABLE local_site ALTER TABLE local_site DROP COLUMN failed_signups_rate; +ALTER TABLE local_site + DROP COLUMN language_usage_percent; + ALTER TABLE local_site RENAME local_posts TO posts; ALTER TABLE local_site RENAME local_comments TO comments; diff --git a/migrations/2026-03-08-173221-0000_additional-statistics/up.sql b/migrations/2026-03-08-173221-0000_additional-statistics/up.sql index d68ad2bffe..5e27e6d13b 100644 --- a/migrations/2026-03-08-173221-0000_additional-statistics/up.sql +++ b/migrations/2026-03-08-173221-0000_additional-statistics/up.sql @@ -17,9 +17,6 @@ ALTER TABLE local_site ALTER TABLE local_site ADD COLUMN user_retention_percent integer NOT NULL DEFAULT 0; -ALTER TABLE local_site - ADD COLUMN local_post_english_percent integer NOT NULL DEFAULT 0; - ALTER TABLE local_site ADD COLUMN ban_rate integer NOT NULL DEFAULT 0; @@ -29,6 +26,9 @@ ALTER TABLE local_site ALTER TABLE local_site ADD COLUMN failed_signups_rate integer NOT NULL DEFAULT 0; +ALTER TABLE local_site + ADD COLUMN language_usage_percent jsonb NOT NULL DEFAULT '{}'::jsonb; + ALTER TABLE local_site RENAME posts TO local_posts; ALTER TABLE local_site RENAME comments TO local_comments; From 8cc4aafdce2716d23ec80b6ea5b32201fdea69bb Mon Sep 17 00:00:00 2001 From: malsadev Date: Thu, 12 Mar 2026 12:10:50 -0400 Subject: [PATCH 07/12] Update db_schema with language_usage_percent field --- crates/db_schema/src/source/local_site.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/db_schema/src/source/local_site.rs b/crates/db_schema/src/source/local_site.rs index b2f3e8b882..2cdcfcd8a8 100644 --- a/crates/db_schema/src/source/local_site.rs +++ b/crates/db_schema/src/source/local_site.rs @@ -15,6 +15,7 @@ use lemmy_db_schema_file::{ }, }; use serde::{Deserialize, Serialize}; +use serde_json::Value; use serde_with::skip_serializing_none; #[skip_serializing_none] @@ -119,10 +120,10 @@ pub struct LocalSite { pub total_users: i32, pub total_communities: i32, pub user_retention_percent: i32, - pub local_post_english_percent: i32, pub ban_rate: i32, pub accepted_signups_rate: i32, pub failed_signups_rate: i32, + pub language_usage_percent: Value, } #[derive(Clone, derive_new::new)] From 57a74e0dc194ae6bb4d61735d29b256ec4b8f322 Mon Sep 17 00:00:00 2001 From: malsadev Date: Thu, 12 Mar 2026 12:11:49 -0400 Subject: [PATCH 08/12] Update schema.rs --- crates/db_schema_file/src/schema.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/db_schema_file/src/schema.rs b/crates/db_schema_file/src/schema.rs index 7920f44a3b..a73bea0f1e 100644 --- a/crates/db_schema_file/src/schema.rs +++ b/crates/db_schema_file/src/schema.rs @@ -433,10 +433,10 @@ diesel::table! { total_users -> Int4, total_communities -> Int4, user_retention_percent -> Int4, - local_post_english_percent -> Int4, ban_rate -> Int4, accepted_signups_rate -> Int4, failed_signups_rate -> Int4, + language_usage_percent -> Jsonb, } } From 7e2fff7e083877b29f3d65ffc341c1e372807aa0 Mon Sep 17 00:00:00 2001 From: malsadev Date: Thu, 12 Mar 2026 12:13:43 -0400 Subject: [PATCH 09/12] Add full features flag to dump_schema script --- scripts/dump_schema.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dump_schema.sh b/scripts/dump_schema.sh index dd35fef99c..a541bd5d5f 100755 --- a/scripts/dump_schema.sh +++ b/scripts/dump_schema.sh @@ -9,7 +9,7 @@ cd "$CWD/../" source scripts/start_dev_db.sh -cargo run --package lemmy_diesel_utils +cargo run --package lemmy_diesel_utils --features full pg_dump --no-owner --no-privileges --no-table-access-method --schema-only --exclude-schema=r --no-sync -f schema.sqldump pg_ctl stop From a4b6b7acefdb98d769bc954365542835139e911b Mon Sep 17 00:00:00 2001 From: malsadev Date: Thu, 12 Mar 2026 12:27:49 -0400 Subject: [PATCH 10/12] Add test_process_language_breakdown test --- crates/routes/src/utils/scheduled_tasks.rs | 126 ++++++++++++++++++++- 1 file changed, 122 insertions(+), 4 deletions(-) diff --git a/crates/routes/src/utils/scheduled_tasks.rs b/crates/routes/src/utils/scheduled_tasks.rs index fb4d5b6478..bc3beb1036 100644 --- a/crates/routes/src/utils/scheduled_tasks.rs +++ b/crates/routes/src/utils/scheduled_tasks.rs @@ -730,6 +730,7 @@ mod tests { use lemmy_db_schema::{ source::{ community::{Community, CommunityInsertForm}, + language::Language, person::{Person, PersonInsertForm}, post::{Post, PostActions, PostInsertForm, PostLikeForm}, }, @@ -831,20 +832,137 @@ mod tests { let pool = &mut context.pool(); // Setup local site let data = TestData::create(pool).await?; + // insert some local data + let community = Community::create( + pool, + &CommunityInsertForm::new( + data.instance.id, + "name".to_owned(), + "title".to_owned(), + "pubkey".to_owned(), + ), + ) + .await?; + let person = Person::create( + pool, + &PersonInsertForm::new("felicity".to_owned(), "pubkey".to_owned(), data.instance.id), + ) + .await?; + let _post = Post::create( + pool, + &PostInsertForm::new("i am grrreat".to_owned(), person.id, community.id), + ) + .await?; + + // insert linked instances + let instance0 = Instance::read_or_create(pool, "example0.com").await?; + + // insert some federated data + let community = Community::create( + pool, + &CommunityInsertForm::new( + instance0.id, + "name".to_owned(), + "title".to_owned(), + "pubkey".to_owned(), + ), + ) + .await?; + let person = Person::create( + pool, + &PersonInsertForm::new("felicity".to_owned(), "pubkey".to_owned(), instance0.id), + ) + .await?; + let _post = Post::create( + pool, + &PostInsertForm::new("i am grrreat".to_owned(), person.id, community.id), + ) + .await?; - // insert test data - let _instance0 = Instance::read_or_create(pool, "example0.com").await?; let _instance1 = Instance::read_or_create(pool, "example1.com").await?; + let local_site_before = SiteView::read_local(pool).await?.local_site; + assert_eq!(0, local_site_before.linked_instances); + assert_eq!(0, local_site_before.total_posts); + assert_eq!(0, local_site_before.total_comments); + assert_eq!(0, local_site_before.total_users); + assert_eq!(0, local_site_before.total_communities); + // run the query update_stats(pool).await?; let local_site_after = SiteView::read_local(pool).await?.local_site; - dbg!(&local_site_after); - assert_eq!(2, local_site_after.linked_instances.unwrap()); + assert_eq!(2, local_site_after.linked_instances); + assert_eq!(2, local_site_after.total_posts); + assert_eq!(0, local_site_after.total_comments); + assert_eq!(4, local_site_after.total_users); + assert_eq!(2, local_site_after.total_communities); data.delete(pool).await?; Instance::delete_all(pool).await?; Ok(()) } + + #[tokio::test] + #[serial] + async fn test_process_language_breakdown() -> LemmyResult<()> { + let context = LemmyContext::init_test_context().await; + let pool = &mut context.pool(); + + let data = TestData::create(pool).await?; + let community = Community::create( + pool, + &CommunityInsertForm::new( + data.instance.id, + "name".to_owned(), + "title".to_owned(), + "pubkey".to_owned(), + ), + ) + .await?; + let person = Person::create( + pool, + &PersonInsertForm::new("felicity".to_owned(), "pubkey".to_owned(), data.instance.id), + ) + .await?; + + let en_id = Language::read_id_from_code(pool, "en").await?; + let de_id = Language::read_id_from_code(pool, "de").await?; + + // Create 2 English posts and 1 German post (expect 67% and 33%) + for _ in 0..2 { + Post::create( + pool, + &PostInsertForm { + language_id: Some(en_id), + ..PostInsertForm::new("english post".to_owned(), person.id, community.id) + }, + ) + .await?; + } + Post::create( + pool, + &PostInsertForm { + language_id: Some(de_id), + ..PostInsertForm::new("german post".to_owned(), person.id, community.id) + }, + ) + .await?; + + let conn = &mut get_conn(pool).await?; + process_language_breakdown(conn).await?; + let local_site = SiteView::read_local(pool).await?.local_site; + + assert_eq!( + local_site.language_usage_percent["en"].as_f64(), + Some(66.67) + ); + assert_eq!( + local_site.language_usage_percent["de"].as_f64(), + Some(33.33) + ); + + data.delete(pool).await?; + Ok(()) + } } From bc9679e362ef106a0c4bf0e89c2abee8b73574e6 Mon Sep 17 00:00:00 2001 From: malsadev Date: Thu, 12 Mar 2026 12:34:55 -0400 Subject: [PATCH 11/12] Add process_language_breakdown scheduled task --- Cargo.lock | 1 + crates/routes/Cargo.toml | 1 + crates/routes/src/utils/scheduled_tasks.rs | 103 ++++++++++++++++++++- 3 files changed, 103 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 79bc5ab691..444a1cd60e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4190,6 +4190,7 @@ dependencies = [ "rosetta-i18n", "rss", "serde", + "serde_json", "serial_test", "strum 0.28.0", "tokio", diff --git a/crates/routes/Cargo.toml b/crates/routes/Cargo.toml index 472d980c1f..259271f845 100644 --- a/crates/routes/Cargo.toml +++ b/crates/routes/Cargo.toml @@ -63,6 +63,7 @@ diesel-uplete.workspace = true lemmy_diesel_utils = { workspace = true } rosetta-i18n = { workspace = true } strum = { workspace = true } +serde_json = {workspace = true} [dev-dependencies] pretty_assertions.workspace = true diff --git a/crates/routes/src/utils/scheduled_tasks.rs b/crates/routes/src/utils/scheduled_tasks.rs index bc3beb1036..586146b772 100644 --- a/crates/routes/src/utils/scheduled_tasks.rs +++ b/crates/routes/src/utils/scheduled_tasks.rs @@ -5,11 +5,13 @@ use clokwerk::{AsyncScheduler, TimeUnits as CTimeUnits}; use diesel::{ BoolExpressionMethods, ExpressionMethods, + JoinOnDsl, NullableExpressionMethods, QueryDsl, QueryableByName, SelectableHelper, dsl::{IntervalDsl, count, count_star, exists, not, update}, + prelude::Queryable, query_builder::AsQuery, sql_query, sql_types::{BigInt, Integer, Timestamptz}, @@ -39,6 +41,7 @@ use lemmy_db_schema_file::schema::{ federation_queue_state, instance, instance_actions, + language, local_site, local_user, person, @@ -58,6 +61,7 @@ use lemmy_utils::{ error::{LemmyErrorType, LemmyResult}, }; use reqwest_middleware::ClientWithMiddleware; +use serde_json::{Map, Value}; use std::time::Duration; use tracing::{info, warn}; @@ -360,6 +364,14 @@ struct CommunityAggregatesUpdateResult { community_id: i32, } +#[derive(Queryable, Debug)] +struct PostCountSelectResult { + #[diesel(sql_type = VarChar)] + lang_code: String, + #[diesel(sql_type = Integer)] + post_count: i64, +} + /// Re-calculate the site and community active counts for a given interval async fn active_counts(pool: &mut DbPool<'_>, interval: (&str, &str)) -> LemmyResult<()> { info!( @@ -516,7 +528,7 @@ async fn update_local_user_count(pool: &mut DbPool<'_>) -> LemmyResult<()> { .map(i32::try_from)??; update(local_site::table) - .set(local_site::users.eq(user_count)) + .set(local_site::local_users.eq(user_count)) .execute(conn) .await?; @@ -525,7 +537,7 @@ async fn update_local_user_count(pool: &mut DbPool<'_>) -> LemmyResult<()> { } async fn update_stats(pool: &mut DbPool<'_>) -> LemmyResult<()> { - info!("Updating the linked instance count ..."); + info!("Updating stats ..."); let conn = &mut get_conn(pool).await?; let linked_instance_count = instance::table @@ -546,10 +558,97 @@ async fn update_stats(pool: &mut DbPool<'_>) -> LemmyResult<()> { .execute(conn) .await?; + let total_post_count = post::table + .select(count_star()) + .first::(conn) + .await + .map(i32::try_from)??; + + update(local_site::table) + .set(local_site::total_posts.eq(total_post_count)) + .execute(conn) + .await?; + + let total_comment_count = comment::table + .select(count_star()) + .first::(conn) + .await + .map(i32::try_from)??; + + update(local_site::table) + .set(local_site::total_comments.eq(total_comment_count)) + .execute(conn) + .await?; + + let total_user_count = person::table + .select(count_star()) + .first::(conn) + .await + .map(i32::try_from)??; + + update(local_site::table) + .set(local_site::total_users.eq(total_user_count)) + .execute(conn) + .await?; + + let total_community_count = community::table + .select(count_star()) + .first::(conn) + .await + .map(i32::try_from)??; + + update(local_site::table) + .set(local_site::total_communities.eq(total_community_count)) + .execute(conn) + .await?; + + process_language_breakdown(conn).await?; + info!("Done."); Ok(()) } +// Update db with percentage breakdown of local posts per language tag +async fn process_language_breakdown(conn: &mut AsyncPgConnection) -> LemmyResult<()> { + info!("Updating local language usage percentages ..."); + let local_post_count = local_site::table + .select(local_site::local_posts) + .get_result::(conn) + .await?; + + if local_post_count == 0 { + return Ok(()); + } + + let post_lang_breakdown = post::table + .inner_join(language::table.on(post::language_id.eq(language::id))) + .filter(post::local.eq(true)) + .group_by(language::code) + .select((language::code, count_star())) + .load::(conn) + .await?; + + let mut post_counts = Map::new(); + + for post_count in post_lang_breakdown { + post_counts.insert( + post_count.lang_code, + Value::Number( + serde_json::Number::from_f64( + (post_count.post_count as f64 * 10000.0 / local_post_count as f64).round() / 100.0, + ) + .unwrap_or(serde_json::Number::from(0)), + ), + ); + } + + update(local_site::table) + .set(local_site::language_usage_percent.eq(Value::Object(post_counts))) + .execute(conn) + .await?; + + Ok(()) +} /// Set banned to false after ban expires async fn update_banned_when_expired(pool: &mut DbPool<'_>) -> LemmyResult<()> { From 4811a93d740d8314fdab2f3c399f80221b58db63 Mon Sep 17 00:00:00 2001 From: malsadev Date: Thu, 12 Mar 2026 14:06:19 -0400 Subject: [PATCH 12/12] Clippy --- Cargo.lock | 1 + crates/db_schema/Cargo.toml | 2 +- crates/routes/Cargo.toml | 2 +- crates/routes/src/utils/scheduled_tasks.rs | 2 +- migrations/2026-03-08-173221-0000_additional-statistics/up.sql | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 444a1cd60e..d82d3380ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7386,6 +7386,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "756050066659291d47a554a9f558125db17428b073c5ffce1daf5dcb0f7231d8" dependencies = [ "chrono", + "serde_json", "thiserror 2.0.17", "ts-rs-macros", "url", diff --git a/crates/db_schema/Cargo.toml b/crates/db_schema/Cargo.toml index 4aef0421ba..b8b6426010 100644 --- a/crates/db_schema/Cargo.toml +++ b/crates/db_schema/Cargo.toml @@ -52,7 +52,7 @@ diesel-derive-newtype = { workspace = true, optional = true } diesel-async = { workspace = true, optional = true } diesel-uplete = { workspace = true, optional = true } diesel_ltree = { workspace = true, optional = true } -ts-rs = { workspace = true, optional = true } +ts-rs = { workspace = true, optional = true, features = ["serde-json-impl"] } tokio = { workspace = true, optional = true } i-love-jesus = { workspace = true, optional = true } derive-new.workspace = true diff --git a/crates/routes/Cargo.toml b/crates/routes/Cargo.toml index 259271f845..bd6af80da9 100644 --- a/crates/routes/Cargo.toml +++ b/crates/routes/Cargo.toml @@ -63,7 +63,7 @@ diesel-uplete.workspace = true lemmy_diesel_utils = { workspace = true } rosetta-i18n = { workspace = true } strum = { workspace = true } -serde_json = {workspace = true} +serde_json = { workspace = true } [dev-dependencies] pretty_assertions.workspace = true diff --git a/crates/routes/src/utils/scheduled_tasks.rs b/crates/routes/src/utils/scheduled_tasks.rs index 586146b772..3e8df27fb9 100644 --- a/crates/routes/src/utils/scheduled_tasks.rs +++ b/crates/routes/src/utils/scheduled_tasks.rs @@ -635,7 +635,7 @@ async fn process_language_breakdown(conn: &mut AsyncPgConnection) -> LemmyResult post_count.lang_code, Value::Number( serde_json::Number::from_f64( - (post_count.post_count as f64 * 10000.0 / local_post_count as f64).round() / 100.0, + (post_count.post_count as f64 * 10000.0 / f64::from(local_post_count)).round() / 100.0, ) .unwrap_or(serde_json::Number::from(0)), ), diff --git a/migrations/2026-03-08-173221-0000_additional-statistics/up.sql b/migrations/2026-03-08-173221-0000_additional-statistics/up.sql index 5e27e6d13b..dea1afb5be 100644 --- a/migrations/2026-03-08-173221-0000_additional-statistics/up.sql +++ b/migrations/2026-03-08-173221-0000_additional-statistics/up.sql @@ -27,7 +27,7 @@ ALTER TABLE local_site ADD COLUMN failed_signups_rate integer NOT NULL DEFAULT 0; ALTER TABLE local_site - ADD COLUMN language_usage_percent jsonb NOT NULL DEFAULT '{}'::jsonb; + ADD COLUMN language_usage_percent jsonb NOT NULL DEFAULT '{}'::jsonb; ALTER TABLE local_site RENAME posts TO local_posts;