diff --git a/Cargo.lock b/Cargo.lock index 78f5e41450..f6b5e3dc5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4227,6 +4227,7 @@ dependencies = [ "rosetta-i18n", "rss", "serde", + "serde_json", "serial_test", "strum 0.28.0", "tokio", @@ -7405,6 +7406,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "756050066659291d47a554a9f558125db17428b073c5ffce1daf5dcb0f7231d8" dependencies = [ "chrono", + "serde_json", "thiserror 2.0.18", "ts-rs-macros", "url", diff --git a/crates/api/routes_v3/src/convert.rs b/crates/api/routes_v3/src/convert.rs index c3d6703a82..5e5fa94e1a 100644 --- a/crates/api/routes_v3/src/convert.rs +++ b/crates/api/routes_v3/src/convert.rs @@ -556,10 +556,10 @@ pub(crate) fn convert_site_view(site_view: SiteView) -> SiteViewV3 { let counts = SiteAggregates { site_id: SiteIdV3(site.id.0), - users: local_site.users.into(), - posts: local_site.posts.into(), - comments: local_site.comments.into(), - communities: local_site.communities.into(), + users: local_site.local_users.into(), + posts: local_site.local_posts.into(), + comments: local_site.local_comments.into(), + communities: local_site.local_communities.into(), users_active_day: local_site.users_active_day.into(), users_active_week: local_site.users_active_week.into(), users_active_month: local_site.users_active_month.into(), diff --git a/crates/db_schema/Cargo.toml b/crates/db_schema/Cargo.toml index 4bd96ebf45..6f74218c64 100644 --- a/crates/db_schema/Cargo.toml +++ b/crates/db_schema/Cargo.toml @@ -51,7 +51,7 @@ diesel-derive-newtype = { workspace = true, optional = true } diesel-async = { workspace = true, optional = true } diesel-uplete = { workspace = true, optional = true } diesel_ltree = { workspace = true, optional = true } -ts-rs = { workspace = true, optional = true } +ts-rs = { workspace = true, optional = true, features = ["serde-json-impl"] } tokio = { workspace = true, optional = true } i-love-jesus = { workspace = true, optional = true } derive-new.workspace = true diff --git a/crates/db_schema/src/impls/local_site.rs b/crates/db_schema/src/impls/local_site.rs index bed35c28fc..16c97f838a 100644 --- a/crates/db_schema/src/impls/local_site.rs +++ b/crates/db_schema/src/impls/local_site.rs @@ -125,15 +125,15 @@ mod tests { // TODO: this is unstable, sometimes it returns 0 users, sometimes 1 //assert_eq!(0, site_aggregates_before_delete.users); - assert_eq!(1, site_aggregates_before_delete.communities); - assert_eq!(2, site_aggregates_before_delete.posts); - assert_eq!(2, site_aggregates_before_delete.comments); + assert_eq!(1, site_aggregates_before_delete.local_communities); + assert_eq!(2, site_aggregates_before_delete.local_posts); + assert_eq!(2, site_aggregates_before_delete.local_comments); // Try a post delete Post::delete(pool, inserted_post.id).await?; let site_aggregates_after_post_delete = read_local_site(pool).await?; - assert_eq!(1, site_aggregates_after_post_delete.posts); - assert_eq!(0, site_aggregates_after_post_delete.comments); + assert_eq!(1, site_aggregates_after_post_delete.local_posts); + assert_eq!(0, site_aggregates_after_post_delete.local_comments); // This shouuld delete all the associated rows, and fire triggers let person_num_deleted = Person::delete(pool, inserted_person.id).await?; @@ -165,7 +165,7 @@ mod tests { let (data, inserted_person, inserted_community) = prepare_site_with_community(pool).await?; let site_aggregates_before = read_local_site(pool).await?; - assert_eq!(1, site_aggregates_before.communities); + assert_eq!(1, site_aggregates_before.local_communities); Community::update( pool, @@ -178,7 +178,7 @@ mod tests { .await?; let site_aggregates_after_delete = read_local_site(pool).await?; - assert_eq!(0, site_aggregates_after_delete.communities); + assert_eq!(0, site_aggregates_after_delete.local_communities); Community::update( pool, @@ -201,7 +201,7 @@ mod tests { .await?; let site_aggregates_after_remove = read_local_site(pool).await?; - assert_eq!(0, site_aggregates_after_remove.communities); + assert_eq!(0, site_aggregates_after_remove.local_communities); Community::update( pool, @@ -214,7 +214,7 @@ mod tests { .await?; let site_aggregates_after_remove_delete = read_local_site(pool).await?; - assert_eq!(0, site_aggregates_after_remove_delete.communities); + assert_eq!(0, site_aggregates_after_remove_delete.local_communities); Community::delete(pool, inserted_community.id).await?; Person::delete(pool, inserted_person.id).await?; diff --git a/crates/db_schema/src/source/local_site.rs b/crates/db_schema/src/source/local_site.rs index b69f160cfc..1154741c20 100644 --- a/crates/db_schema/src/source/local_site.rs +++ b/crates/db_schema/src/source/local_site.rs @@ -15,6 +15,7 @@ use lemmy_db_schema_file::{ }, }; use serde::{Deserialize, Serialize}; +use serde_json::Value; use serde_with::skip_serializing_none; #[skip_serializing_none] @@ -78,10 +79,10 @@ pub struct LocalSite { pub default_post_time_range_seconds: Option, /// Block NSFW content being created pub nsfw_content_disallowed: bool, - pub users: i32, - pub posts: i32, - pub comments: i32, - pub communities: i32, + pub local_users: i32, + pub local_posts: i32, + pub local_comments: i32, + pub local_communities: i32, /// The number of users with any activity in the last day. pub users_active_day: i32, /// The number of users with any activity in the last week. @@ -113,6 +114,16 @@ pub struct LocalSite { /// This affects post and comment images, but not avatars and banners. pub image_allow_video_uploads: bool, pub image_upload_disabled: bool, + pub linked_instances: i32, + pub total_posts: i32, + pub total_comments: i32, + pub total_users: i32, + pub total_communities: i32, + pub user_retention_percent: i32, + pub ban_rate: i32, + pub accepted_signups_rate: i32, + pub failed_signups_rate: i32, + pub language_usage_percent: Value, } #[derive(Clone, derive_new::new)] diff --git a/crates/db_schema_file/src/schema.rs b/crates/db_schema_file/src/schema.rs index da93f2c24f..1369f1f525 100644 --- a/crates/db_schema_file/src/schema.rs +++ b/crates/db_schema_file/src/schema.rs @@ -407,10 +407,10 @@ diesel::table! { comment_downvotes -> FederationModeEnum, default_post_time_range_seconds -> Nullable, nsfw_content_disallowed -> Bool, - users -> Int4, - posts -> Int4, - comments -> Int4, - communities -> Int4, + local_users -> Int4, + local_posts -> Int4, + local_comments -> Int4, + local_communities -> Int4, users_active_day -> Int4, users_active_week -> Int4, users_active_month -> Int4, @@ -428,6 +428,16 @@ diesel::table! { image_max_upload_size -> Int4, image_allow_video_uploads -> Bool, image_upload_disabled -> Bool, + linked_instances -> Int4, + total_posts -> Int4, + total_comments -> Int4, + total_users -> Int4, + total_communities -> Int4, + user_retention_percent -> Int4, + ban_rate -> Int4, + accepted_signups_rate -> Int4, + failed_signups_rate -> Int4, + language_usage_percent -> Jsonb, } } diff --git a/crates/diesel_utils/replaceable_schema/triggers.sql b/crates/diesel_utils/replaceable_schema/triggers.sql index a6a89e8ae0..9f854e1ba8 100644 --- a/crates/diesel_utils/replaceable_schema/triggers.sql +++ b/crates/diesel_utils/replaceable_schema/triggers.sql @@ -169,7 +169,7 @@ WHERE UPDATE local_site AS a SET - comments = a.comments + diff.comments + local_comments = a.local_comments + diff.comments FROM ( SELECT coalesce(sum(count_diff), 0) AS comments @@ -223,7 +223,7 @@ WHERE UPDATE local_site AS a SET - posts = a.posts + diff.posts + local_posts = a.local_posts + diff.posts FROM ( SELECT coalesce(sum(count_diff), 0) AS posts @@ -242,7 +242,7 @@ BEGIN UPDATE local_site AS a SET - communities = a.communities + diff.communities + local_communities = a.local_communities + diff.communities FROM ( SELECT coalesce(sum(count_diff), 0) AS communities diff --git a/crates/routes/Cargo.toml b/crates/routes/Cargo.toml index e5240255a6..86577b69df 100644 --- a/crates/routes/Cargo.toml +++ b/crates/routes/Cargo.toml @@ -64,6 +64,7 @@ diesel-uplete.workspace = true lemmy_diesel_utils = { workspace = true } rosetta-i18n = { workspace = true } strum = { workspace = true } +serde_json = { workspace = true } ts-rs = { workspace = true, optional = true } [dev-dependencies] diff --git a/crates/routes/src/nodeinfo.rs b/crates/routes/src/nodeinfo.rs index 58098e9e0a..c46f92e477 100644 --- a/crates/routes/src/nodeinfo.rs +++ b/crates/routes/src/nodeinfo.rs @@ -59,12 +59,12 @@ async fn node_info(context: web::Data) -> Result) -> LemmyResult<()> { .await .inspect_err(|e| warn!("Failed to update local user count: {e}")) .ok(); + update_stats(&mut context.pool()) + .await + .inspect_err(|e| warn!("Failed to update stats: {e}")) + .ok(); overwrite_deleted_posts_and_comments(&mut context.pool()) .await .inspect_err(|e| warn!("Failed to overwrite deleted posts/comments: {e}")) @@ -354,6 +364,14 @@ struct CommunityAggregatesUpdateResult { community_id: i32, } +#[derive(Queryable, Debug)] +struct PostCountSelectResult { + #[diesel(sql_type = VarChar)] + lang_code: String, + #[diesel(sql_type = Integer)] + post_count: i64, +} + /// Re-calculate the site and community active counts for a given interval async fn active_counts(pool: &mut DbPool<'_>, interval: (&str, &str)) -> LemmyResult<()> { info!( @@ -510,11 +528,125 @@ async fn update_local_user_count(pool: &mut DbPool<'_>) -> LemmyResult<()> { .map(i32::try_from)??; update(local_site::table) - .set(local_site::users.eq(user_count)) + .set(local_site::local_users.eq(user_count)) + .execute(conn) + .await?; + + info!("Done."); + Ok(()) +} + +async fn update_stats(pool: &mut DbPool<'_>) -> LemmyResult<()> { + info!("Updating stats ..."); + + let conn = &mut get_conn(pool).await?; + let linked_instance_count = instance::table + // omit instance representing the local site + .left_join(site::table.left_join(local_site::table)) + .filter(local_site::id.is_null()) + .left_join(federation_blocklist::table) + .left_join(federation_allowlist::table) + .left_join(federation_queue_state::table) + .filter(federation_blocklist::instance_id.is_null()) + .select(count_star()) + .first::(conn) + .await + .map(i32::try_from)??; + + update(local_site::table) + .set(local_site::linked_instances.eq(linked_instance_count)) + .execute(conn) + .await?; + + let total_post_count = post::table + .select(count_star()) + .first::(conn) + .await + .map(i32::try_from)??; + + update(local_site::table) + .set(local_site::total_posts.eq(total_post_count)) + .execute(conn) + .await?; + + let total_comment_count = comment::table + .select(count_star()) + .first::(conn) + .await + .map(i32::try_from)??; + + update(local_site::table) + .set(local_site::total_comments.eq(total_comment_count)) + .execute(conn) + .await?; + + let total_user_count = person::table + .select(count_star()) + .first::(conn) + .await + .map(i32::try_from)??; + + update(local_site::table) + .set(local_site::total_users.eq(total_user_count)) + .execute(conn) + .await?; + + let total_community_count = community::table + .select(count_star()) + .first::(conn) + .await + .map(i32::try_from)??; + + update(local_site::table) + .set(local_site::total_communities.eq(total_community_count)) .execute(conn) .await?; + process_language_breakdown(conn).await?; + info!("Done."); + + Ok(()) +} +// Update db with percentage breakdown of local posts per language tag +async fn process_language_breakdown(conn: &mut AsyncPgConnection) -> LemmyResult<()> { + info!("Updating local language usage percentages ..."); + let local_post_count = local_site::table + .select(local_site::local_posts) + .get_result::(conn) + .await?; + + if local_post_count == 0 { + return Ok(()); + } + + let post_lang_breakdown = post::table + .inner_join(language::table.on(post::language_id.eq(language::id))) + .filter(post::local.eq(true)) + .group_by(language::code) + .select((language::code, count_star())) + .load::(conn) + .await?; + + let mut post_counts = Map::new(); + + for post_count in post_lang_breakdown { + post_counts.insert( + post_count.lang_code, + Value::Number( + serde_json::Number::from_f64( + (post_count.post_count as f64 * 10000.0 / f64::from(local_post_count)).round() / 100.0, + ) + .unwrap_or(serde_json::Number::from(0)), + ), + ); + } + + update(local_site::table) + .set(local_site::language_usage_percent.eq(Value::Object(post_counts))) + .execute(conn) + .await?; + Ok(()) } @@ -698,6 +830,7 @@ mod tests { use lemmy_db_schema::{ source::{ community::{Community, CommunityInsertForm}, + language::Language, person::{Person, PersonInsertForm}, post::{Post, PostActions, PostInsertForm, PostLikeForm}, }, @@ -787,4 +920,145 @@ mod tests { data.delete(pool).await?; Ok(()) } + + #[tokio::test] + #[serial] + async fn test_update_stats() -> LemmyResult<()> { + let context = LemmyContext::init_test_context().await; + let pool = &mut context.pool(); + // Setup local site + let data = TestData::create(pool).await?; + // insert some local data + let community = Community::create( + pool, + &CommunityInsertForm::new( + data.instance.id, + "name".to_owned(), + "title".to_owned(), + "pubkey".to_owned(), + ), + ) + .await?; + let person = Person::create( + pool, + &PersonInsertForm::new("felicity".to_owned(), "pubkey".to_owned(), data.instance.id), + ) + .await?; + let _post = Post::create( + pool, + &PostInsertForm::new("i am grrreat".to_owned(), person.id, community.id), + ) + .await?; + + // insert linked instances + let instance0 = Instance::read_or_create(pool, "example0.com").await?; + + // insert some federated data + let community = Community::create( + pool, + &CommunityInsertForm::new( + instance0.id, + "name".to_owned(), + "title".to_owned(), + "pubkey".to_owned(), + ), + ) + .await?; + let person = Person::create( + pool, + &PersonInsertForm::new("felicity".to_owned(), "pubkey".to_owned(), instance0.id), + ) + .await?; + let _post = Post::create( + pool, + &PostInsertForm::new("i am grrreat".to_owned(), person.id, community.id), + ) + .await?; + + let _instance1 = Instance::read_or_create(pool, "example1.com").await?; + + let local_site_before = SiteView::read_local(pool).await?.local_site; + assert_eq!(0, local_site_before.linked_instances); + assert_eq!(0, local_site_before.total_posts); + assert_eq!(0, local_site_before.total_comments); + assert_eq!(0, local_site_before.total_users); + assert_eq!(0, local_site_before.total_communities); + + // run the query + update_stats(pool).await?; + let local_site_after = SiteView::read_local(pool).await?.local_site; + + assert_eq!(2, local_site_after.linked_instances); + assert_eq!(2, local_site_after.total_posts); + assert_eq!(0, local_site_after.total_comments); + assert_eq!(4, local_site_after.total_users); + assert_eq!(2, local_site_after.total_communities); + + data.delete(pool).await?; + Instance::delete_all(pool).await?; + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_process_language_breakdown() -> LemmyResult<()> { + let context = LemmyContext::init_test_context().await; + let pool = &mut context.pool(); + + let data = TestData::create(pool).await?; + let community = Community::create( + pool, + &CommunityInsertForm::new( + data.instance.id, + "name".to_owned(), + "title".to_owned(), + "pubkey".to_owned(), + ), + ) + .await?; + let person = Person::create( + pool, + &PersonInsertForm::new("felicity".to_owned(), "pubkey".to_owned(), data.instance.id), + ) + .await?; + + let en_id = Language::read_id_from_code(pool, "en").await?; + let de_id = Language::read_id_from_code(pool, "de").await?; + + // Create 2 English posts and 1 German post (expect 67% and 33%) + for _ in 0..2 { + Post::create( + pool, + &PostInsertForm { + language_id: Some(en_id), + ..PostInsertForm::new("english post".to_owned(), person.id, community.id) + }, + ) + .await?; + } + Post::create( + pool, + &PostInsertForm { + language_id: Some(de_id), + ..PostInsertForm::new("german post".to_owned(), person.id, community.id) + }, + ) + .await?; + + let conn = &mut get_conn(pool).await?; + process_language_breakdown(conn).await?; + let local_site = SiteView::read_local(pool).await?.local_site; + + assert_eq!( + local_site.language_usage_percent["en"].as_f64(), + Some(66.67) + ); + assert_eq!( + local_site.language_usage_percent["de"].as_f64(), + Some(33.33) + ); + + data.delete(pool).await?; + Ok(()) + } } diff --git a/migrations/2026-03-08-173221-0000_additional-statistics/down.sql b/migrations/2026-03-08-173221-0000_additional-statistics/down.sql new file mode 100644 index 0000000000..d751940e86 --- /dev/null +++ b/migrations/2026-03-08-173221-0000_additional-statistics/down.sql @@ -0,0 +1,42 @@ +-- This file should undo anything in `up.sql` +ALTER TABLE local_site + DROP COLUMN linked_instances; + +ALTER TABLE local_site + DROP COLUMN total_posts; + +ALTER TABLE local_site + DROP COLUMN total_comments; + +ALTER TABLE local_site + DROP COLUMN total_users; + +ALTER TABLE local_site + DROP COLUMN total_communities; + +ALTER TABLE local_site + DROP COLUMN user_retention_percent; + +ALTER TABLE local_site + DROP COLUMN local_post_english_percent; + +ALTER TABLE local_site + DROP COLUMN ban_rate; + +ALTER TABLE local_site + DROP COLUMN accepted_signups_rate; + +ALTER TABLE local_site + DROP COLUMN failed_signups_rate; + +ALTER TABLE local_site + DROP COLUMN language_usage_percent; + +ALTER TABLE local_site RENAME local_posts TO posts; + +ALTER TABLE local_site RENAME local_comments TO comments; + +ALTER TABLE local_site RENAME local_users TO users; + +ALTER TABLE local_site RENAME local_communities TO communities; + diff --git a/migrations/2026-03-08-173221-0000_additional-statistics/up.sql b/migrations/2026-03-08-173221-0000_additional-statistics/up.sql new file mode 100644 index 0000000000..dea1afb5be --- /dev/null +++ b/migrations/2026-03-08-173221-0000_additional-statistics/up.sql @@ -0,0 +1,39 @@ +-- Your SQL goes here +ALTER TABLE local_site + ADD COLUMN linked_instances integer NOT NULL DEFAULT 0; + +ALTER TABLE local_site + ADD COLUMN total_posts integer NOT NULL DEFAULT 0; + +ALTER TABLE local_site + ADD COLUMN total_comments integer NOT NULL DEFAULT 0; + +ALTER TABLE local_site + ADD COLUMN total_users integer NOT NULL DEFAULT 0; + +ALTER TABLE local_site + ADD COLUMN total_communities integer NOT NULL DEFAULT 0; + +ALTER TABLE local_site + ADD COLUMN user_retention_percent integer NOT NULL DEFAULT 0; + +ALTER TABLE local_site + ADD COLUMN ban_rate integer NOT NULL DEFAULT 0; + +ALTER TABLE local_site + ADD COLUMN accepted_signups_rate integer NOT NULL DEFAULT 0; + +ALTER TABLE local_site + ADD COLUMN failed_signups_rate integer NOT NULL DEFAULT 0; + +ALTER TABLE local_site + ADD COLUMN language_usage_percent jsonb NOT NULL DEFAULT '{}'::jsonb; + +ALTER TABLE local_site RENAME posts TO local_posts; + +ALTER TABLE local_site RENAME comments TO local_comments; + +ALTER TABLE local_site RENAME users TO local_users; + +ALTER TABLE local_site RENAME communities TO local_communities; +