From 5f566a60ce6af3afa291036d0bfc1cd05b5a35b1 Mon Sep 17 00:00:00 2001 From: ety001 Date: Mon, 13 Jul 2026 00:50:43 +0800 Subject: [PATCH 1/6] perf: hardcode follow state filter and cache follower/following queries get_followers, get_followers_by_page, and get_following_by_page hit the DB on every call (only get_following had a 30s cache). The parameterized `state IN :state` tuple binding also prevented the planner from matching the v23 partial index idx_follows_follower_state_created_desc (WHERE state IN (1,3)), forcing a fallback to the full-ASC ix5a with a cross-value sort -- the 21.9s query in the 2026-07-12 outage. Hardcode `state IN (1,3)` / `state IN (2,3)` literals so the planner can use the partial index, add a 30s cache to the three uncached functions, and complete the INNER JOIN migration for get_following_by_page (missed in 3274329 and 67c6d5e where its three siblings were converted). --- hive/server/condenser_api/cursor.py | 100 ++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 22 deletions(-) diff --git a/hive/server/condenser_api/cursor.py b/hive/server/condenser_api/cursor.py index 7ef52256..a16ec855 100644 --- a/hive/server/condenser_api/cursor.py +++ b/hive/server/condenser_api/cursor.py @@ -55,7 +55,13 @@ async def get_followers(db, account: str, start: str, follow_type: str, limit: i """Get a list of accounts following a given account.""" account_id = await _get_account_id(db, account) start_id = await _get_account_id(db, start) if start else None - state = (2,3) if follow_type == 'ignore' else (1,3) + # Hardcode state IN (...) so the planner can match the partial index + # idx_follows_following_state_created_desc (WHERE state IN (1,3)). + # A parameterized `state IN :state` tuple cannot be proven to satisfy the + # partial-index predicate at plan time. + state_clause = ("AND hf.state IN (2, 3)" + if follow_type == 'ignore' + else "AND hf.state IN (1, 3)") seek = '' if start_id: @@ -71,19 +77,34 @@ async def get_followers(db, account: str, start: str, follow_type: str, limit: i FROM hive_follows hf INNER JOIN hive_accounts ha ON hf.follower = ha.id WHERE hf.following = :account_id - AND hf.state IN :state %s + %s %s ORDER BY hf.created_at DESC LIMIT :limit - """ % seek + """ % (state_clause, seek) + + # Generate cache key with all parameters that affect the result + cache_key_parts = [ + 'get_followers', + str(account_id), + follow_type or 'blog', + str(start_id) if start_id else '', + str(limit) + ] + cache_key = '_'.join(cache_key_parts) return await db.query_all(sql, account_id=account_id, start_id=start_id, - state=state, limit=limit) + limit=limit, + cache_key=cache_key, cache_ttl=30) async def get_followers_by_page(db, account: str, page: int, page_size: int, follow_type: str): """Get a list of accounts following a given account.""" account_id = await _get_account_id(db, account) - state = (2,3) if follow_type == 'ignore' else (1,3) + # Hardcode state IN (...) so the planner can match the partial index + # idx_follows_following_state_created_desc (WHERE state IN (1,3)). + state_clause = ("AND hf.state IN (2, 3)" + if follow_type == 'ignore' + else "AND hf.state IN (1, 3)") # Optimized: Use INNER JOIN instead of LEFT JOIN for better performance # This assumes data integrity (all follower IDs exist in hive_accounts) @@ -92,19 +113,34 @@ async def get_followers_by_page(db, account: str, page: int, page_size: int, fol FROM hive_follows hf INNER JOIN hive_accounts ha ON hf.follower = ha.id WHERE hf.following = :account_id - AND hf.state IN :state + %s ORDER BY hf.created_at DESC LIMIT :limit OFFSET :offset - """ + """ % state_clause + + # Generate cache key with all parameters that affect the result + cache_key_parts = [ + 'get_followers_by_page', + str(account_id), + follow_type or 'blog', + str(page), + str(page_size) + ] + cache_key = '_'.join(cache_key_parts) return await db.query_all(sql, account_id=account_id, - state=state, limit=page_size, offset=page*page_size) + limit=page_size, offset=page*page_size, + cache_key=cache_key, cache_ttl=30) async def get_following(db, account: str, start: str, follow_type: str, limit: int): """Get a list of accounts followed by a given account.""" account_id = await _get_account_id(db, account) start_id = await _get_account_id(db, start) if start else None - state = (2, 3) if follow_type == 'ignore' else (1, 3) + # Hardcode state IN (...) so the planner can match the partial index + # idx_follows_follower_state_created_desc (WHERE state IN (1,3)). + state_clause = ("AND hf.state IN (2, 3)" + if follow_type == 'ignore' + else "AND hf.state IN (1, 3)") seek = '' if start_id: @@ -120,10 +156,10 @@ async def get_following(db, account: str, start: str, follow_type: str, limit: i FROM hive_follows hf INNER JOIN hive_accounts ha ON hf.following = ha.id WHERE hf.follower = :account_id - AND hf.state IN :state %s + %s %s ORDER BY hf.created_at DESC LIMIT :limit - """ % seek + """ % (state_clause, seek) # Generate cache key with all parameters that affect the result cache_key_parts = [ @@ -136,26 +172,46 @@ async def get_following(db, account: str, start: str, follow_type: str, limit: i cache_key = '_'.join(cache_key_parts) return await db.query_all(sql, account_id=account_id, start_id=start_id, - state=state, limit=limit, + limit=limit, cache_key=cache_key, cache_ttl=30) async def get_following_by_page(db, account: str, page: int, page_size: int, follow_type: str): """Get a list of accounts followed by a given account.""" account_id = await _get_account_id(db, account) - state = (2, 3) if follow_type == 'ignore' else (1, 3) - + # Hardcode state IN (...) so the planner can match the partial index + # idx_follows_follower_state_created_desc (WHERE state IN (1,3)). + state_clause = ("AND hf.state IN (2, 3)" + if follow_type == 'ignore' + else "AND hf.state IN (1, 3)") + + # Optimized: Use INNER JOIN instead of LEFT JOIN for better performance. + # This aligns with the other three follow* functions, which were migrated + # to INNER JOIN (3274329, 67c6d5e); this function was missed at the time. + # Assumes data integrity (all following IDs exist in hive_accounts). sql = """ - SELECT name,reputation,state FROM hive_follows hf - LEFT JOIN hive_accounts ON hf.following = id - WHERE hf.follower = :account_id - AND state IN :state - ORDER BY hf.created_at DESC - LIMIT :limit OFFSET :offset - """ + SELECT ha.name, ha.reputation, hf.state + FROM hive_follows hf + INNER JOIN hive_accounts ha ON hf.following = ha.id + WHERE hf.follower = :account_id + %s + ORDER BY hf.created_at DESC + LIMIT :limit OFFSET :offset + """ % state_clause + + # Generate cache key with all parameters that affect the result + cache_key_parts = [ + 'get_following_by_page', + str(account_id), + follow_type or 'blog', + str(page), + str(page_size) + ] + cache_key = '_'.join(cache_key_parts) return await db.query_all(sql, account_id=account_id, - state=state, limit=page_size, offset=page*page_size) + limit=page_size, offset=page*page_size, + cache_key=cache_key, cache_ttl=30) async def get_follow_counts(db, account: str): From eed7f5ea5e5e102bb6f38a53e08d69790167429a Mon Sep 17 00:00:00 2001 From: ety001 Date: Mon, 13 Jul 2026 00:54:19 +0800 Subject: [PATCH 2/6] perf: cache pids_by_category and pids_by_query results (60s TTL) Both tag-filtering queries ran uncached on every call. pids_by_blog in the same file already uses query_col with cache_key/cache_ttl=30; apply the same pattern with a 60s TTL. Hot tags (e.g. dmania) match tens of thousands of post_ids via the `IN (subquery)` and re-running it per request was the 12.9s query in the 2026-07-12 outage. --- hive/server/bridge_api/cursor.py | 13 ++++++++++++- hive/server/condenser_api/cursor.py | 14 +++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/hive/server/bridge_api/cursor.py b/hive/server/bridge_api/cursor.py index 9e2c294e..d2f74f2e 100644 --- a/hive/server/bridge_api/cursor.py +++ b/hive/server/bridge_api/cursor.py @@ -243,7 +243,18 @@ async def pids_by_category(db, tag, sort, last_id, limit): ORDER BY %s DESC, post_id LIMIT :limit """ % (table, ' AND '.join(where), field)) - return await db.query_col(sql, tag=tag, last_id=last_id, limit=limit) + # Generate cache key with all parameters that affect the result + cache_key_parts = [ + 'pids_by_category', + str(sort), + str(tag), + str(last_id), + str(limit) + ] + cache_key = '_'.join(cache_key_parts) + + return await db.query_col(sql, tag=tag, last_id=last_id, limit=limit, + cache_key=cache_key, cache_ttl=60) async def _subscribed(db, account_id): diff --git a/hive/server/condenser_api/cursor.py b/hive/server/condenser_api/cursor.py index a16ec855..c7eaac0c 100644 --- a/hive/server/condenser_api/cursor.py +++ b/hive/server/condenser_api/cursor.py @@ -304,7 +304,19 @@ async def pids_by_query(db, sort, start_author, start_permlink, limit, tag): sql = ("SELECT post_id FROM %s WHERE %s ORDER BY %s DESC LIMIT :limit" % (table, ' AND '.join(where), field)) - return await db.query_col(sql, tag=tag, start_id=start_id, limit=limit) + # Generate cache key with all parameters that affect the result + cache_key_parts = [ + 'pids_by_query', + str(sort), + str(tag), + str(start_author), + str(start_permlink), + str(limit) + ] + cache_key = '_'.join(cache_key_parts) + + return await db.query_col(sql, tag=tag, start_id=start_id, limit=limit, + cache_key=cache_key, cache_ttl=60) async def pids_by_blog(db, account: str, start_author: str = '', From d35e4a48246d70252bd732a521441950955413d5 Mon Sep 17 00:00:00 2001 From: ety001 Date: Mon, 13 Jul 2026 00:54:35 +0800 Subject: [PATCH 3/6] perf: add symmetric follows index and drop duplicates (v28->v29) get_followers queries `WHERE following = :id ORDER BY created_at DESC` but the only partial index idx_follows_follower_state_created_desc is follower-led (serves get_following). Add a following-led counterpart idx_follows_following_state_created_desc so get_followers no longer falls back to the full-ASC ix5a. Also drop hive_follows_5a/5b: legacy v9 duplicates of ix5a/ix5b (identical column sets), pure write-amplification burden, never managed by _disableable_indexes. Bumps DB_VERSION 28->29. --- hive/db/db_state.py | 21 +++++++++++++++++++++ hive/db/schema.py | 7 +++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/hive/db/db_state.py b/hive/db/db_state.py index b99bdebe..9cd5741a 100644 --- a/hive/db/db_state.py +++ b/hive/db/db_state.py @@ -504,6 +504,27 @@ def _check_migrations(cls): log.info("[HIVE] hive_notifs_dst_post_type_idx index created") cls._set_ver(28) + if cls._ver == 28: + # Performance: symmetric partial index for get_followers + drop dup indexes + # get_followers queries `WHERE following = :id ... ORDER BY created_at DESC` + # but only had idx_follows_follower_state_created_desc (follower-led, serves + # get_following). Add a following-led counterpart so the planner can satisfy + # get_followers without falling back to the full-ASC ix5a + cross-value sort. + # Also drop hive_follows_5a/5b: legacy v9 duplicates of ix5a/ix5b (identical + # columns), pure write-amplification burden, never managed by _disableable_indexes. + log.info("[HIVE] Creating idx_follows_following_state_created_desc index...") + cls.db().query(""" + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_follows_following_state_created_desc + ON hive_follows (following, state, created_at DESC, follower) + WHERE state IN (1,3) + """) + log.info("[HIVE] Dropping duplicate hive_follows_5a/5b indexes...") + cls.db().query("DROP INDEX CONCURRENTLY IF EXISTS hive_follows_5a") + cls.db().query("DROP INDEX CONCURRENTLY IF EXISTS hive_follows_5b") + cls.db().query("ANALYZE hive_follows") + log.info("[HIVE] hive_follows index optimization complete") + cls._set_ver(29) + reset_autovac(cls.db()) log.info("[HIVE] db version: %d", cls._ver) diff --git a/hive/db/schema.py b/hive/db/schema.py index 09210a89..ab9a08fd 100644 --- a/hive/db/schema.py +++ b/hive/db/schema.py @@ -10,7 +10,7 @@ #pylint: disable=line-too-long, too-many-lines, bad-whitespace -DB_VERSION = 28 +DB_VERSION = 29 def build_metadata(): """Build schema def with SqlAlchemy""" @@ -112,7 +112,10 @@ def build_metadata(): sa.Index('hive_follows_ix5a', 'following', 'state', 'created_at', 'follower'), sa.Index('hive_follows_ix5b', 'follower', 'state', 'created_at', 'following'), # Note: idx_follows_follower_following_state and idx_follows_follower_state_created_desc - # are created via migration (v23) because they use DESC and WHERE clauses + # are created via migration (v23) because they use DESC and WHERE clauses. + # v29: idx_follows_following_state_created_desc added via migration (symmetric + # to idx_follows_follower_state_created_desc, serving get_followers). + # v29: hive_follows_5a/5b (v9 migration duplicates of ix5a/ix5b) dropped. ) sa.Table( From 56a60e7bdfefd94e6781973a3189c87d05335184 Mon Sep 17 00:00:00 2001 From: ety001 Date: Mon, 13 Jul 2026 00:54:59 +0800 Subject: [PATCH 4/6] perf: reduce fetch_ranks frequency from hourly to every 6h Accounts.fetch_ranks runs `SELECT id FROM hive_accounts ORDER BY vote_weight DESC` (full-table sort of million-scale rows, ~12s) on the indexer's sync connection. Although separate from the API pool, it competes for RDS IOPS (which hit the 3000 cap during the 2026-07-12 outage). Rank is an approximate score for notification buckets; a 6h stale window is acceptable. --- hive/indexer/sync.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hive/indexer/sync.py b/hive/indexer/sync.py index 9595f830..43fad526 100644 --- a/hive/indexer/sync.py +++ b/hive/indexer/sync.py @@ -219,8 +219,13 @@ def listen(self): if num % 1200 == 0: #1hr log.warning("head block %d @ %s", num, block['timestamp']) log.info("[LIVE] hourly stats") - Accounts.fetch_ranks() #Community.recalc_pending_payouts() + if num % 7200 == 0: #6hr + # Rank is an approximate score used for notification buckets; a 6h + # stale window is acceptable. Run less often than the previous 1h + # cadence to avoid the full-table ORDER BY vote_weight DESC (~12s, + # million-scale) competing for RDS IOPS with API queries. + Accounts.fetch_ranks() if num % 200 == 0: #10min Community.recalc_pending_payouts() if num % 100 == 0: #5min From 3f62c9525dcdc8143cdc2fa63b5f2ece704368c4 Mon Sep 17 00:00:00 2001 From: ety001 Date: Mon, 13 Jul 2026 09:41:13 +0800 Subject: [PATCH 5/6] docs: clarify follow state-filter comment and harden start_id cache key Address PR #373 audit feedback: - The state-filter comment claimed hardcoding "matches the partial index" for both branches, but the ignore branch (state IN (2,3)) cannot match any WHERE state IN (1,3) partial index. Rewrite the 4 comments to state precisely: normal branch matches the partial index; ignore branch falls back to ix5a/ix5b (no regression) but still benefits from a hardcoded literal via better cardinality estimates vs a tuple bind. - Cache key used `str(start_id) if start_id else ''`, which collapses both None and a (theoretical) id 0 to ''. Switch to an explicit 'none' sentinel for None so it can never collide with any integer id. --- hive/server/condenser_api/cursor.py | 35 +++++++++++++++++++---------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/hive/server/condenser_api/cursor.py b/hive/server/condenser_api/cursor.py index c7eaac0c..39258352 100644 --- a/hive/server/condenser_api/cursor.py +++ b/hive/server/condenser_api/cursor.py @@ -55,10 +55,12 @@ async def get_followers(db, account: str, start: str, follow_type: str, limit: i """Get a list of accounts following a given account.""" account_id = await _get_account_id(db, account) start_id = await _get_account_id(db, start) if start else None - # Hardcode state IN (...) so the planner can match the partial index - # idx_follows_following_state_created_desc (WHERE state IN (1,3)). - # A parameterized `state IN :state` tuple cannot be proven to satisfy the - # partial-index predicate at plan time. + # Hardcode the state filter (instead of a parameterized `state IN :state` + # tuple) so the normal branch (state IN (1,3)) can match the partial index + # idx_follows_following_state_created_desc. The ignore branch (state IN + # (2,3)) cannot match that partial index (predicate is state IN (1,3)) and + # falls back to ix5a; hardcoding still beats a tuple bind via better + # cardinality estimates. state_clause = ("AND hf.state IN (2, 3)" if follow_type == 'ignore' else "AND hf.state IN (1, 3)") @@ -87,7 +89,7 @@ async def get_followers(db, account: str, start: str, follow_type: str, limit: i 'get_followers', str(account_id), follow_type or 'blog', - str(start_id) if start_id else '', + 'none' if start_id is None else str(start_id), str(limit) ] cache_key = '_'.join(cache_key_parts) @@ -100,8 +102,11 @@ async def get_followers(db, account: str, start: str, follow_type: str, limit: i async def get_followers_by_page(db, account: str, page: int, page_size: int, follow_type: str): """Get a list of accounts following a given account.""" account_id = await _get_account_id(db, account) - # Hardcode state IN (...) so the planner can match the partial index - # idx_follows_following_state_created_desc (WHERE state IN (1,3)). + # Hardcode the state filter so the normal branch (state IN (1,3)) can match + # the partial index idx_follows_following_state_created_desc. The ignore + # branch (state IN (2,3)) cannot match that partial index (predicate is + # state IN (1,3)) and falls back to ix5a; hardcoding still beats a tuple + # bind via better cardinality estimates. state_clause = ("AND hf.state IN (2, 3)" if follow_type == 'ignore' else "AND hf.state IN (1, 3)") @@ -136,8 +141,11 @@ async def get_following(db, account: str, start: str, follow_type: str, limit: i """Get a list of accounts followed by a given account.""" account_id = await _get_account_id(db, account) start_id = await _get_account_id(db, start) if start else None - # Hardcode state IN (...) so the planner can match the partial index - # idx_follows_follower_state_created_desc (WHERE state IN (1,3)). + # Hardcode the state filter so the normal branch (state IN (1,3)) can match + # the partial index idx_follows_follower_state_created_desc. The ignore + # branch (state IN (2,3)) cannot match that partial index (predicate is + # state IN (1,3)) and falls back to ix5b; hardcoding still beats a tuple + # bind via better cardinality estimates. state_clause = ("AND hf.state IN (2, 3)" if follow_type == 'ignore' else "AND hf.state IN (1, 3)") @@ -166,7 +174,7 @@ async def get_following(db, account: str, start: str, follow_type: str, limit: i 'get_following', str(account_id), follow_type or 'blog', - str(start_id) if start_id else '', + 'none' if start_id is None else str(start_id), str(limit) ] cache_key = '_'.join(cache_key_parts) @@ -179,8 +187,11 @@ async def get_following(db, account: str, start: str, follow_type: str, limit: i async def get_following_by_page(db, account: str, page: int, page_size: int, follow_type: str): """Get a list of accounts followed by a given account.""" account_id = await _get_account_id(db, account) - # Hardcode state IN (...) so the planner can match the partial index - # idx_follows_follower_state_created_desc (WHERE state IN (1,3)). + # Hardcode the state filter so the normal branch (state IN (1,3)) can match + # the partial index idx_follows_follower_state_created_desc. The ignore + # branch (state IN (2,3)) cannot match that partial index (predicate is + # state IN (1,3)) and falls back to ix5b; hardcoding still beats a tuple + # bind via better cardinality estimates. state_clause = ("AND hf.state IN (2, 3)" if follow_type == 'ignore' else "AND hf.state IN (1, 3)") From 0410a42426a94deb46542b6aab9b0b2a5db594ed Mon Sep 17 00:00:00 2001 From: ety001 Date: Mon, 13 Jul 2026 13:11:30 +0800 Subject: [PATCH 6/6] perf: extend ignore (muted) follow cache TTL to 300s The ignore branch (state IN (2,3)) cannot match any WHERE state IN (1,3) partial index and falls back to the full-ASC ix5a/ix5b scan (3.5-9s per query in production, observed via Scalyr). Although already cached at 30s TTL, the cache churns fast: each account/start_id/limit combination is a distinct key, and 30s expiry forces frequent cold-cache DB hits on this slow path. Muted (ignore) relationships change rarely compared to normal blog follows, so a 300s (5min) TTL is acceptable -- at worst a 5min delay reflecting an un-mute. Normal follows stay at 30s. Applies uniformly to get_followers, get_followers_by_page, get_following, get_following_by_page. --- hive/server/condenser_api/cursor.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/hive/server/condenser_api/cursor.py b/hive/server/condenser_api/cursor.py index 39258352..3a749e01 100644 --- a/hive/server/condenser_api/cursor.py +++ b/hive/server/condenser_api/cursor.py @@ -93,10 +93,14 @@ async def get_followers(db, account: str, start: str, follow_type: str, limit: i str(limit) ] cache_key = '_'.join(cache_key_parts) + # ignore (muted) relationships change rarely; use a longer TTL to avoid + # cache churn on the slow ix5a fallback path (no partial index for state + # IN (2,3)). Normal blog follows stay at 30s. + cache_ttl = 300 if follow_type == 'ignore' else 30 return await db.query_all(sql, account_id=account_id, start_id=start_id, limit=limit, - cache_key=cache_key, cache_ttl=30) + cache_key=cache_key, cache_ttl=cache_ttl) async def get_followers_by_page(db, account: str, page: int, page_size: int, follow_type: str): @@ -132,10 +136,11 @@ async def get_followers_by_page(db, account: str, page: int, page_size: int, fol str(page_size) ] cache_key = '_'.join(cache_key_parts) + cache_ttl = 300 if follow_type == 'ignore' else 30 return await db.query_all(sql, account_id=account_id, limit=page_size, offset=page*page_size, - cache_key=cache_key, cache_ttl=30) + cache_key=cache_key, cache_ttl=cache_ttl) async def get_following(db, account: str, start: str, follow_type: str, limit: int): """Get a list of accounts followed by a given account.""" @@ -178,10 +183,11 @@ async def get_following(db, account: str, start: str, follow_type: str, limit: i str(limit) ] cache_key = '_'.join(cache_key_parts) + cache_ttl = 300 if follow_type == 'ignore' else 30 return await db.query_all(sql, account_id=account_id, start_id=start_id, limit=limit, - cache_key=cache_key, cache_ttl=30) + cache_key=cache_key, cache_ttl=cache_ttl) async def get_following_by_page(db, account: str, page: int, page_size: int, follow_type: str): @@ -219,10 +225,11 @@ async def get_following_by_page(db, account: str, page: int, page_size: int, fol str(page_size) ] cache_key = '_'.join(cache_key_parts) + cache_ttl = 300 if follow_type == 'ignore' else 30 return await db.query_all(sql, account_id=account_id, limit=page_size, offset=page*page_size, - cache_key=cache_key, cache_ttl=30) + cache_key=cache_key, cache_ttl=cache_ttl) async def get_follow_counts(db, account: str):