diff --git a/hive/db/db_state.py b/hive/db/db_state.py index b99bdebe..9cd5741a 100644 --- a/hive/db/db_state.py +++ b/hive/db/db_state.py @@ -504,6 +504,27 @@ def _check_migrations(cls): log.info("[HIVE] hive_notifs_dst_post_type_idx index created") cls._set_ver(28) + if cls._ver == 28: + # Performance: symmetric partial index for get_followers + drop dup indexes + # get_followers queries `WHERE following = :id ... ORDER BY created_at DESC` + # but only had idx_follows_follower_state_created_desc (follower-led, serves + # get_following). Add a following-led counterpart so the planner can satisfy + # get_followers without falling back to the full-ASC ix5a + cross-value sort. + # Also drop hive_follows_5a/5b: legacy v9 duplicates of ix5a/ix5b (identical + # columns), pure write-amplification burden, never managed by _disableable_indexes. + log.info("[HIVE] Creating idx_follows_following_state_created_desc index...") + cls.db().query(""" + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_follows_following_state_created_desc + ON hive_follows (following, state, created_at DESC, follower) + WHERE state IN (1,3) + """) + log.info("[HIVE] Dropping duplicate hive_follows_5a/5b indexes...") + cls.db().query("DROP INDEX CONCURRENTLY IF EXISTS hive_follows_5a") + cls.db().query("DROP INDEX CONCURRENTLY IF EXISTS hive_follows_5b") + cls.db().query("ANALYZE hive_follows") + log.info("[HIVE] hive_follows index optimization complete") + cls._set_ver(29) + reset_autovac(cls.db()) log.info("[HIVE] db version: %d", cls._ver) diff --git a/hive/db/schema.py b/hive/db/schema.py index 09210a89..ab9a08fd 100644 --- a/hive/db/schema.py +++ b/hive/db/schema.py @@ -10,7 +10,7 @@ #pylint: disable=line-too-long, too-many-lines, bad-whitespace -DB_VERSION = 28 +DB_VERSION = 29 def build_metadata(): """Build schema def with SqlAlchemy""" @@ -112,7 +112,10 @@ def build_metadata(): sa.Index('hive_follows_ix5a', 'following', 'state', 'created_at', 'follower'), sa.Index('hive_follows_ix5b', 'follower', 'state', 'created_at', 'following'), # Note: idx_follows_follower_following_state and idx_follows_follower_state_created_desc - # are created via migration (v23) because they use DESC and WHERE clauses + # are created via migration (v23) because they use DESC and WHERE clauses. + # v29: idx_follows_following_state_created_desc added via migration (symmetric + # to idx_follows_follower_state_created_desc, serving get_followers). + # v29: hive_follows_5a/5b (v9 migration duplicates of ix5a/ix5b) dropped. ) sa.Table( diff --git a/hive/indexer/sync.py b/hive/indexer/sync.py index 9595f830..43fad526 100644 --- a/hive/indexer/sync.py +++ b/hive/indexer/sync.py @@ -219,8 +219,13 @@ def listen(self): if num % 1200 == 0: #1hr log.warning("head block %d @ %s", num, block['timestamp']) log.info("[LIVE] hourly stats") - Accounts.fetch_ranks() #Community.recalc_pending_payouts() + if num % 7200 == 0: #6hr + # Rank is an approximate score used for notification buckets; a 6h + # stale window is acceptable. Run less often than the previous 1h + # cadence to avoid the full-table ORDER BY vote_weight DESC (~12s, + # million-scale) competing for RDS IOPS with API queries. + Accounts.fetch_ranks() if num % 200 == 0: #10min Community.recalc_pending_payouts() if num % 100 == 0: #5min diff --git a/hive/server/bridge_api/cursor.py b/hive/server/bridge_api/cursor.py index 9e2c294e..d2f74f2e 100644 --- a/hive/server/bridge_api/cursor.py +++ b/hive/server/bridge_api/cursor.py @@ -243,7 +243,18 @@ async def pids_by_category(db, tag, sort, last_id, limit): ORDER BY %s DESC, post_id LIMIT :limit """ % (table, ' AND '.join(where), field)) - return await db.query_col(sql, tag=tag, last_id=last_id, limit=limit) + # Generate cache key with all parameters that affect the result + cache_key_parts = [ + 'pids_by_category', + str(sort), + str(tag), + str(last_id), + str(limit) + ] + cache_key = '_'.join(cache_key_parts) + + return await db.query_col(sql, tag=tag, last_id=last_id, limit=limit, + cache_key=cache_key, cache_ttl=60) async def _subscribed(db, account_id): diff --git a/hive/server/condenser_api/cursor.py b/hive/server/condenser_api/cursor.py index 7ef52256..3a749e01 100644 --- a/hive/server/condenser_api/cursor.py +++ b/hive/server/condenser_api/cursor.py @@ -55,7 +55,15 @@ async def get_followers(db, account: str, start: str, follow_type: str, limit: i """Get a list of accounts following a given account.""" account_id = await _get_account_id(db, account) start_id = await _get_account_id(db, start) if start else None - state = (2,3) if follow_type == 'ignore' else (1,3) + # Hardcode the state filter (instead of a parameterized `state IN :state` + # tuple) so the normal branch (state IN (1,3)) can match the partial index + # idx_follows_following_state_created_desc. The ignore branch (state IN + # (2,3)) cannot match that partial index (predicate is state IN (1,3)) and + # falls back to ix5a; hardcoding still beats a tuple bind via better + # cardinality estimates. + state_clause = ("AND hf.state IN (2, 3)" + if follow_type == 'ignore' + else "AND hf.state IN (1, 3)") seek = '' if start_id: @@ -71,19 +79,41 @@ async def get_followers(db, account: str, start: str, follow_type: str, limit: i FROM hive_follows hf INNER JOIN hive_accounts ha ON hf.follower = ha.id WHERE hf.following = :account_id - AND hf.state IN :state %s + %s %s ORDER BY hf.created_at DESC LIMIT :limit - """ % seek + """ % (state_clause, seek) + + # Generate cache key with all parameters that affect the result + cache_key_parts = [ + 'get_followers', + str(account_id), + follow_type or 'blog', + 'none' if start_id is None else str(start_id), + str(limit) + ] + cache_key = '_'.join(cache_key_parts) + # ignore (muted) relationships change rarely; use a longer TTL to avoid + # cache churn on the slow ix5a fallback path (no partial index for state + # IN (2,3)). Normal blog follows stay at 30s. + cache_ttl = 300 if follow_type == 'ignore' else 30 return await db.query_all(sql, account_id=account_id, start_id=start_id, - state=state, limit=limit) + limit=limit, + cache_key=cache_key, cache_ttl=cache_ttl) async def get_followers_by_page(db, account: str, page: int, page_size: int, follow_type: str): """Get a list of accounts following a given account.""" account_id = await _get_account_id(db, account) - state = (2,3) if follow_type == 'ignore' else (1,3) + # Hardcode the state filter so the normal branch (state IN (1,3)) can match + # the partial index idx_follows_following_state_created_desc. The ignore + # branch (state IN (2,3)) cannot match that partial index (predicate is + # state IN (1,3)) and falls back to ix5a; hardcoding still beats a tuple + # bind via better cardinality estimates. + state_clause = ("AND hf.state IN (2, 3)" + if follow_type == 'ignore' + else "AND hf.state IN (1, 3)") # Optimized: Use INNER JOIN instead of LEFT JOIN for better performance # This assumes data integrity (all follower IDs exist in hive_accounts) @@ -92,19 +122,38 @@ async def get_followers_by_page(db, account: str, page: int, page_size: int, fol FROM hive_follows hf INNER JOIN hive_accounts ha ON hf.follower = ha.id WHERE hf.following = :account_id - AND hf.state IN :state + %s ORDER BY hf.created_at DESC LIMIT :limit OFFSET :offset - """ + """ % state_clause + + # Generate cache key with all parameters that affect the result + cache_key_parts = [ + 'get_followers_by_page', + str(account_id), + follow_type or 'blog', + str(page), + str(page_size) + ] + cache_key = '_'.join(cache_key_parts) + cache_ttl = 300 if follow_type == 'ignore' else 30 return await db.query_all(sql, account_id=account_id, - state=state, limit=page_size, offset=page*page_size) + limit=page_size, offset=page*page_size, + cache_key=cache_key, cache_ttl=cache_ttl) async def get_following(db, account: str, start: str, follow_type: str, limit: int): """Get a list of accounts followed by a given account.""" account_id = await _get_account_id(db, account) start_id = await _get_account_id(db, start) if start else None - state = (2, 3) if follow_type == 'ignore' else (1, 3) + # Hardcode the state filter so the normal branch (state IN (1,3)) can match + # the partial index idx_follows_follower_state_created_desc. The ignore + # branch (state IN (2,3)) cannot match that partial index (predicate is + # state IN (1,3)) and falls back to ix5b; hardcoding still beats a tuple + # bind via better cardinality estimates. + state_clause = ("AND hf.state IN (2, 3)" + if follow_type == 'ignore' + else "AND hf.state IN (1, 3)") seek = '' if start_id: @@ -120,42 +169,67 @@ async def get_following(db, account: str, start: str, follow_type: str, limit: i FROM hive_follows hf INNER JOIN hive_accounts ha ON hf.following = ha.id WHERE hf.follower = :account_id - AND hf.state IN :state %s + %s %s ORDER BY hf.created_at DESC LIMIT :limit - """ % seek + """ % (state_clause, seek) # Generate cache key with all parameters that affect the result cache_key_parts = [ 'get_following', str(account_id), follow_type or 'blog', - str(start_id) if start_id else '', + 'none' if start_id is None else str(start_id), str(limit) ] cache_key = '_'.join(cache_key_parts) + cache_ttl = 300 if follow_type == 'ignore' else 30 return await db.query_all(sql, account_id=account_id, start_id=start_id, - state=state, limit=limit, - cache_key=cache_key, cache_ttl=30) + limit=limit, + cache_key=cache_key, cache_ttl=cache_ttl) async def get_following_by_page(db, account: str, page: int, page_size: int, follow_type: str): """Get a list of accounts followed by a given account.""" account_id = await _get_account_id(db, account) - state = (2, 3) if follow_type == 'ignore' else (1, 3) - + # Hardcode the state filter so the normal branch (state IN (1,3)) can match + # the partial index idx_follows_follower_state_created_desc. The ignore + # branch (state IN (2,3)) cannot match that partial index (predicate is + # state IN (1,3)) and falls back to ix5b; hardcoding still beats a tuple + # bind via better cardinality estimates. + state_clause = ("AND hf.state IN (2, 3)" + if follow_type == 'ignore' + else "AND hf.state IN (1, 3)") + + # Optimized: Use INNER JOIN instead of LEFT JOIN for better performance. + # This aligns with the other three follow* functions, which were migrated + # to INNER JOIN (3274329, 67c6d5e); this function was missed at the time. + # Assumes data integrity (all following IDs exist in hive_accounts). sql = """ - SELECT name,reputation,state FROM hive_follows hf - LEFT JOIN hive_accounts ON hf.following = id - WHERE hf.follower = :account_id - AND state IN :state - ORDER BY hf.created_at DESC - LIMIT :limit OFFSET :offset - """ + SELECT ha.name, ha.reputation, hf.state + FROM hive_follows hf + INNER JOIN hive_accounts ha ON hf.following = ha.id + WHERE hf.follower = :account_id + %s + ORDER BY hf.created_at DESC + LIMIT :limit OFFSET :offset + """ % state_clause + + # Generate cache key with all parameters that affect the result + cache_key_parts = [ + 'get_following_by_page', + str(account_id), + follow_type or 'blog', + str(page), + str(page_size) + ] + cache_key = '_'.join(cache_key_parts) + cache_ttl = 300 if follow_type == 'ignore' else 30 return await db.query_all(sql, account_id=account_id, - state=state, limit=page_size, offset=page*page_size) + limit=page_size, offset=page*page_size, + cache_key=cache_key, cache_ttl=cache_ttl) async def get_follow_counts(db, account: str): @@ -248,7 +322,19 @@ async def pids_by_query(db, sort, start_author, start_permlink, limit, tag): sql = ("SELECT post_id FROM %s WHERE %s ORDER BY %s DESC LIMIT :limit" % (table, ' AND '.join(where), field)) - return await db.query_col(sql, tag=tag, start_id=start_id, limit=limit) + # Generate cache key with all parameters that affect the result + cache_key_parts = [ + 'pids_by_query', + str(sort), + str(tag), + str(start_author), + str(start_permlink), + str(limit) + ] + cache_key = '_'.join(cache_key_parts) + + return await db.query_col(sql, tag=tag, start_id=start_id, limit=limit, + cache_key=cache_key, cache_ttl=60) async def pids_by_blog(db, account: str, start_author: str = '',