From e71f50b23fcf0e9e87f947dcdae7a5db94e8c60b Mon Sep 17 00:00:00 2001 From: YimingZang Date: Thu, 6 Aug 2026 09:43:39 -0700 Subject: [PATCH 01/11] Implement new Giga GarbageCollector interface --- sei-db/config/giga_config.go | 2 +- sei-db/config/toml_test.go | 1 + sei-db/ledger_db/block/block_db_test.go | 2 +- .../block/littblock/litt_block_config.go | 43 ++- .../block/littblock/litt_block_db.go | 14 +- .../block/littblock/litt_block_gc.go | 77 +++++ .../block/littblock/litt_block_gc_test.go | 152 ++++++++ .../littblock/litt_block_stranding_test.go | 2 +- sei-db/ledger_db/receipt/litt_receipt_gc.go | 74 ++++ .../ledger_db/receipt/litt_receipt_gc_test.go | 111 ++++++ .../ledger_db/receipt/litt_receipt_store.go | 45 +-- sei-db/management/gc/api.go | 23 +- .../gc/storage_garbage_collector.go | 20 +- .../gc/storage_garbage_collector_test.go | 53 ++- sei-db/seiwal/seiwal.go | 13 +- sei-db/state_db/sc/flatkv/config/config.go | 22 ++ sei-db/state_db/sc/flatkv/snapshot.go | 14 +- sei-db/state_db/sc/flatkv/store.go | 80 ++--- sei-db/state_db/sc/flatkv/store_gc.go | 180 ++++++++++ sei-db/state_db/sc/flatkv/store_gc_test.go | 327 ++++++++++++++++++ .../state_db/statewal/state_wal_brick_test.go | 2 +- sei-db/state_db/statewal/state_wal_config.go | 36 ++ sei-db/state_db/statewal/state_wal_gc.go | 81 +++++ sei-db/state_db/statewal/state_wal_gc_test.go | 256 ++++++++++++++ sei-db/state_db/statewal/state_wal_impl.go | 29 +- sei-tendermint/config/autobahn.go | 6 +- .../internal/p2p/giga_router_fullnode_test.go | 2 +- .../p2p/giga_router_validator_test.go | 4 +- 28 files changed, 1560 insertions(+), 111 deletions(-) create mode 100644 sei-db/ledger_db/block/littblock/litt_block_gc.go create mode 100644 sei-db/ledger_db/block/littblock/litt_block_gc_test.go create mode 100644 sei-db/ledger_db/receipt/litt_receipt_gc.go create mode 100644 sei-db/ledger_db/receipt/litt_receipt_gc_test.go create mode 100644 sei-db/state_db/sc/flatkv/store_gc.go create mode 100644 sei-db/state_db/sc/flatkv/store_gc_test.go create mode 100644 sei-db/state_db/statewal/state_wal_gc.go create mode 100644 sei-db/state_db/statewal/state_wal_gc_test.go diff --git a/sei-db/config/giga_config.go b/sei-db/config/giga_config.go index 6aad843839..ae70cb095b 100644 --- a/sei-db/config/giga_config.go +++ b/sei-db/config/giga_config.go @@ -22,7 +22,7 @@ type GigaStorageConfig struct { FlatKVConfig *flatkvConfig.Config SSConfig StateStoreConfig ReceiptDBConfig ReceiptStoreConfig - BlockDBConfig *littblock.LittBlockConfig + BlockDBConfig *littblock.BlockDBConfig PruningConfig *gc.StorageGarbageCollectorConfig } diff --git a/sei-db/config/toml_test.go b/sei-db/config/toml_test.go index deee57a2ea..5ffb274af6 100644 --- a/sei-db/config/toml_test.go +++ b/sei-db/config/toml_test.go @@ -58,6 +58,7 @@ func TestStateCommitConfigTemplate(t *testing.T) { require.NotContains(t, flatKVSection, "snapshot-interval", "FlatKV snapshot-interval should not be exposed in app.toml") require.NotContains(t, flatKVSection, "snapshot-keep-recent", "FlatKV snapshot-keep-recent should not be exposed in app.toml") require.NotContains(t, flatKVSection, "enable-read-write-metrics", "FlatKV read/write metrics flag should not be exposed in app.toml") + require.NotContains(t, flatKVSection, "external-pruning", "FlatKV external-pruning should not be exposed in app.toml") // sc-snapshot-writer-limit is intentionally removed from template (hardcoded to 4) // but old configs with this field still parse fine via mapstructure diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index 7b8bde798b..acd57e0878 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -1402,7 +1402,7 @@ func TestMemblockPruneIntoCohortRoundsDown(t *testing.T) { // littConfig builds a littblock config rooted at dir with a tiny retention so // the prune watermark is the sole observable reclamation gate in tests. -func littConfig(t *testing.T, dir string) *littblock.LittBlockConfig { +func littConfig(t *testing.T, dir string) *littblock.BlockDBConfig { cfg, err := littblock.DefaultConfig(dir) require.NoError(t, err) cfg.Retention = time.Nanosecond diff --git a/sei-db/ledger_db/block/littblock/litt_block_config.go b/sei-db/ledger_db/block/littblock/litt_block_config.go index 1efa7e8d08..c4e8d57655 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_config.go +++ b/sei-db/ledger_db/block/littblock/litt_block_config.go @@ -5,10 +5,11 @@ import ( "time" littdb "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt" + "github.com/sei-protocol/sei-chain/sei-db/management/gc" ) -// LittBlockConfig configures a LittDB-backed types.BlockDB. -type LittBlockConfig struct { +// BlockDBConfig configures a LittDB-backed types.BlockDB. +type BlockDBConfig struct { // Litt is the underlying LittDB configuration, including the data directory // paths. The block store builds its two tables (blocks, qcs) on top of this // DB. Required; use DefaultConfig to obtain one with sane defaults, then @@ -20,23 +21,45 @@ type LittBlockConfig struct { // watermark to advance past the record, so even an over-eager watermark // cannot delete data younger than Retention. Must be positive. Retention time.Duration + + // RetentionWindow is how much history this store keeps beyond the shared rollback + // window of the StorageGarbageCollector that manages it, in blocks. It is what + // gc.PrunableStore.GetRetentionWindow answers: + // + // > 0 → that many blocks of history beyond the rollback window + // 0 → keep history to serve rollback window only + // -1 → never prune this store (gc.InfiniteRetentionWindow) + // + // Zero does NOT mean "keep everything" here, unlike the KeepRecent fields on + // StateStoreConfig and ReceiptStoreConfig, where 0 disables pruning. It is the most + // aggressive setting this field has; "keep everything" is -1. Assigning a KeepRecent + // value to this field inverts the retention it asks for. + // + // This is an input to a minimum shared across every managed store, not a policy applied + // to this store alone: a deep window here also holds back receiptDB and the SC/SS + // snapshots. Must be >= gc.InfiniteRetentionWindow. + // + // Independent of Retention, which is a wall-clock TTL failsafe underneath the watermark. + // Both must permit reclamation before any record is dropped. + RetentionWindow int64 } -// DefaultConfig returns a LittBlockConfig preloaded with all defaults, rooted at +// DefaultConfig returns a BlockDBConfig preloaded with all defaults, rooted at // dir. Override fields as needed, then pass it to NewBlockDB (which validates). -func DefaultConfig(dir string) (*LittBlockConfig, error) { +func DefaultConfig(dir string) (*BlockDBConfig, error) { littConfig, err := littdb.DefaultConfig(dir) if err != nil { return nil, fmt.Errorf("failed to build litt config: %w", err) } - return &LittBlockConfig{ - Litt: littConfig, - Retention: 24 * time.Hour, + return &BlockDBConfig{ + Litt: littConfig, + Retention: 24 * time.Hour, + RetentionWindow: 10000, }, nil } // Validate performs a sanity check on the configuration. -func (c *LittBlockConfig) Validate() error { +func (c *BlockDBConfig) Validate() error { if c == nil { return fmt.Errorf("config is required") } @@ -46,5 +69,9 @@ func (c *LittBlockConfig) Validate() error { if c.Retention <= 0 { return fmt.Errorf("config.Retention must be positive (got %s)", c.Retention) } + if c.RetentionWindow < gc.InfiniteRetentionWindow { + return fmt.Errorf("config.RetentionWindow must be >= %d (got %d)", + gc.InfiniteRetentionWindow, c.RetentionWindow) + } return nil } diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index f29576b62d..97a677362f 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -12,18 +12,19 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" ) -// ledgerTableName is the single table holding both blocks and QCs. They share +// TableName is the single table holding both blocks and QCs. They share // one table so a crash leaves a contiguous write-order prefix spanning both // record kinds (see NewBlockDB), which is what guarantees a persisted block is // always covered by a persisted QC. -const ledgerTableName = "ledger" +const TableName = "blocks" var _ types.BlockDB = (*blockDB)(nil) // blockDB is a durable types.BlockDB backed by LittDB type blockDB struct { - db littdb.DB - table littdb.Table + db littdb.DB + table littdb.Table + config *BlockDBConfig // watermark is a retention floor, always a QC boundary (a GlobalRange().First): // PruneBefore rounds a requested prune point down to the start of the cohort @@ -77,7 +78,7 @@ type blockDB struct { // underlying LittDB is built from config.Litt, and the two tables apply // config.Retention as a TTL failsafe (pruning never reclaims data younger than // that even once the watermark has advanced past it). -func NewBlockDB(config *LittBlockConfig) (types.BlockDB, error) { +func NewBlockDB(config *BlockDBConfig) (types.BlockDB, error) { if err := config.Validate(); err != nil { return nil, fmt.Errorf("invalid block db config: %w", err) } @@ -96,7 +97,7 @@ func NewBlockDB(config *LittBlockConfig) (types.BlockDB, error) { // guarantees a persisted block is always covered by a persisted QC. It also // backs the write-order cursors and contiguous-QC recovery. ShardingFactor // > 1, or splitting blocks and QCs across two tables, would void this. - tableConfig := littdb.DefaultTableConfig(ledgerTableName) + tableConfig := littdb.DefaultTableConfig(TableName) tableConfig.TTL = config.Retention tableConfig.GCFilter = s.gcFilter tableConfig.ShardingFactor = 1 // DO NOT CHANGE!! @@ -107,6 +108,7 @@ func NewBlockDB(config *LittBlockConfig) (types.BlockDB, error) { } s.table = table + s.config = config if err := s.recoverCursors(); err != nil { _ = db.Close() diff --git a/sei-db/ledger_db/block/littblock/litt_block_gc.go b/sei-db/ledger_db/block/littblock/litt_block_gc.go new file mode 100644 index 0000000000..73a514fd9e --- /dev/null +++ b/sei-db/ledger_db/block/littblock/litt_block_gc.go @@ -0,0 +1,77 @@ +package littblock + +import ( + "github.com/sei-protocol/sei-chain/sei-db/management/gc" + "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" +) + +// blockDB joins the shared prune cycle as a contiguous store: everything from its retention +// floor up to its head is retained, so it can serve a rollback to any height in between. The +// collector owns the decision of how deep to prune; PruneBefore stays the direct entry point +// for callers holding a types.BlockDB. +var _ gc.PrunableStore = (*blockDB)(nil) + +func (s *blockDB) Name() string { + return "BlockDB" +} + +// ExternalPruning is unconditionally true: this store has no pruner of its own for the collector to +// collide with. LittDB's GC reclaims what PruneBefore has already released, on a config.Retention +// timer, so it enforces no retention policy — it only carries out one this store has recorded. +func (s *blockDB) ExternalPruning() bool { + return true +} + +// PruneBelow advances the retention watermark to blockNumber. It only records the watermark; +// reclamation happens on LittDB's own GC schedule and no earlier than config.Retention (see +// PruneBefore). +// +// blockNumber is a minimum shared across every managed store, so it may sit above this store's +// own head — a store that ingests ahead of blockDB pulls the head up, and a QC boundary can +// leave the newest retained cohort below it. PruneBefore caps the request at the newest +// retained (block, QC) pair, which is what keeps the store from emptying itself here. +func (s *blockDB) PruneBelow(blockNumber uint64) error { + return s.PruneBefore(types.GlobalBlockNumber(blockNumber)) +} + +// GetRetentionWindow reports the configured history beyond the collector's shared rollback +// window. See BlockDBConfig.RetentionWindow for the meaning of each value, and note that it is +// an input to a fleet-wide minimum rather than a policy applied to this store alone. +func (s *blockDB) GetRetentionWindow() int64 { + if s.config.RetentionWindow < 0 { + return gc.InfiniteRetentionWindow + } + return s.config.RetentionWindow +} + +// GetPruningBoundary returns cutLine, the contract's answer for a contiguous store: every block +// at or above the watermark is retained, so cutLine itself is restorable and nothing below it +// has to be held back. +// +// Unconditional on purpose. A store whose floor already sits above cutLine — bootstrapped +// mid-chain, or pruned there by an earlier cycle — still answers cutLine, because the +// PruneBefore that follows is a no-op on it while a lower answer would hold back every other +// store. CannotServeRollback is never right here: this store fills from its own ingest path, +// so it has no replay range for another store's data to protect. +func (s *blockDB) GetPruningBoundary(cutLine uint64) uint64 { + return cutLine +} + +// GetLatestBlock returns the newest block number written, or 0 when none has been. +// +// Global block numbers start at genesis block 0, so a store holding only that block is +// indistinguishable from an empty one and is excluded from the collector's head. That is the +// safe direction: it drops out of the head minimum rather than dragging every store's cut line +// to 0, and the prune it then receives is capped by PruneBefore to a no-op. +// +// Reports the written cursor, not the flushed one. A block that a crash would lose still counts +// as ingested — recovery re-derives this cursor from what survived, so the head can only move +// back, never past a prune that was already issued. +func (s *blockDB) GetLatestBlock() (uint64, error) { + s.mu.Lock() + defer s.mu.Unlock() + if !s.hasBlocks { + return 0, nil + } + return uint64(s.lastBlockNumber), nil +} diff --git a/sei-db/ledger_db/block/littblock/litt_block_gc_test.go b/sei-db/ledger_db/block/littblock/litt_block_gc_test.go new file mode 100644 index 0000000000..450a466699 --- /dev/null +++ b/sei-db/ledger_db/block/littblock/litt_block_gc_test.go @@ -0,0 +1,152 @@ +package littblock + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/management/gc" + "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" +) + +// gcConfig builds a config for the collector-facing tests: reclamation is gated solely by the +// prune watermark (tiny TTL) and only ForceGC reclaims, so a watermark assertion is never +// confused by a background pass. Segment sizing is left at its defaults — these tests are about +// the decisions the collector drives, not about which segment a record lands in. +func gcConfig(t *testing.T, dir string) *BlockDBConfig { + cfg, err := DefaultConfig(dir) + require.NoError(t, err) + cfg.Retention = time.Nanosecond + cfg.Litt.GCPeriod = time.Hour + cfg.Litt.Fsync = false + return cfg +} + +// openForGC opens a store at dir and returns it as the collector sees it, closing it on cleanup. +// Opening a store is the expensive part of these tests, so each one opens as few as it can. +func openForGC(t *testing.T, dir string) (types.BlockDB, gc.PrunableStore) { + t.Helper() + db, err := NewBlockDB(gcConfig(t, dir)) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + store, ok := db.(gc.PrunableStore) + require.True(t, ok, "a littblock store must satisfy gc.PrunableStore") + return db, store +} + +// The head the collector reads is the newest block written — not the newest QC's coverage, since +// a QC is written before the blocks it covers — and 0 while nothing has been ingested, so an +// empty store drops out of the head minimum instead of dragging every cut line to 0. The reopen +// at the end covers recovery: a store that reported 0 after a restart would let the other stores +// prune past a height this one still holds. +func TestGCLatestBlock(t *testing.T) { + dir := t.TempDir() + rng := utils.TestRngFromSeed(1) + db, store := openForGC(t, dir) + + latest, err := store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(0), latest, "a store that has ingested nothing has no head") + + writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19, QCs [0,5)..[15,20) + latest, err = store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(19), latest) + + // A QC covering 20..24 is written but none of its blocks are, so the head must not move. + require.NoError(t, db.WriteQC(types.GenFullCommitQCRange(rng, 20, 25))) + latest, err = store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(19), latest, "a QC ahead of its blocks must not advance the head") + + require.NoError(t, db.WriteBlock(20, types.GenBlock(rng))) + require.NoError(t, db.Flush()) + require.NoError(t, db.Close()) + + _, reopened := openForGC(t, dir) + latest, err = reopened.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(20), latest, "the head must be re-derived on open") +} + +// The window the collector applies is the configured one, sentinel included: -1 has to survive +// the trip verbatim or an operator asking for "never prune" gets a 1-block window instead. +func TestGCRetentionWindowComesFromConfig(t *testing.T) { + for _, retentionWindow := range []int64{gc.InfiniteRetentionWindow, 100_000} { + cfg := gcConfig(t, t.TempDir()) + cfg.RetentionWindow = retentionWindow + + db, err := NewBlockDB(cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + + require.Equal(t, retentionWindow, db.(gc.PrunableStore).GetRetentionWindow()) + } +} + +// A contiguous store answers the cut line it was given whatever it holds. The states below are +// the ones where a snapshot store would answer differently, and answering under cutLine in any +// of them would hold every other store back to this store's floor for no benefit. +// +// The prune assertions ride along on the same store because both methods are answers about the +// same three states, and opening a second store to re-reach them is the slow part of this file. +func TestGCPruningBoundaryAndPruneBelow(t *testing.T) { + db, store := openForGC(t, t.TempDir()) + rng := utils.TestRngFromSeed(2) + impl := db.(*blockDB) + + // Empty. The prune that follows is a no-op rather than an error — a store still filling is + // pruned like any other — and the boundary is still cutLine, since CannotServeRollback here + // would stall every other store. + require.Equal(t, uint64(42), store.GetPruningBoundary(42)) + require.NoError(t, store.PruneBelow(1_000)) + require.Equal(t, uint64(0), impl.watermark.Load()) + + writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19, QCs [0,5),[5,10),[10,15),[15,20) + require.Equal(t, uint64(7), store.GetPruningBoundary(7)) + // Above the head, which happens whenever another store's data puts the head above this one's. + require.Equal(t, uint64(1_000), store.GetPruningBoundary(1_000)) + + require.NoError(t, store.PruneBelow(7)) + require.Equal(t, uint64(5), impl.watermark.Load(), "a prune inside QC[5,10) rounds down to its start") + require.NoError(t, store.PruneBelow(3)) + require.Equal(t, uint64(5), impl.watermark.Load(), "the watermark must not move backwards") + + // Pruned above the cut line: nothing below it survives to protect, so cutLine still stands. + require.Equal(t, uint64(4), store.GetPruningBoundary(4)) +} + +// The collector prunes every store to a shared minimum, so a store lagging the head is asked to +// prune past everything it holds. The never-empty cap turns that into a prune to the newest +// cohort rather than a store that can serve nothing. +func TestGCPruneBelowAboveHeadIsCapped(t *testing.T) { + db, store := openForGC(t, t.TempDir()) + rng := utils.TestRngFromSeed(3) + + writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19, newest cohort QC[15,20) + require.NoError(t, store.PruneBelow(1_000)) + require.Equal(t, uint64(15), db.(*blockDB).watermark.Load(), "a prune past the head is capped to the newest cohort") + + for n := types.GlobalBlockNumber(15); n < 20; n++ { + blk, err := db.ReadBlockByNumber(n) + require.NoError(t, err) + require.True(t, blk.IsPresent(), "block %d in the newest cohort must survive", n) + } +} + +func TestConfigValidateRetentionWindow(t *testing.T) { + cfg, err := DefaultConfig(t.TempDir()) + require.NoError(t, err) + + for _, retentionWindow := range []int64{gc.InfiniteRetentionWindow, 0, 1} { + cfg.RetentionWindow = retentionWindow + require.NoError(t, cfg.Validate()) + } + + // Below InfiniteRetentionWindow there is no meaning left to assign, and the collector reads + // any negative value as infinite retention — so a typo'd -2 would silently disable pruning. + cfg.RetentionWindow = -2 + require.ErrorContains(t, cfg.Validate(), "RetentionWindow") +} diff --git a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go index dad0065f8f..ec070534ff 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go +++ b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go @@ -15,7 +15,7 @@ import ( // MaxSegmentKeyCount, so a test can place a segment boundary at a precise key. // Retention is tiny (the prune watermark is the sole reclamation gate) and GC is // effectively background-disabled so ForceGC is the only thing that reclaims. -func strandingConfig(t *testing.T, dir string, maxSegmentKeyCount uint32) *LittBlockConfig { +func strandingConfig(t *testing.T, dir string, maxSegmentKeyCount uint32) *BlockDBConfig { cfg, err := DefaultConfig(dir) require.NoError(t, err) cfg.Retention = time.Nanosecond diff --git a/sei-db/ledger_db/receipt/litt_receipt_gc.go b/sei-db/ledger_db/receipt/litt_receipt_gc.go new file mode 100644 index 0000000000..b4033b106c --- /dev/null +++ b/sei-db/ledger_db/receipt/litt_receipt_gc.go @@ -0,0 +1,74 @@ +package receipt + +import "github.com/sei-protocol/sei-chain/sei-db/management/gc" + +// littReceiptStore joins the shared prune cycle as a contiguous store: it holds every block from +// its retention floor up to its head, so any height in between can serve a rollback. +// +// Only the littidx backend participates. The pebble backend delegates to the state store, which +// prunes on its own KeepRecent schedule. +var _ gc.PrunableStore = (*littReceiptStore)(nil) + +func (s *littReceiptStore) Name() string { + return "ReceiptDB" +} + +// ExternalPruning is unconditionally true: the background pruner this store used to run was removed +// when it joined the collector, precisely because it pruned to latest-KeepRecent with no knowledge of +// the shared rollback window. Nothing here prunes on its own any more. +func (s *littReceiptStore) ExternalPruning() bool { + return true +} + +// PruneBelow advances the retention floor to blockNumber and drops the tag-index entries below +// it. Receipt bodies are not deleted here: litt expires them by TTL, and reads below the floor +// return not-found in the meantime (see belowRetentionFloor), so visible retention follows this +// call even when reclamation lags it. +func (s *littReceiptStore) PruneBelow(blockNumber uint64) error { + return s.pruneBlocksBelow(blockNumber) +} + +// GetRetentionWindow translates KeepRecent into the collector's window. +// +// The two disagree on what 0 means, and the disagreement is the whole reason this is not a plain +// field read: KeepRecent 0 means "keep everything, never prune" (it is derived from +// min-retain-blocks, and 0 is the default), while a 0 here means "keep nothing beyond the shared +// RollbackWindow" — the most aggressive answer available. Returning it verbatim would prune a +// store configured to retain forever back to the rollback window. gc.InfiniteRetentionWindow is +// the sentinel that carries the intended meaning, and it is a legitimate answer for a contiguous +// store, which holds no replay range another store depends on. +// +// Negative values are folded into the same case: KeepRecent is never negative in practice, and +// this store has historically read <= 0 as "pruning off", so folding them keeps that reading +// intact rather than passing an out-of-contract value to the collector. +func (s *littReceiptStore) GetRetentionWindow() int64 { + if s.keepRecent <= 0 { + return gc.InfiniteRetentionWindow + } + return s.keepRecent +} + +// GetPruningBoundary returns cutLine, the contract's answer for a contiguous store: every block +// at or above the floor is retained, so cutLine itself is restorable and nothing below it has to +// be held back. +// +// Unconditional on purpose. A store whose floor already sits above cutLine — pruned there by an +// earlier cycle, or still backfilling — also answers cutLine, because the PruneBelow that follows +// is a no-op on it while a lower answer would hold every other store back to this store's floor. +// CannotServeRollback is never right here: receipts are written from this node's own execution +// path, so no other store replays out of them. +func (s *littReceiptStore) GetPruningBoundary(cutLine uint64) uint64 { + return cutLine +} + +// GetLatestBlock returns the newest block whose receipts have been written, or 0 when none have. +// 0 keeps the store out of the collector's head minimum rather than dragging every store's cut +// line down to it — the right trade while a store is still filling, since the prune it then +// receives only moves a floor that has no data under it. +func (s *littReceiptStore) GetLatestBlock() (uint64, error) { + latest := s.latestVersion.Load() + if latest <= 0 { + return 0, nil + } + return uint64(latest), nil //nolint:gosec // guarded non-negative above +} diff --git a/sei-db/ledger_db/receipt/litt_receipt_gc_test.go b/sei-db/ledger_db/receipt/litt_receipt_gc_test.go new file mode 100644 index 0000000000..957fa78dcd --- /dev/null +++ b/sei-db/ledger_db/receipt/litt_receipt_gc_test.go @@ -0,0 +1,111 @@ +package receipt_test + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common" + storetypes "github.com/sei-protocol/sei-chain/sei-cosmos/store/types" + "github.com/sei-protocol/sei-chain/sei-cosmos/testutil" + sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" + dbconfig "github.com/sei-protocol/sei-chain/sei-db/config" + "github.com/sei-protocol/sei-chain/sei-db/ledger_db/receipt" + "github.com/sei-protocol/sei-chain/sei-db/management/gc" + "github.com/stretchr/testify/require" +) + +// setupLittIdxForGC opens a littidx store with the given KeepRecent and no background pruner, so +// the only thing that moves the retention floor is the collector call under test. +func setupLittIdxForGC(t *testing.T, keepRecent int) (receipt.ReceiptStore, gc.PrunableStore, sdk.Context) { + t.Helper() + storeKey := storetypes.NewKVStoreKey("evm") + tkey := storetypes.NewTransientStoreKey("evm_transient") + ctx := testutil.DefaultContext(storeKey, tkey).WithBlockHeight(1) + + cfg := dbconfig.DefaultReceiptStoreConfig() + cfg.Backend = "littidx" + cfg.DBDirectory = t.TempDir() + cfg.KeepRecent = keepRecent + cfg.PruneIntervalSeconds = 0 + + store, err := receipt.NewReceiptStore(cfg, storeKey) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + prunable, ok := store.(gc.PrunableStore) + require.True(t, ok, "a littidx receipt store must satisfy gc.PrunableStore") + return store, prunable, ctx +} + +// KeepRecent and GetRetentionWindow disagree about 0, so the translation is the behavior worth +// pinning: KeepRecent 0 means "keep everything" and is the default, while a literal 0 answer +// means "keep only the shared rollback window". Returning the field verbatim would prune a store +// configured to retain forever back to ~1_000 blocks. +func TestReceiptGCRetentionWindowMapsKeepRecent(t *testing.T) { + for _, tc := range []struct { + name string + keepRecent int + want int64 + }{ + {name: "keep everything", keepRecent: 0, want: gc.InfiniteRetentionWindow}, + {name: "bounded retention", keepRecent: 100_000, want: 100_000}, + } { + t.Run(tc.name, func(t *testing.T) { + _, prunable, _ := setupLittIdxForGC(t, tc.keepRecent) + require.Equal(t, tc.want, prunable.GetRetentionWindow()) + }) + } +} + +// The head is 0 until receipts land, which keeps a store that is still filling out of the +// collector's head minimum instead of dragging every store's cut line down to it. +func TestReceiptGCLatestBlock(t *testing.T) { + store, prunable, ctx := setupLittIdxForGC(t, 0) + + latest, err := prunable.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(0), latest, "a store that has ingested nothing has no head") + + addr := common.HexToAddress("0xabcd") + topic := common.HexToHash("0x1111") + writeLitBlock(t, store, ctx, 1, litReceipt(1, 0, addr, topic)) + writeLitBlock(t, store, ctx, 2, litReceipt(2, 0, addr, topic)) + writeLitBlock(t, store, ctx, 3, litReceipt(3, 0, addr, topic)) + + latest, err = prunable.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(3), latest) + require.Equal(t, int64(3), store.LatestVersion(), "the collector's head must agree with the store's own version") +} + +// A contiguous store answers the cut line it was given whatever it holds, and the prune that +// follows moves the retention floor to it — which is what makes the receipts below it stop being +// served, even though litt reclaims their bodies later on its own TTL schedule. +func TestReceiptGCPruningBoundaryAndPruneBelow(t *testing.T) { + store, prunable, ctx := setupLittIdxForGC(t, 0) + addr := common.HexToAddress("0xabcd") + topic := common.HexToHash("0x1111") + + // Holding nothing: the boundary is still cutLine, since CannotServeRollback here would stall + // every other store rather than protect anything. + require.Equal(t, uint64(42), prunable.GetPruningBoundary(42)) + + for block := uint64(1); block <= 3; block++ { + writeLitBlock(t, store, ctx, block, litReceipt(block, 0, addr, topic)) + } + require.Equal(t, uint64(2), prunable.GetPruningBoundary(2)) + // Above the head, which happens whenever another store's data puts the head above this one's. + require.Equal(t, uint64(1_000), prunable.GetPruningBoundary(1_000)) + + require.NoError(t, prunable.PruneBelow(3)) + require.Equal(t, int64(3), store.EarliestVersion(), "PruneBelow must advance the retention floor") + + _, err := store.GetReceiptFromStore(ctx, litTxHash(1, 0)) + require.ErrorIs(t, err, receipt.ErrNotFound, "a receipt below the floor must not be served") + kept, err := store.GetReceiptFromStore(ctx, litTxHash(3, 0)) + require.NoError(t, err) + require.Equal(t, uint64(3), kept.BlockNumber) + + // The floor only advances: a later, lower cycle must not re-expose what was pruned. + require.NoError(t, prunable.PruneBelow(1)) + require.Equal(t, int64(3), store.EarliestVersion()) +} diff --git a/sei-db/ledger_db/receipt/litt_receipt_store.go b/sei-db/ledger_db/receipt/litt_receipt_store.go index 069bc127a5..4dc2f34c14 100644 --- a/sei-db/ledger_db/receipt/litt_receipt_store.go +++ b/sei-db/ledger_db/receipt/litt_receipt_store.go @@ -5,7 +5,6 @@ import ( "encoding/binary" "errors" "fmt" - "math/rand" "os" "path/filepath" "sort" @@ -51,8 +50,15 @@ import ( // missing body. The tight interval is only affordable because litt flushes its // keymap asynchronously off the control loop. Retention: receipt values // expire via litt's per-table TTL (time based), tag keys are pruned by block -// range, and reads enforce the KeepRecent floor, so visible retention never -// exceeds KeepRecent regardless of GC timing. +// range, and reads enforce the retention floor, so visible retention never +// exceeds that floor regardless of GC timing. +// +// Pruning is driven from outside, by the StorageGarbageCollector through the +// gc.PrunableStore implementation in litt_receipt_gc.go. This store deliberately +// runs no pruner of its own: KeepRecent alone sees neither the shared rollback +// window nor the other stores' floors, so a local pruner would delete the +// rollback headroom the collector exists to preserve. KeepRecent remains what +// this store asks the collector to retain, and what sizes the litt TTL. type littReceiptStore struct { values litt.DB receipts litt.Table @@ -63,7 +69,6 @@ type littReceiptStore struct { earliestVersion atomic.Int64 keepRecent int64 - pruneInterval int64 logFilterParallelism int stopBackground chan struct{} backgroundWg sync.WaitGroup @@ -162,13 +167,11 @@ func newLittReceiptStore(cfg dbconfig.ReceiptStoreConfig, storeKey sdk.StoreKey) index: index, storeKey: storeKey, keepRecent: int64(cfg.KeepRecent), - pruneInterval: int64(cfg.PruneIntervalSeconds), logFilterParallelism: logFilterParallelism, stopBackground: make(chan struct{}), } s.latestVersion.Store(s.readMeta(receiptLatestVersionKey)) s.earliestVersion.Store(s.readMeta(receiptEarliestVersionKey)) - s.startPruning() s.startFlusher() return s, nil } @@ -398,36 +401,12 @@ func (s *littReceiptStore) Close() error { return err } -func (s *littReceiptStore) startPruning() { - if s.keepRecent <= 0 || s.pruneInterval <= 0 { - return - } - s.backgroundWg.Add(1) - go func() { - defer s.backgroundWg.Done() - for { - // Keep exactly keepRecent blocks, [latest-keepRecent+1, latest]; the - // +1 matches the pebble backend (which retains keepRecent, not +1). - pruneBefore := s.latestVersion.Load() - s.keepRecent + 1 - if pruneBefore > 0 { - if err := s.pruneBlocksBelow(uint64(pruneBefore)); err != nil { - logger.Error("failed to prune littdb receipt store", "before-block", pruneBefore, "err", err) - } - } - // Jittered cadence, matching the other receipt pruners. - sleep := time.Duration(float64(s.pruneInterval)*(1+rand.Float64())) * time.Second - select { - case <-s.stopBackground: - return - case <-time.After(sleep): - } - } - }() -} - // pruneBlocksBelow deletes the tag entries in [earliest, cutoff) and advances // the retention floor. Receipt values are reclaimed independently by litt's TTL // GC; the read-time floor keeps them invisible in the meantime. +// +// Called by the StorageGarbageCollector via PruneBelow, which is the only thing +// that advances this floor in production — see the type doc. func (s *littReceiptStore) pruneBlocksBelow(cutoff uint64) error { floor := uint64(0) if earliest := s.earliestVersion.Load(); earliest > 0 { diff --git a/sei-db/management/gc/api.go b/sei-db/management/gc/api.go index 89644074e0..803124b7cd 100644 --- a/sei-db/management/gc/api.go +++ b/sei-db/management/gc/api.go @@ -36,9 +36,30 @@ type PrunableStore interface { Name() string // PruneBelow may drop data for all blocks below blockNumber. The store may perform the - // deletion asynchronously. + // deletion asynchronously. Only called when ExternalPruning reports true. PruneBelow(blockNumber uint64) error + // ExternalPruning reports whether this store's retention is the collector's to enforce. + // + // true → the collector prunes it; any pruner inside the store must stand down + // false → the store prunes itself; the collector never calls PruneBelow on it + // + // This exists so that "the collector manages this store" and "the store's own pruner is + // off" are one fact rather than two settings that can disagree. A store with an internal + // pruner answers from the same field that pruner consults, which is what makes the two + // mutually exclusive by construction instead of by wiring discipline. A store with no + // pruner of its own returns true unconditionally. + // + // false does NOT withdraw the store from the decision. It is still asked for + // GetLatestBlock and GetPruningBoundary, and still holds the shared minimum down, because + // what a self-pruning store retains can be exactly what another store must replay from — + // SC's snapshots are useless if the WAL beneath them has been pruned away. Dropping it + // from the vote would prune that range out from under it. Opting out of being pruned and + // opting out of protecting others are different things, and only the first is on offer. + // + // Read once per cycle, so a store may change its answer between cycles but not within one. + ExternalPruning() bool + // GetRetentionWindow is how many extra blocks beyond the shared RollbackWindow this store // needs to keep servable, where head is the min non-zero GetLatestBlock. Three answers: // diff --git a/sei-db/management/gc/storage_garbage_collector.go b/sei-db/management/gc/storage_garbage_collector.go index 7dc50f3333..8e70d2349e 100644 --- a/sei-db/management/gc/storage_garbage_collector.go +++ b/sei-db/management/gc/storage_garbage_collector.go @@ -24,7 +24,7 @@ var logger = seilog.NewLogger("db", "gc") // 3. ask GetPruningBoundary(cutLine): a positive answer votes, and CannotServeRollback // abandons the cycle while RollbackWindow > 0 // 4. pruneHeight = min of the positive answers; PruneBelow(pruneHeight) on every store -// that answered positively +// that answered positively and reports ExternalPruning // // Step 4 prunes to the shared minimum rather than to each store's own boundary so that a // retained snapshot stays restorable: contiguous stores must still hold the blocks that follow @@ -147,6 +147,7 @@ func prune(config *StorageGarbageCollectorConfig, stores []PrunableStore) error } decisions[i].cutLine = cutLine + decisions[i].externalPruning = store.ExternalPruning() decisions[i].boundary = store.GetPruningBoundary(cutLine) if decisions[i].boundary == CannotServeRollback { // One blocker is enough to abandon the cycle, but the rest are still asked rather than @@ -192,6 +193,12 @@ func prune(config *StorageGarbageCollectorConfig, stores []PrunableStore) error if decisions[i].boundary == 0 { continue } + // A self-pruning store still voted above, and is still protected by the minimum that + // vote produced. What it does not get is a PruneBelow, because its own pruner is the + // one enforcing its retention (see PrunableStore.ExternalPruning). + if !decisions[i].externalPruning { + continue + } if err := store.PruneBelow(pruneHeight); err != nil { pruneErrs = errors.Join(pruneErrs, fmt.Errorf("failed to prune %s below %d: %w", store.Name(), pruneHeight, err)) } @@ -203,14 +210,19 @@ func prune(config *StorageGarbageCollectorConfig, stores []PrunableStore) error // log. cutLine == 0 means the store was never asked, which is what separates it from a store that // was asked and answered 0. type storeDecision struct { - cutLine uint64 - boundary uint64 + cutLine uint64 + boundary uint64 + externalPruning bool } // describeDecisions renders one entry per store, in store order, for the prune log. The three // outcomes render differently on purpose — never asked, cannot serve a rollback, and reported a // boundary — since collapsing them would cost exactly the distinction wanted when auditing a // deletion after the fact. Each asked store carries the cut line its answer is a function of. +// +// A store that voted but prunes itself is tagged selfPruned, because its boundary is in the +// minimum below while no deletion follows on it — without the tag that reads like a prune that +// silently did nothing. func describeDecisions(stores []PrunableStore, decisions []storeDecision) string { var sb strings.Builder for i, store := range stores { @@ -222,6 +234,8 @@ func describeDecisions(stores []PrunableStore, decisions []storeDecision) string fmt.Fprintf(&sb, "%s=notAsked", store.Name()) case decisions[i].boundary == CannotServeRollback: fmt.Fprintf(&sb, "%s=cannotServeRollback(cutLine=%d)", store.Name(), decisions[i].cutLine) + case !decisions[i].externalPruning: + fmt.Fprintf(&sb, "%s=%d(cutLine=%d,selfPruned)", store.Name(), decisions[i].boundary, decisions[i].cutLine) default: fmt.Fprintf(&sb, "%s=%d(cutLine=%d)", store.Name(), decisions[i].boundary, decisions[i].cutLine) } diff --git a/sei-db/management/gc/storage_garbage_collector_test.go b/sei-db/management/gc/storage_garbage_collector_test.go index 89d562179c..2ecfbcc6eb 100644 --- a/sei-db/management/gc/storage_garbage_collector_test.go +++ b/sei-db/management/gc/storage_garbage_collector_test.go @@ -20,6 +20,9 @@ type mockStore struct { pruningBoundary func(cutLine uint64) uint64 getErr error pruneErr error + // selfPruned inverts ExternalPruning: the constructors below build collector-managed stores, + // which is the common case, so opting out is what a test has to say explicitly. + selfPruned bool pruneBelowCalled atomic.Bool prunedBelow atomic.Uint64 @@ -73,6 +76,17 @@ func withRetentionWindow(store *mockStore, retention int64) *mockStore { return store } +// withSelfPruning marks a store as enforcing its own retention, so the collector must vote it in but +// never call PruneBelow on it. +func withSelfPruning(store *mockStore) *mockStore { + store.selfPruned = true + return store +} + +func (m *mockStore) ExternalPruning() bool { + return !m.selfPruned +} + func (m *mockStore) Name() string { return m.name } @@ -503,20 +517,51 @@ func TestDescribeDecisionsRendersEachOutcomeDistinctly(t *testing.T) { withRetentionWindow(contiguousStore("archiveWAL", 100_000), InfiniteRetentionWindow), snapshotStore("sc", 100_000), contiguousStore("stateWAL", 100_000), + withSelfPruning(snapshotStore("selfSC", 100_000, 50_000)), ) decisions := []storeDecision{ - {cutLine: 0, boundary: CannotServeRollback}, // skipped before being asked - {cutLine: 90_000, boundary: CannotServeRollback}, // asked, cannot serve a rollback - {cutLine: 90_000, boundary: 90_000}, // asked, reported a boundary + {cutLine: 0, boundary: CannotServeRollback}, // skipped before being asked + {cutLine: 90_000, boundary: CannotServeRollback}, // asked, cannot serve a rollback + {cutLine: 90_000, boundary: 90_000, externalPruning: true}, // asked, reported a boundary + {cutLine: 90_000, boundary: 50_000}, // asked, but prunes itself } require.Equal(t, "archiveWAL=notAsked sc=cannotServeRollback(cutLine=90000) "+ - "stateWAL=90000(cutLine=90000)", + "stateWAL=90000(cutLine=90000) selfSC=50000(cutLine=90000,selfPruned)", describeDecisions(stores, decisions), ) } +// The whole point of ExternalPruning: a self-pruning store is still a full participant in the +// decision — its boundary drags the shared minimum down and protects the range it replays from — +// while never being pruned by the collector. Withdrawing it from the vote instead would prune the +// WAL to 99_000 and strand its snapshot at 50_000 with nothing to replay forward. +func TestPruneSkipsSelfPruningStoreButKeepsItsVote(t *testing.T) { + selfPruned := withSelfPruning(snapshotStore("sc", 100_000, 50_000)) + wal := contiguousStore("stateWAL", 100_000) + + require.NoError(t, prune(testConfig(t, 1_000), prunableStores(selfPruned, wal))) + + require.Equal(t, uint64(1), selfPruned.boundaryCalls.Load(), "a self-pruning store is still asked") + require.False(t, selfPruned.pruneBelowCalled.Load(), "its own pruner enforces its retention") + require.True(t, wal.pruneBelowCalled.Load()) + require.Equal(t, uint64(50_000), wal.prunedBelow.Load(), + "the self-pruning store's boundary must still hold the WAL back to its snapshot") +} + +// A self-pruning store that cannot serve a rollback still abandons the cycle. Who deletes its data +// is a separate question from whether the range it needs exists yet, and only the latter is what +// CannotServeRollback reports. +func TestPruneSelfPruningStoreStillBlocksCycle(t *testing.T) { + blocker := withSelfPruning(snapshotStore("sc", 100_000)) // no snapshots + wal := contiguousStore("stateWAL", 100_000) + + require.NoError(t, prune(testConfig(t, 1_000), prunableStores(blocker, wal))) + + require.False(t, wal.pruneBelowCalled.Load(), "the cycle must be abandoned") +} + // One blocker abandons the cycle, but every store is still asked so the blocked cycle can log a // complete decision set. Under RollbackWindow 0 the guarantee is waived and the blocker is ignored // outright, which is what lets the later stores prune. diff --git a/sei-db/seiwal/seiwal.go b/sei-db/seiwal/seiwal.go index e0ce5f161a..f1efaa2ab0 100644 --- a/sei-db/seiwal/seiwal.go +++ b/sei-db/seiwal/seiwal.go @@ -18,7 +18,7 @@ var ErrIteratorRange = errors.New("invalid iterator range") // // A WAL instance is not safe for concurrent use: its methods must not be called from multiple // goroutines simultaneously. Callers that share a WAL across goroutines must serialize access -// themselves. +// themselves. PruneBefore is the sole exception; see its doc for the guarantee implementations owe. // // Slices are not copied at the call boundary. Any slice passed into a WAL method — the payload and every // slice reachable through it — must not be modified after the call: the WAL may retain it and read it @@ -64,6 +64,17 @@ type WAL[T any] interface { // async and lazy, and implementations are free to delay it arbitrarily long. Pruning removes whole // sealed files only, so records may survive above the requested threshold until their containing file // is fully below it. + // + // Unlike every other method here, PruneBefore may be called from a goroutine other than the WAL's + // owner, concurrently with any method including Append and Close, and implementations must support + // that without external serialization. Retention is driven by a garbage collector on its own + // goroutine, and requiring it to take the writer's turn would mean either blocking the writer or + // deferring the prune until the writer next runs — the latter stalling reclamation indefinitely on a + // WAL that has stopped receiving appends. + // + // Concurrent calls are unordered with respect to appends: whether a record appended around the same + // instant is pruned is unspecified. This costs nothing, because which records a prune actually + // reclaims is already approximate — it drops whole sealed files, and may defer the work arbitrarily. PruneBefore(lowestIndexToKeep uint64) error // Iterator returns an iterator over the WAL across the inclusive index range [startIndex, endIndex]. diff --git a/sei-db/state_db/sc/flatkv/config/config.go b/sei-db/state_db/sc/flatkv/config/config.go index 764e3fba39..f75a42d785 100644 --- a/sei-db/state_db/sc/flatkv/config/config.go +++ b/sei-db/state_db/sc/flatkv/config/config.go @@ -39,9 +39,31 @@ type Config struct { // SnapshotKeepRecent defines how many old snapshots to keep besides the // latest one. 0 means keep only the current snapshot (no old snapshots). + // Ignored entirely when ExternalPruning is set. // Default: 1 SnapshotKeepRecent uint32 `mapstructure:"snapshot-keep-recent"` + // ExternalPruning hands retention to the StorageGarbageCollector: the store stops pruning its + // own snapshots (SnapshotKeepRecent) and stops truncating the state WAL, and answers + // gc.PrunableStore.ExternalPruning with this value so the collector takes both over. + // + // Set this only when the store is actually registered with a running collector. Nothing here + // can check that, and the failure is silent in the expensive direction: snapshots and the WAL + // then grow without bound, because the machinery that used to bound them has stood down and + // nothing replaced it. + // + // It is one field rather than two so the count-based pruner and the collector can never both + // be enforcing retention — they would disagree, and SnapshotKeepRecent would win by deleting + // the snapshot the collector was holding for the rollback window. + // + // Retention changes shape when this is on, it does not merely move: snapshots are kept by + // height rather than by count, so how many exist becomes RollbackWindow / SnapshotInterval + // instead of SnapshotKeepRecent + 1. A deep rollback window with a short interval retains far + // more snapshots than the count-based default. + // + // Default: false + ExternalPruning bool `mapstructure:"external-pruning"` + // EnablePebbleMetrics defines if the Pebble metrics should be enabled. // Default: true EnablePebbleMetrics bool `mapstructure:"enable-pebble-metrics"` diff --git a/sei-db/state_db/sc/flatkv/snapshot.go b/sei-db/state_db/sc/flatkv/snapshot.go index 9dd738f84d..12f6059ede 100644 --- a/sei-db/state_db/sc/flatkv/snapshot.go +++ b/sei-db/state_db/sc/flatkv/snapshot.go @@ -526,7 +526,16 @@ func (s *CommitStore) WriteSnapshot(_ string) (err error) { // pruneSnapshots removes old snapshots beyond SnapshotKeepRecent, keeping // the latest snapshot (currentVersion) plus the N most recent older ones. // Best-effort: errors are logged but do not fail the snapshot operation. +// +// Disabled by config.ExternalPruning, which hands retention to the +// StorageGarbageCollector. Both must never run: this one counts snapshots and +// knows nothing of the shared rollback window, so it would delete the snapshot +// the collector is holding to serve it. func (s *CommitStore) pruneSnapshots(dir string, currentVersion int64) int { + if s.config.ExternalPruning { + return 0 + } + start := time.Now() defer func() { otelMetrics.SnapshotPruneLatency.Record(s.ctx, secondsSince(start)) @@ -726,8 +735,11 @@ func (s *CommitStore) Rollback(targetVersion int64) (err error) { // to any retained snapshot. Scheduling the truncation is best-effort in that it is skipped when there is // nothing to prune against, but a prune that fails is not a benign outcome: it only fails when the WAL is // already dead, which means commits will fail from that point on. +// +// Disabled by config.ExternalPruning, under which the WAL is a managed store in its own right and the +// collector prunes it to a floor derived from every store, not just from this one's snapshots. func (s *CommitStore) tryTruncateWAL() { - if s.wal == nil { + if s.wal == nil || s.config.ExternalPruning { return } diff --git a/sei-db/state_db/sc/flatkv/store.go b/sei-db/state_db/sc/flatkv/store.go index 2adaf26976..5f7c07c0c9 100644 --- a/sei-db/state_db/sc/flatkv/store.go +++ b/sei-db/state_db/sc/flatkv/store.go @@ -58,44 +58,7 @@ const ( // dataDBDirs lists all data DB directory names (used for per-DB LtHash iteration). var dataDBDirs = []string{accountDBDir, codeDBDir, storageDBDir, miscDBDir} -// InitializeDataDirectories sets the DataDir for each nested PebbleDB config -// that does not already have one, using DataDir as the base path. The DBs live -// under the working directory: /working/. -func InitializeDataDirectories(c *config.Config) { - workDir := filepath.Join(c.DataDir, workingDirName) - if c.AccountDBConfig.DataDir == "" { - c.AccountDBConfig.DataDir = filepath.Join(workDir, accountDBDir) - } - if c.CodeDBConfig.DataDir == "" { - c.CodeDBConfig.DataDir = filepath.Join(workDir, codeDBDir) - } - if c.StorageDBConfig.DataDir == "" { - c.StorageDBConfig.DataDir = filepath.Join(workDir, storageDBDir) - } - if c.MiscDBConfig.DataDir == "" { - c.MiscDBConfig.DataDir = filepath.Join(workDir, miscDBDir) - } - if c.MetadataDBConfig.DataDir == "" { - c.MetadataDBConfig.DataDir = filepath.Join(workDir, metadataDir) - } - applyPebbleMetricsConfig(c) -} - -func applyPebbleMetricsConfig(c *config.Config) { - // Keep a single FlatKV-level knob for Pebble internal metrics. Per-DB - // EnableMetrics values are intentionally overwritten here. - c.AccountDBConfig.EnableMetrics = c.EnablePebbleMetrics - c.CodeDBConfig.EnableMetrics = c.EnablePebbleMetrics - c.StorageDBConfig.EnableMetrics = c.EnablePebbleMetrics - c.MiscDBConfig.EnableMetrics = c.EnablePebbleMetrics - c.MetadataDBConfig.EnableMetrics = c.EnablePebbleMetrics - - c.AccountDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics - c.CodeDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics - c.StorageDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics - c.MiscDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics - c.MetadataDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics -} +var _ Store = (*CommitStore)(nil) // CommitStore implements flatkv.Store for EVM state storage. // @@ -221,8 +184,6 @@ type CommitStore struct { ltCalc *lthash.HashCalculator } -var _ Store = (*CommitStore)(nil) - // dataDBs returns the four data PebbleDB instances in fixed iteration order: // accountDB, codeDB, storageDB, miscDB. metadataDB is excluded. func (s *CommitStore) dataDBs() []seidbtypes.KeyValueDB { @@ -909,3 +870,42 @@ func (s *CommitStore) reopenWAL() error { func (s *CommitStore) GetPhaseTimer() *metrics.PhaseTimer { return s.phaseTimer } + +// InitializeDataDirectories sets the DataDir for each nested PebbleDB config +// that does not already have one, using DataDir as the base path. The DBs live +// under the working directory: /working/. +func InitializeDataDirectories(c *config.Config) { + workDir := filepath.Join(c.DataDir, workingDirName) + if c.AccountDBConfig.DataDir == "" { + c.AccountDBConfig.DataDir = filepath.Join(workDir, accountDBDir) + } + if c.CodeDBConfig.DataDir == "" { + c.CodeDBConfig.DataDir = filepath.Join(workDir, codeDBDir) + } + if c.StorageDBConfig.DataDir == "" { + c.StorageDBConfig.DataDir = filepath.Join(workDir, storageDBDir) + } + if c.MiscDBConfig.DataDir == "" { + c.MiscDBConfig.DataDir = filepath.Join(workDir, miscDBDir) + } + if c.MetadataDBConfig.DataDir == "" { + c.MetadataDBConfig.DataDir = filepath.Join(workDir, metadataDir) + } + applyPebbleMetricsConfig(c) +} + +func applyPebbleMetricsConfig(c *config.Config) { + // Keep a single FlatKV-level knob for Pebble internal metrics. Per-DB + // EnableMetrics values are intentionally overwritten here. + c.AccountDBConfig.EnableMetrics = c.EnablePebbleMetrics + c.CodeDBConfig.EnableMetrics = c.EnablePebbleMetrics + c.StorageDBConfig.EnableMetrics = c.EnablePebbleMetrics + c.MiscDBConfig.EnableMetrics = c.EnablePebbleMetrics + c.MetadataDBConfig.EnableMetrics = c.EnablePebbleMetrics + + c.AccountDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics + c.CodeDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics + c.StorageDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics + c.MiscDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics + c.MetadataDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics +} diff --git a/sei-db/state_db/sc/flatkv/store_gc.go b/sei-db/state_db/sc/flatkv/store_gc.go new file mode 100644 index 0000000000..c9e8310e6b --- /dev/null +++ b/sei-db/state_db/sc/flatkv/store_gc.go @@ -0,0 +1,180 @@ +package flatkv + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/sei-protocol/sei-chain/sei-db/management/gc" +) + +// CommitStore joins the shared prune cycle as a snapshot store: it can restore only at a snapshot +// boundary, replaying the state WAL forward from there to reach any higher height. So the history it +// must hold to serve a rollback is not a block range but the newest snapshot at or below the target, +// which is what GetPruningBoundary reports and what holds the WAL back for it. +// +// Participation is conditional on config.ExternalPruning, which is what stands the store's own two +// pruners down — see ExternalPruning below. With it unset the store is still asked for its boundary, +// and still protects the WAL it replays from, but enforces its own retention by snapshot count. +// +// Precondition beyond the collector's own: the state WAL must be managed alongside this store. The +// snapshot alone only restores the exact height it was taken at; every height above it needs the WAL +// blocks that follow. Managed without the WAL, this store answers a boundary nothing acts on while the +// WAL is pruned on its own schedule — and under ExternalPruning tryTruncateWAL has stood down too, so +// nothing bounds that WAL at all. +// +// Two of these methods run on the collector's goroutine while the store commits on another, so both +// avoid the plain fields Commit mutates: GetLatestBlock takes the read lock, and the snapshot-directory +// methods read the filesystem, which is already the synchronization boundary between snapshot writes +// and everything that reads them. +var _ gc.PrunableStore = (*CommitStore)(nil) + +func (s *CommitStore) Name() string { + return "FlatKV" +} + +// ExternalPruning reports config.ExternalPruning, the same field pruneSnapshots and tryTruncateWAL +// consult to stand down. Reading one field from all three is what makes "the collector prunes this +// store" and "this store does not prune itself" a single fact: there is no combination of settings +// that turns both on, so the count-based pruner can never delete a snapshot the collector is holding. +// +// It is a construction-time value, so the answer is stable for the life of the store and safe to read +// from the collector's goroutine. +func (s *CommitStore) ExternalPruning() bool { + return s.config.ExternalPruning +} + +// PruneBelow deletes every snapshot below blockNumber, leaving the WAL alone — the collector prunes +// that directly as its own store. +// +// The boundary snapshot this store reported survives, because blockNumber is a minimum across stores +// and so never exceeds that boundary (see GetPruningBoundary). It is therefore always left with at +// least one snapshot to restore from. +// +// Deletion is not all-or-nothing: a snapshot that fails to delete is reported and the rest are still +// attempted, since each snapshot directory is independent and a single undeletable one must not strand +// the disk space of the others. Already-gone snapshots are not an error — the SnapshotKeepRecent +// pruner in WriteSnapshot deletes from the same set, and losing that race means the work is done. +func (s *CommitStore) PruneBelow(blockNumber uint64) error { + if blockNumber == 0 { + return nil + } + + dir := s.flatkvDir() + + // The active snapshot is what the next open resolves to, so it is never a candidate however deep the + // request. Nothing in the contract should produce such a request — a boundary is never above the + // snapshot it names — but the cost of the check is one readlink against making a wrong answer + // elsewhere unbootable here. + _, activeVersion, err := currentSnapshotDir(dir) + if err != nil { + return fmt.Errorf("resolve active snapshot before pruning: %w", err) + } + + var errs error + pruned := 0 + scanErr := traverseSnapshots(dir, true, func(version int64) (bool, error) { + if uint64(version) >= blockNumber { //nolint:gosec // snapshot versions are non-negative + return true, nil // ascending, so nothing further is a candidate + } + if version == activeVersion { + return false, nil + } + if err := atomicRemoveDir(filepath.Join(dir, snapshotName(version))); err != nil { + if !os.IsNotExist(err) { + errs = errors.Join(errs, fmt.Errorf("remove snapshot %d: %w", version, err)) + } + return false, nil + } + pruned++ + logger.Info("pruned snapshot below retention floor", "version", version, "floor", blockNumber) + return false, nil + }) + if scanErr != nil { + errs = errors.Join(errs, fmt.Errorf("scan snapshots: %w", scanErr)) + } + return errs +} + +// GetRetentionWindow reports 0: this store asks for no history of its own beyond the collector's shared +// rollback window, which is the contract for a snapshot store (see gc.PrunableStore). +// +// 0 is what keeps it in the shared minimum, and being in that minimum is how its snapshots are +// protected — it answers its oldest needed snapshot as a boundary and the WAL is held there with it. +// InfiniteRetentionWindow would do the opposite of what it reads like: a store with no cut line is +// never asked for a boundary, so the WAL would be pruned to its own cut line and the snapshots left +// with nothing to replay from. +// +// How deep this store can actually restore is a function of SnapshotInterval and SnapshotKeepRecent, +// not of anything declared here. Those two decide which snapshots exist; if they retain less than +// RollbackWindow of history, the window is not servable no matter what this returns. +func (s *CommitStore) GetRetentionWindow() int64 { + return 0 +} + +// GetPruningBoundary returns the newest snapshot at or below cutLine — the oldest point this store must +// keep to restore to cutLine, since restoring to anything above that snapshot replays the WAL forward +// from it. +// +// When every snapshot sits above cutLine it returns cutLine, per the contract: none of them can be +// dropped, so holding the other stores back to the oldest of them would buy nothing. Note what this +// means operationally — the store cannot in fact restore to cutLine in that case, only to its oldest +// snapshot. That is a snapshot-retention shortfall (SnapshotInterval × SnapshotKeepRecent shallower +// than RollbackWindow), and reporting it here as a boundary would not fix it, it would only stall +// every other store's pruning while the gap persisted. +// +// The initial empty snapshot at version 0 is not a boundary. It restores to no committed height, and +// reaching any height from it requires replaying the WAL from the very first block — which is exactly +// CannotServeRollback, both in meaning and in the value 0 would collide with. +// +// A failed directory scan is also reported as CannotServeRollback, abandoning the cycle. Not knowing +// which snapshots exist means not knowing which blocks are needed to replay from them, and the WAL is +// what would be pruned on the strength of a guess. +func (s *CommitStore) GetPruningBoundary(cutLine uint64) uint64 { + var newestAtOrBelow, newest int64 + err := traverseSnapshots(s.flatkvDir(), false, func(version int64) (bool, error) { + if version < 1 { + return false, nil + } + if newest == 0 { + newest = version // descending, so the first is the newest + } + if uint64(version) <= cutLine { //nolint:gosec // guarded >= 1 above + newestAtOrBelow = version + return true, nil + } + return false, nil + }) + if err != nil { + logger.Error("failed to scan snapshots for pruning boundary; blocking the prune cycle", + "cutLine", cutLine, "err", err) + return gc.CannotServeRollback + } + + switch { + case newestAtOrBelow > 0: + return uint64(newestAtOrBelow) //nolint:gosec // guarded > 0 + case newest > 0: + return cutLine + default: + return gc.CannotServeRollback + } +} + +// GetLatestBlock returns the highest committed version, or 0 when nothing has been committed. +// +// This is the committed version rather than the newest snapshot: it is the store's ingest position, +// which is what the collector takes a minimum over to find the fleet's head. The snapshot layout only +// enters through GetPruningBoundary. +// +// Takes the read lock because Commit advances this field under the write lock, and the collector reads +// it from its own goroutine. +func (s *CommitStore) GetLatestBlock() (uint64, error) { + s.mu.RLock() + defer s.mu.RUnlock() + if s.committedVersion <= 0 { + return 0, nil + } + return uint64(s.committedVersion), nil //nolint:gosec // guarded positive above +} diff --git a/sei-db/state_db/sc/flatkv/store_gc_test.go b/sei-db/state_db/sc/flatkv/store_gc_test.go new file mode 100644 index 0000000000..b233d4eccb --- /dev/null +++ b/sei-db/state_db/sc/flatkv/store_gc_test.go @@ -0,0 +1,327 @@ +package flatkv + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/management/gc" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" +) + +// The GC surface reads the snapshot directory and two plain fields, so most of it can be exercised +// without opening the five PebbleDBs a real store carries. Only the tests that assert against +// snapshots WriteSnapshot produced, or against a concurrent committer, pay for a live store. +func gcStore(t *testing.T, dir string) (*CommitStore, gc.PrunableStore) { + t.Helper() + s := &CommitStore{config: config.Config{DataDir: dir}} + return s, s +} + +func mkSnapshots(t *testing.T, dir string, versions ...int64) { + t.Helper() + for _, v := range versions { + require.NoError(t, os.MkdirAll(filepath.Join(dir, snapshotName(v)), 0750)) + } +} + +func snapshotVersions(t *testing.T, dir string) []int64 { + t.Helper() + var found []int64 + require.NoError(t, traverseSnapshots(dir, true, func(v int64) (bool, error) { + found = append(found, v) + return false, nil + })) + return found +} + +// A snapshot store asks for no history of its own: 0 is what keeps it inside the collector's shared +// minimum, which is what protects its snapshots. See GetRetentionWindow for why the sentinel that +// reads like "keep more" would do the opposite here. +func TestGCRetentionWindowIsZero(t *testing.T) { + _, store := gcStore(t, t.TempDir()) + require.Equal(t, int64(0), store.GetRetentionWindow()) +} + +// The head is the committed version, not the newest snapshot: it is this store's ingest position, and +// the collector takes a minimum over those to find the fleet's head. +func TestGCLatestBlockIsCommittedVersion(t *testing.T) { + s, store := gcStore(t, t.TempDir()) + + latest, err := store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(0), latest, "nothing committed yet") + + // Snapshots do not move the head; commits do. + mkSnapshots(t, s.flatkvDir(), 10, 20) + latest, err = store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(0), latest) + + s.committedVersion = 42 + latest, err = store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(42), latest) +} + +// Restoring to a height means starting at the newest snapshot at or below it and replaying the WAL +// forward, so that snapshot is the oldest thing this store must keep. +func TestGCPruningBoundaryIsNewestSnapshotAtOrBelowCutLine(t *testing.T) { + s, store := gcStore(t, t.TempDir()) + mkSnapshots(t, s.flatkvDir(), 5, 10, 20) + + for _, tc := range []struct { + cutLine uint64 + want uint64 + why string + }{ + {cutLine: 15, want: 10, why: "newest at or below"}, + {cutLine: 20, want: 20, why: "a snapshot exactly on the cut line serves it"}, + {cutLine: 100, want: 20, why: "far above every snapshot"}, + {cutLine: 5, want: 5, why: "the oldest snapshot exactly on the cut line"}, + } { + require.Equal(t, tc.want, store.GetPruningBoundary(tc.cutLine), tc.why) + } +} + +// The answer must never exceed the cut line: the collector takes a minimum across stores without +// clamping, so a higher answer here would raise pruneHeight above head-RollbackWindow and prune away +// the rollback window every other store is holding. +func TestGCPruningBoundaryNeverExceedsCutLine(t *testing.T) { + s, store := gcStore(t, t.TempDir()) + mkSnapshots(t, s.flatkvDir(), 7, 13, 29) + + for cutLine := uint64(1); cutLine <= 40; cutLine++ { + require.LessOrEqual(t, store.GetPruningBoundary(cutLine), cutLine, + "boundary above cut line at cutLine=%d", cutLine) + } +} + +// Every snapshot above the cut line means none can be dropped, so the store answers the cut line +// rather than its oldest snapshot — holding the fleet back to that snapshot would buy nothing. The +// store genuinely cannot restore to the cut line here; that is a snapshot-retention shortfall, not +// something a lower answer would repair. +func TestGCPruningBoundaryAllSnapshotsAboveCutLine(t *testing.T) { + s, store := gcStore(t, t.TempDir()) + mkSnapshots(t, s.flatkvDir(), 500, 900) + + require.Equal(t, uint64(100), store.GetPruningBoundary(100)) +} + +// With no snapshot to restore from, the store stops the cycle rather than dropping out of it: it will +// replay forward once its first snapshot lands, and the WAL holds the range it will replay from. +func TestGCPruningBoundaryWithoutSnapshots(t *testing.T) { + s, store := gcStore(t, t.TempDir()) + require.Equal(t, gc.CannotServeRollback, store.GetPruningBoundary(100)) + + // A directory that does not exist yet reads the same way. + require.NoError(t, os.RemoveAll(s.flatkvDir())) + require.Equal(t, gc.CannotServeRollback, store.GetPruningBoundary(100)) +} + +// The initial empty snapshot restores to no committed height, so reaching any height from it needs the +// WAL from its very first block — which is what CannotServeRollback already means, and is the value 0 +// would collide with anyway. +func TestGCPruningBoundaryIgnoresInitialSnapshot(t *testing.T) { + s, store := gcStore(t, t.TempDir()) + mkSnapshots(t, s.flatkvDir(), 0) + + require.Equal(t, gc.CannotServeRollback, store.GetPruningBoundary(100)) + + // With a real snapshot present, version 0 still contributes nothing. + mkSnapshots(t, s.flatkvDir(), 30) + require.Equal(t, uint64(30), store.GetPruningBoundary(100)) + require.Equal(t, uint64(10), store.GetPruningBoundary(10), "not 0, and not the initial snapshot") +} + +// Not knowing which snapshots exist means not knowing which blocks are needed to replay from them, and +// the WAL would be pruned on the strength of that answer. So a failed scan blocks the cycle. +func TestGCPruningBoundaryScanFailureBlocksTheCycle(t *testing.T) { + dir := filepath.Join(t.TempDir(), "flatkv") + require.NoError(t, os.WriteFile(dir, []byte("not a directory"), 0600)) + + _, store := gcStore(t, dir) + require.Equal(t, gc.CannotServeRollback, store.GetPruningBoundary(100)) +} + +// PruneBelow drops every snapshot under the floor. The floor is a minimum across stores and so never +// exceeds the boundary this store reported, which is why the snapshot it named always survives. +func TestGCPruneBelowDeletesSnapshotsBelowFloor(t *testing.T) { + s, store := gcStore(t, t.TempDir()) + dir := s.flatkvDir() + mkSnapshots(t, dir, 0, 5, 10, 20, 30) + require.NoError(t, updateCurrentSymlink(dir, snapshotName(30))) + + require.NoError(t, store.PruneBelow(20)) + require.Equal(t, []int64{20, 30}, snapshotVersions(t, dir)) + + // Idempotent: the same floor twice deletes nothing more. + require.NoError(t, store.PruneBelow(20)) + require.Equal(t, []int64{20, 30}, snapshotVersions(t, dir)) +} + +// A floor of 0 is the collector's "nothing to do" and must not be read as "delete everything below +// any height". +func TestGCPruneBelowZeroIsNoOp(t *testing.T) { + s, store := gcStore(t, t.TempDir()) + dir := s.flatkvDir() + mkSnapshots(t, dir, 5, 10) + require.NoError(t, updateCurrentSymlink(dir, snapshotName(10))) + + require.NoError(t, store.PruneBelow(0)) + require.Equal(t, []int64{5, 10}, snapshotVersions(t, dir)) +} + +// The active snapshot is what the next open resolves to, so it survives however deep the request. The +// contract should never produce such a request; this pins that a wrong answer elsewhere cannot leave +// the store unbootable here. +func TestGCPruneBelowKeepsActiveSnapshot(t *testing.T) { + s, store := gcStore(t, t.TempDir()) + dir := s.flatkvDir() + mkSnapshots(t, dir, 5, 10) + require.NoError(t, updateCurrentSymlink(dir, snapshotName(5))) + + require.NoError(t, store.PruneBelow(1_000)) + require.Equal(t, []int64{5}, snapshotVersions(t, dir)) +} + +// Without a resolvable active snapshot there is nothing to protect the deletion against, so the prune +// is refused rather than run blind. +func TestGCPruneBelowRefusesWithoutActiveSnapshot(t *testing.T) { + s, store := gcStore(t, t.TempDir()) + dir := s.flatkvDir() + mkSnapshots(t, dir, 5, 10) + + require.Error(t, store.PruneBelow(10)) + require.Equal(t, []int64{5, 10}, snapshotVersions(t, dir)) +} + +// ExternalPruning is off unless asked for: a store built without a collector must keep pruning +// itself, since standing down with nothing to replace it grows snapshots without bound. +func TestGCExternalPruningDefaultsOff(t *testing.T) { + _, store := gcStore(t, t.TempDir()) + require.False(t, store.ExternalPruning()) + + s := &CommitStore{config: config.Config{DataDir: t.TempDir(), ExternalPruning: true}} + require.True(t, s.ExternalPruning()) +} + +// The guarantee the flag exists for: the count-based pruner and the collector are never both +// enforcing retention, because both read this one field. SnapshotKeepRecent 0 is the most +// destructive setting there is — delete every snapshot but the current one — so if anything can +// defeat the stand-down, it shows here. +func TestGCExternalPruningStandsDownSnapshotPruner(t *testing.T) { + run := func(external bool) []int64 { + dir := t.TempDir() + s := &CommitStore{ + ctx: t.Context(), + config: config.Config{ + DataDir: dir, + SnapshotKeepRecent: 0, + ExternalPruning: external, + }, + } + mkSnapshots(t, dir, 5, 10, 15) + s.pruneSnapshots(dir, 15) + return snapshotVersions(t, dir) + } + + require.Equal(t, []int64{15}, run(false), + "left to itself the count-based pruner takes everything below the current snapshot") + require.Equal(t, []int64{5, 10, 15}, run(true), + "under the collector it must not delete the snapshot being held for the rollback window") +} + +// End to end against snapshots WriteSnapshot actually produced, including that the store still opens +// afterwards — the prune must not disturb what the next open resolves through. +func TestGCPrunesRealSnapshotsAndStoreStillOpens(t *testing.T) { + dir := t.TempDir() + cfg := config.DefaultTestConfig(t) + cfg.DataDir = filepath.Join(dir, flatkvRootDir) + cfg.SnapshotInterval = 2 + // Configured the way a collector-managed store actually is. SnapshotKeepRecent is deliberately + // left at its default of 1 to show ExternalPruning overrides it rather than cooperating with it: + // under that count, snapshots 0 and 2 would be gone before the collector ever saw them. + cfg.ExternalPruning = true + + s, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + require.NoError(t, s.LoadLatest()) + + for range 6 { + commitAndCheck(t, s) + } + // A fresh store starts at the initial snapshot; the interval adds one every other block. + require.Equal(t, []int64{0, 2, 4, 6}, snapshotVersions(t, cfg.DataDir)) + + store, ok := any(s).(gc.PrunableStore) + require.True(t, ok, "a FlatKV commit store must satisfy gc.PrunableStore") + + latest, err := store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(6), latest) + + boundary := store.GetPruningBoundary(5) + require.Equal(t, uint64(4), boundary, "newest snapshot at or below the cut line") + + require.NoError(t, store.PruneBelow(boundary)) + require.Equal(t, []int64{4, 6}, snapshotVersions(t, cfg.DataDir)) + + require.NoError(t, s.Close()) + + cfg2 := config.DefaultTestConfig(t) + cfg2.DataDir = cfg.DataDir + s2, err := newCommitStoreWithWAL(t.Context(), cfg2) + require.NoError(t, err) + require.NoError(t, s2.LoadLatest()) + require.Equal(t, int64(6), s2.Version(), "the pruned store reopens at its committed version") + require.NoError(t, s2.Close()) +} + +// The collector runs on its own goroutine while the store commits on another. Only the race detector +// can judge this: CommitStore keeps its committed version in a plain field that Commit advances under +// the write lock, and WriteSnapshot rewrites the snapshot directory the other two methods scan. +func TestGCConcurrentWithCommitter(t *testing.T) { + dir := t.TempDir() + cfg := config.DefaultTestConfig(t) + cfg.DataDir = filepath.Join(dir, flatkvRootDir) + cfg.SnapshotInterval = 5 + cfg.SnapshotKeepRecent = 1_000 // the collector is the only deleter here + + s, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + require.NoError(t, s.LoadLatest()) + defer func() { require.NoError(t, s.Close()) }() + + store := gc.PrunableStore(s) + + const blocks = 30 + done := make(chan struct{}) + go func() { + defer close(done) + for version := int64(1); version <= blocks; version++ { + // Assertions are not allowed off the main test goroutine; fail loudly instead. + if _, err := s.Commit(version); err != nil { + panic(err) + } + } + }() + + // Drive prune cycles exactly as the collector does, off the committer's goroutine. + for range blocks { + head, err := store.GetLatestBlock() + require.NoError(t, err) + require.LessOrEqual(t, head, uint64(blocks)) + if head <= 10 { + continue + } + boundary := store.GetPruningBoundary(head - 10) + require.LessOrEqual(t, boundary, head-10, "boundary must never exceed the cut line") + if boundary != gc.CannotServeRollback { + require.NoError(t, store.PruneBelow(boundary)) + } + } + <-done +} diff --git a/sei-db/state_db/statewal/state_wal_brick_test.go b/sei-db/state_db/statewal/state_wal_brick_test.go index a047a2d78f..9282c8f6b3 100644 --- a/sei-db/state_db/statewal/state_wal_brick_test.go +++ b/sei-db/state_db/statewal/state_wal_brick_test.go @@ -65,7 +65,7 @@ func (f *fakeWAL) Close() error { func newFakeStateWAL(t *testing.T, f *fakeWAL) StateWAL { t.Helper() - w, err := newStateWAL(f) + w, err := newStateWAL(f, 0) require.NoError(t, err) return w } diff --git a/sei-db/state_db/statewal/state_wal_config.go b/sei-db/state_db/statewal/state_wal_config.go index 6aa7eec9bc..334a63b2d7 100644 --- a/sei-db/state_db/statewal/state_wal_config.go +++ b/sei-db/state_db/statewal/state_wal_config.go @@ -1,8 +1,10 @@ package statewal import ( + "fmt" "time" + "github.com/sei-protocol/sei-chain/sei-db/management/gc" "github.com/sei-protocol/sei-chain/sei-db/seiwal" ) @@ -39,9 +41,38 @@ type Config struct { // The interval at which the underlying WAL samples the buffered depth of its internal channels into the // seiwal_queue_depth gauge. Zero or negative disables sampling. MetricsSampleInterval time.Duration + + // RetentionWindow is how much history this WAL keeps beyond the shared rollback window of the + // StorageGarbageCollector that manages it, in blocks. It is what gc.PrunableStore.GetRetentionWindow + // answers: + // + // > 0 → that many blocks of history beyond the rollback window + // 0 → the rollback window only, plus whatever the other managed stores hold it back for + // -1 → never prune this WAL (gc.InfiniteRetentionWindow) + // + // Zero does NOT mean "keep everything" here, unlike the KeepRecent fields on StateStoreConfig and + // ReceiptStoreConfig, where 0 disables pruning. It is the most aggressive setting this field has; + // "keep everything" is -1. Assigning a KeepRecent value to this field inverts the retention it + // asks for. + // + // Leave this at 0 unless something outside SC/SS needs the history. The WAL is what SC and SS replay + // from, and they already hold it back on their own: each answers its oldest live snapshot as its + // pruning boundary, and the collector prunes every store to the shared minimum. Depth declared here is + // additive on top of that, and — because the minimum is shared — it retains the whole fleet that much + // further back, not just this WAL. Its purpose is a need the snapshot stores do not express, such as + // serving catch-up to a peer further behind than any live snapshot. + // + // -1 is for a WAL that must never be reclaimed at all. It is not a way to protect a replay range: a + // store with no cut line is never asked for a boundary and never pruned, so the WAL simply grows + // without bound. Must be >= gc.InfiniteRetentionWindow. + RetentionWindow int64 } // Constructor for a default state WAL configuration for the WAL at path, identified by name. +// +// RetentionWindow defaults to 0 — no history beyond what SC and SS hold the WAL back for — because that is +// the depth the WAL is actually required to have, and any other default would retain every managed store +// that much further back. See the field for when to raise it. func DefaultConfig(path string, name string) *Config { s := seiwal.DefaultConfig(path, name) return &Config{ @@ -53,11 +84,16 @@ func DefaultConfig(path string, name string) *Config { FsyncOnFlush: s.FsyncOnFlush, IteratorPrefetchSize: s.IteratorPrefetchSize, MetricsSampleInterval: s.MetricsSampleInterval, + RetentionWindow: 0, } } // Validate the configuration, returning nil if valid, or an error describing the problem if invalid. func (c *Config) Validate() error { + if c.RetentionWindow < gc.InfiniteRetentionWindow { + return fmt.Errorf("RetentionWindow must be >= %d (got %d)", + gc.InfiniteRetentionWindow, c.RetentionWindow) + } return c.toSeiwalConfig().Validate() } diff --git a/sei-db/state_db/statewal/state_wal_gc.go b/sei-db/state_db/statewal/state_wal_gc.go new file mode 100644 index 0000000000..15a6ab13f5 --- /dev/null +++ b/sei-db/state_db/statewal/state_wal_gc.go @@ -0,0 +1,81 @@ +package statewal + +import ( + "fmt" + + "github.com/sei-protocol/sei-chain/sei-db/management/gc" +) + +// stateWALImpl joins the shared prune cycle as a contiguous store: it holds every block from its +// retention floor up to its head, so any height in between can serve a rollback. +// +// This is the only surface of stateWALImpl that may be used from another goroutine (see the type +// doc). Every method here reads a constant, a single atomic, or the WAL underneath — which permits a +// concurrent PruneBefore by contract. None of them touch the plain fields the writer owns. +// +// Precondition beyond the collector's own: SC and SS must be managed alongside this WAL. The WAL is +// what they replay from, and it is their boundaries — the oldest snapshot each still needs — that +// hold it back, since the collector prunes every store to the shared minimum. Managed without them, +// the WAL is pruned to its own cut line and the replay range they depend on goes with it. +var _ gc.PrunableStore = (*stateWALImpl)(nil) + +func (w *stateWALImpl) Name() string { + return "StateWAL" +} + +// ExternalPruning is unconditionally true: the WAL prunes only when told to, by Prune or PruneBelow. +// +// Note that its owner may still prune it directly — FlatKV does, in tryTruncateWAL — which this +// cannot report on, since it is a property of the caller rather than of the WAL. FlatKV stands that +// down under its own ExternalPruning for the same reason this exists. +func (w *stateWALImpl) ExternalPruning() bool { + return true +} + +// PruneBelow schedules removal of the blocks below blockNumber, going straight to the underlying WAL +// from the collector's goroutine. Prune is the equivalent for the WAL's own owner; this exists +// separately only because it must not read the closed/fatalErr bookkeeping that Prune does. +// +// Deliberately does not brick the WAL on failure, unlike every writer-goroutine path here. Bricking +// means writing fatalErr, which the writer reads unsynchronized, so doing it from this goroutine would +// be a data race. Nothing is lost by declining: a prune fails only when the WAL is already closed or +// dead underneath, and the writer's next operation discovers that on its own. +func (w *stateWALImpl) PruneBelow(blockNumber uint64) error { + if err := w.wal.PruneBefore(blockNumber); err != nil { + return fmt.Errorf("failed to prune state WAL below block %d: %w", blockNumber, err) + } + return nil +} + +// GetRetentionWindow reports the configured history beyond the collector's shared rollback window. +// See Config.RetentionWindow for the meaning of each value and why its default of 0 is the depth this +// WAL actually needs: SC and SS hold it back to their oldest live snapshot on their own, by answering +// that snapshot as their pruning boundary, so history declared here is additive on top of that and +// applies to every managed store rather than to this WAL alone. +func (w *stateWALImpl) GetRetentionWindow() int64 { + if w.retentionWindow < 0 { + return gc.InfiniteRetentionWindow + } + return w.retentionWindow +} + +// GetPruningBoundary returns cutLine, the contract's answer for a contiguous store: the WAL holds +// every block from its floor to its head, so cutLine itself is replayable and nothing below it has +// to be held back. +// +// Unconditional on purpose. A WAL whose floor already sits above cutLine — pruned there by an +// earlier cycle, or freshly created above it — also answers cutLine, because the PruneBelow that +// follows is a no-op on it while a lower answer would hold every other store back to this WAL's +// floor. CannotServeRollback is never right here: it is the replay source, not a replay consumer. +func (w *stateWALImpl) GetPruningBoundary(cutLine uint64) uint64 { + return cutLine +} + +// GetLatestBlock returns the highest block ended by SignalEndOfBlock, or 0 when none has been. +// +// A block that has been written but not yet ended is deliberately excluded: it is still buffered, +// not a record, and reporting it would put the collector's head one block above what the WAL can +// actually replay. See stateWALImpl.lastCompletedBlock for the 0 case. +func (w *stateWALImpl) GetLatestBlock() (uint64, error) { + return w.lastCompletedBlock.Load(), nil +} diff --git a/sei-db/state_db/statewal/state_wal_gc_test.go b/sei-db/state_db/statewal/state_wal_gc_test.go new file mode 100644 index 0000000000..fe491296ec --- /dev/null +++ b/sei-db/state_db/statewal/state_wal_gc_test.go @@ -0,0 +1,256 @@ +package statewal + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/management/gc" + "github.com/sei-protocol/sei-chain/sei-db/proto" +) + +// openWALForGC opens a WAL and returns it as the collector sees it, closing it on cleanup. +func openWALForGC(t *testing.T, cfg *Config) (StateWAL, gc.PrunableStore) { + t.Helper() + w := openWAL(t, cfg) + t.Cleanup(func() { require.NoError(t, w.Close()) }) + store, ok := w.(gc.PrunableStore) + require.True(t, ok, "a state WAL must satisfy gc.PrunableStore") + return w, store +} + +// The head is the last block ended by SignalEndOfBlock. A block that has been written but not ended +// is still buffered rather than a record, so counting it would put the collector's head one block +// above what the WAL can replay. +func TestGCLatestBlockCountsOnlyCompletedBlocks(t *testing.T) { + w, store := openWALForGC(t, testConfig(t.TempDir())) + + latest, err := store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(0), latest, "an empty WAL has no head") + + writeBlock(t, w, 1) + writeBlock(t, w, 2) + latest, err = store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(2), latest) + + // Block 3 is written but not ended, so it is not yet in the WAL. + require.NoError(t, w.Write(3, []*proto.NamedChangeSet{makeChangeSet("evm", []byte{3}, []byte{3})})) + latest, err = store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(2), latest, "a block in progress must not count as the head") + + require.NoError(t, w.SignalEndOfBlock()) + latest, err = store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(3), latest) +} + +// The head survives a reopen: a WAL that reported 0 after a restart would drop out of the +// collector's head minimum and let the other stores prune past blocks it still holds. +func TestGCLatestBlockRecoveredOnOpen(t *testing.T) { + cfg := testConfig(t.TempDir()) + + w := openWAL(t, cfg) + for block := uint64(1); block <= 5; block++ { + writeBlock(t, w, block) + } + require.NoError(t, w.Flush()) + require.NoError(t, w.Close()) + + _, store := openWALForGC(t, cfg) + latest, err := store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(5), latest) +} + +// The window is the configured one, sentinel included: -1 has to survive the trip verbatim or an +// operator asking for "never prune" gets a 1-block window instead. The default is 0, which asks for +// no history of its own — what holds the WAL back then is SC/SS answering their oldest live snapshot, +// which the collector applies to every store as the shared minimum. +func TestGCRetentionWindowComesFromConfig(t *testing.T) { + _, store := openWALForGC(t, testConfig(t.TempDir())) + require.Equal(t, int64(0), store.GetRetentionWindow(), "the default asks for no history of its own") + + for _, retentionWindow := range []int64{gc.InfiniteRetentionWindow, 100_000} { + cfg := testConfig(t.TempDir()) + cfg.RetentionWindow = retentionWindow + _, configured := openWALForGC(t, cfg) + require.Equal(t, retentionWindow, configured.GetRetentionWindow()) + } +} + +func TestConfigValidateRetentionWindow(t *testing.T) { + cfg := testConfig(t.TempDir()) + + for _, retentionWindow := range []int64{gc.InfiniteRetentionWindow, 0, 1} { + cfg.RetentionWindow = retentionWindow + require.NoError(t, cfg.Validate()) + } + + // Below InfiniteRetentionWindow there is no meaning left to assign, and GetRetentionWindow reads any + // negative value as infinite retention — so a typo'd -2 would silently disable pruning. + cfg.RetentionWindow = -2 + require.ErrorContains(t, cfg.Validate(), "RetentionWindow") +} + +// A contiguous store answers the cut line it was given whatever it holds, including on an empty WAL +// and above its own head. +func TestGCPruningBoundaryIsAlwaysCutLine(t *testing.T) { + w, store := openWALForGC(t, testConfig(t.TempDir())) + + require.Equal(t, uint64(42), store.GetPruningBoundary(42)) + + for block := uint64(1); block <= 5; block++ { + writeBlock(t, w, block) + } + require.Equal(t, uint64(3), store.GetPruningBoundary(3)) + require.Equal(t, uint64(1_000), store.GetPruningBoundary(1_000)) +} + +// PruneBelow goes straight to the WAL, so reclamation does not wait on write traffic. A WAL that has +// stopped receiving blocks is exactly where a deferred prune would strand history indefinitely, so no +// further block is written here before the result is checked. +func TestGCPruneBelowDoesNotWaitForTheNextBlock(t *testing.T) { + cfg := testConfig(t.TempDir()) + cfg.TargetFileSize = 1 // seal after every block, so whole-file pruning can act per block + w, store := openWALForGC(t, cfg) + + for block := uint64(1); block <= 10; block++ { + writeBlock(t, w, block) + } + require.NoError(t, w.Flush()) + + require.NoError(t, store.PruneBelow(6)) + require.NoError(t, w.Flush()) // the prune is async; order behind it + + ok, first, last, err := w.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(6), first) + require.Equal(t, uint64(10), last) + + require.Equal(t, []uint64{6, 7, 8, 9, 10}, collectBlocks(t, w, 6, 10)) +} + +// Cycles are independent, so a floor lower than one already applied must not walk back the prune it +// performed. Nothing here enforces that — a prune only ever deletes — but the collector is free to +// issue a lower floor after a rollback, and this pins that doing so is harmless rather than a way to +// resurrect blocks or corrupt the range. +func TestGCPruneBelowIgnoresALowerFloor(t *testing.T) { + cfg := testConfig(t.TempDir()) + cfg.TargetFileSize = 1 + w, store := openWALForGC(t, cfg) + + for block := uint64(1); block <= 10; block++ { + writeBlock(t, w, block) + } + require.NoError(t, w.Flush()) + + require.NoError(t, store.PruneBelow(8)) + require.NoError(t, store.PruneBelow(3)) + require.NoError(t, w.Flush()) + + ok, first, _, err := w.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(8), first, "a later, lower floor must not undo the higher one") +} + +// The collector runs on its own goroutine while the WAL's owner writes blocks on another, which is +// the whole reason this surface is separate from the writer-facing one. Only the race detector can +// judge it: stateWALImpl keeps its state in plain fields on the assumption of a single caller, and +// what makes the prune safe to issue from here is seiwal.WAL's concurrency carve-out for PruneBefore. +func TestGCConcurrentWithWriter(t *testing.T) { + cfg := testConfig(t.TempDir()) + cfg.TargetFileSize = 1 + w, store := openWALForGC(t, cfg) + + const blocks = 60 + done := make(chan struct{}) + go func() { + defer close(done) + for block := uint64(1); block <= blocks; block++ { + // Assertions are not allowed off the main test goroutine; fail loudly instead. + cs := []*proto.NamedChangeSet{makeChangeSet("evm", []byte{byte(block)}, []byte{byte(block)})} + if err := w.Write(block, cs); err != nil { + panic(err) + } + if err := w.SignalEndOfBlock(); err != nil { + panic(err) + } + } + }() + + // Drive prune cycles exactly as the collector does, off the writer's goroutine. + for range blocks { + head, err := store.GetLatestBlock() + require.NoError(t, err) + require.LessOrEqual(t, head, uint64(blocks)) + if head > 20 { + require.NoError(t, store.PruneBelow(store.GetPruningBoundary(head-20))) + } + } + <-done + + ok, first, last, err := w.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(blocks), last) + require.LessOrEqual(t, first, uint64(blocks-20), "pruning must never outrun the floor it was given") +} + +// A prune issued before a close is ordered ahead of it rather than dropped, and survives into the next +// session. What must not happen is the close failing, or the WAL bricking, because of it. +func TestGCPruneBelowBeforeClose(t *testing.T) { + cfg := testConfig(t.TempDir()) + cfg.TargetFileSize = 1 + + w := openWAL(t, cfg) + for block := uint64(1); block <= 5; block++ { + writeBlock(t, w, block) + } + require.NoError(t, w.(gc.PrunableStore).PruneBelow(4)) + require.NoError(t, w.Close()) + + w2, store := openWALForGC(t, cfg) + ok, first, last, err := w2.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(4), first) + require.Equal(t, uint64(5), last) + + latest, err := store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(5), latest) +} + +// The collector can still be mid-cycle when the node shuts down, so a prune arriving after the close +// must be reported rather than panic, and must not brick the WAL on the way out — PruneBelow declines +// to write fatalErr precisely because the writer reads it unsynchronized. +func TestGCPruneBelowAfterClose(t *testing.T) { + cfg := testConfig(t.TempDir()) + cfg.TargetFileSize = 1 + + w := openWAL(t, cfg) + for block := uint64(1); block <= 5; block++ { + writeBlock(t, w, block) + } + store, ok := w.(gc.PrunableStore) + require.True(t, ok) + require.NoError(t, w.Close()) + + require.Error(t, store.PruneBelow(4)) + + // The head is still readable, and the close committed what was written. + latest, err := store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(5), latest) + + w2, _ := openWALForGC(t, cfg) + _, first, last, err := w2.GetStoredRange() + require.NoError(t, err) + require.Equal(t, uint64(1), first) + require.Equal(t, uint64(5), last) +} diff --git a/sei-db/state_db/statewal/state_wal_impl.go b/sei-db/state_db/statewal/state_wal_impl.go index 5dbc9e7de1..23fe2e604a 100644 --- a/sei-db/state_db/statewal/state_wal_impl.go +++ b/sei-db/state_db/statewal/state_wal_impl.go @@ -3,6 +3,7 @@ package statewal import ( "errors" "fmt" + "sync/atomic" "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/seiwal" @@ -12,11 +13,20 @@ var _ StateWAL = (*stateWALImpl)(nil) // A WAL for storing state changesets by block number. // -// Not safe for concurrent use; see the StateWAL interface doc. +// Not safe for concurrent use; see the StateWAL interface doc. The gc.PrunableStore surface in +// state_wal_gc.go is the one exception, and it is why lastCompletedBlock below is atomic: the +// collector runs on its own goroutine, so it reads a head one atomic wide and otherwise touches +// none of the plain fields here. It prunes by calling straight through to the WAL underneath, +// which permits a concurrent PruneBefore (see seiwal.WAL). type stateWALImpl struct { // The underlying generic WAL, keyed by block number, whose payload is a block's changesets. wal seiwal.WAL[[]*proto.NamedChangeSet] + // Config.RetentionWindow, the history this WAL asks the garbage collector to keep beyond the shared + // rollback window. Immutable after construction, so the collector reads it off-goroutine without + // synchronization (GetRetentionWindow). + retentionWindow int64 + // Set by Close() so subsequent calls fail fast. A plain field: like the write-ordering state below, it // is only ever touched by the single caller, which must not invoke methods concurrently. closed bool @@ -39,6 +49,15 @@ type stateWALImpl struct { // end-of-block. Ownership is handed to the WAL at end-of-block and a fresh buffer starts for the next // block, so the serialization goroutine never races the wrapper over the backing array. buf []*proto.NamedChangeSet + + // The highest block that has been ended by SignalEndOfBlock, and so is actually a record in the WAL. + // Distinct from currentBlock, which may name a block still accumulating in buf. Atomic because the + // garbage collector reads it off-goroutine (GetLatestBlock); the writer is its only mutator. + // + // 0 doubles as "no block completed yet", which is what GetLatestBlock reports. A WAL whose only + // completed block is block 0 is indistinguishable from an empty one, and that is the safe direction: + // it drops out of the collector's head rather than pulling every store's cut line down to 0. + lastCompletedBlock atomic.Uint64 } // New opens (or creates) a state WAL in the configured directory, recovering any files left behind by a @@ -49,7 +68,7 @@ func New(config *Config) (StateWAL, error) { if err != nil { return nil, fmt.Errorf("failed to open state WAL: %w", err) } - return newStateWAL(wal) + return newStateWAL(wal, config.RetentionWindow) } // GetRange reports the range of block numbers stored in the state WAL directory configured by config, @@ -100,8 +119,8 @@ func VerifyIntegrity(config *Config) error { return nil } -func newStateWAL(wal seiwal.WAL[[]*proto.NamedChangeSet]) (StateWAL, error) { - w := &stateWALImpl{wal: wal} +func newStateWAL(wal seiwal.WAL[[]*proto.NamedChangeSet], retentionWindow int64) (StateWAL, error) { + w := &stateWALImpl{wal: wal, retentionWindow: retentionWindow} // Recover the write-ordering position from the highest block already on disk. ok, _, last, err := wal.Bounds() @@ -113,6 +132,7 @@ func newStateWAL(wal seiwal.WAL[[]*proto.NamedChangeSet]) (StateWAL, error) { w.currentBlock = last w.currentBlockEnded = true w.hasCurrentBlock = true + w.lastCompletedBlock.Store(last) } return w, nil } @@ -156,6 +176,7 @@ func (w *stateWALImpl) SignalEndOfBlock() error { return w.fail(fmt.Errorf("failed to append block %d: %w", w.currentBlock, err)) } w.currentBlockEnded = true + w.lastCompletedBlock.Store(w.currentBlock) w.buf = nil // hand ownership to the WAL; the next block starts a fresh buffer return nil } diff --git a/sei-tendermint/config/autobahn.go b/sei-tendermint/config/autobahn.go index c042a9afc1..27b9590191 100644 --- a/sei-tendermint/config/autobahn.go +++ b/sei-tendermint/config/autobahn.go @@ -125,13 +125,13 @@ func (c AutobahnBlockDBConfig) Validate() error { // LittBlockConfig returns littblock.DefaultConfig(dir) with this config's // optional overrides applied. Fsync is always forced on. -func (c AutobahnBlockDBConfig) LittBlockConfig(dir string) (littblock.LittBlockConfig, error) { +func (c AutobahnBlockDBConfig) LittBlockConfig(dir string) (littblock.BlockDBConfig, error) { if err := c.Validate(); err != nil { - return littblock.LittBlockConfig{}, err + return littblock.BlockDBConfig{}, err } cfg, err := littblock.DefaultConfig(dir) if err != nil { - return littblock.LittBlockConfig{}, fmt.Errorf("littblock.DefaultConfig: %w", err) + return littblock.BlockDBConfig{}, fmt.Errorf("littblock.DefaultConfig: %w", err) } if r, ok := c.Retention.Get(); ok { cfg.Retention = r.Duration() diff --git a/sei-tendermint/internal/p2p/giga_router_fullnode_test.go b/sei-tendermint/internal/p2p/giga_router_fullnode_test.go index 4c1c846b51..5c3ecb0b56 100644 --- a/sei-tendermint/internal/p2p/giga_router_fullnode_test.go +++ b/sei-tendermint/internal/p2p/giga_router_fullnode_test.go @@ -60,7 +60,7 @@ func TestGigaRouter_Fullnode(t *testing.T) { proxyApp := proxy.New(app) dir := t.TempDir() - // Same resolve path as config.AutobahnBlockDBConfig{}.LittBlockConfig + // Same resolve path as config.AutobahnBlockDBConfig{}.BlockDBConfig // (zero overrides); p2p can't import config (import cycle). littCfg, err := littblock.DefaultConfig(filepath.Join(dir, "blockdb")) require.NoError(t, err) diff --git a/sei-tendermint/internal/p2p/giga_router_validator_test.go b/sei-tendermint/internal/p2p/giga_router_validator_test.go index 90c641a4dc..f831e966b4 100644 --- a/sei-tendermint/internal/p2p/giga_router_validator_test.go +++ b/sei-tendermint/internal/p2p/giga_router_validator_test.go @@ -68,7 +68,7 @@ func TestGigaRouter_FinalizeBlocks(t *testing.T) { // In giga mode the CometBFT handshaker is skipped; the router's // runExecute calls InitChain itself on fresh start. dir := t.TempDir() - // Same resolve path as config.AutobahnBlockDBConfig{}.LittBlockConfig + // Same resolve path as config.AutobahnBlockDBConfig{}.BlockDBConfig // (zero overrides); p2p can't import config (import cycle). littCfg, err := littblock.DefaultConfig(filepath.Join(dir, "blockdb")) require.NoError(t, err, "littblock.DefaultConfig[%v]", i) @@ -237,7 +237,7 @@ func TestGigaRouter_EvmProxy(t *testing.T) { require.NoError(t, genDoc.ValidateAndComplete()) dir := t.TempDir() - // Same resolve path as config.AutobahnBlockDBConfig{}.LittBlockConfig + // Same resolve path as config.AutobahnBlockDBConfig{}.BlockDBConfig // (zero overrides); p2p can't import config (import cycle). littCfg, err := littblock.DefaultConfig(filepath.Join(dir, "blockdb")) require.NoError(t, err) From 6935ce769a5378db8165e1e7ec4760e9e3dbb16c Mon Sep 17 00:00:00 2001 From: YimingZang Date: Thu, 6 Aug 2026 10:43:11 -0700 Subject: [PATCH 02/11] Add ExternalPruning for littdb ReceiptStore --- sei-db/config/receipt_config.go | 18 +++++ sei-db/config/receipt_config_fuzz_test.go | 6 ++ sei-db/config/testdata/receipt_store.golden | 1 + sei-db/ledger_db/receipt/litt_receipt_gc.go | 16 +++- .../ledger_db/receipt/litt_receipt_gc_test.go | 76 +++++++++++++++++- .../litt_receipt_pruner_internal_test.go | 70 ++++++++++++++++ .../ledger_db/receipt/litt_receipt_store.go | 80 +++++++++++++++++-- sei-db/ledger_db/receipt/receipt_store.go | 7 ++ 8 files changed, 259 insertions(+), 15 deletions(-) create mode 100644 sei-db/ledger_db/receipt/litt_receipt_pruner_internal_test.go diff --git a/sei-db/config/receipt_config.go b/sei-db/config/receipt_config.go index 2a18f17d52..a2dd8c18f0 100644 --- a/sei-db/config/receipt_config.go +++ b/sei-db/config/receipt_config.go @@ -55,6 +55,24 @@ type ReceiptStoreConfig struct { // default to every 600 seconds PruneIntervalSeconds int `mapstructure:"prune-interval-seconds"` + // ExternalPruning hands retention to the StorageGarbageCollector: the littidx backend + // stops running its own KeepRecent pruner and answers gc.PrunableStore.ExternalPruning + // with this value, so the collector prunes it instead. + // + // Like KeepRecent this is not read from the receipt-store config. It is set by whatever + // constructs the collector, because it is only correct when this store is actually + // registered with a running one — nothing here can check that, and the failure is silent + // and unbounded: the retention floor simply stops advancing. + // + // It is one field rather than two so the KeepRecent pruner and the collector can never + // both be enforcing retention. They would disagree, and the local pruner would win by + // deleting the rollback headroom the collector exists to preserve. + // + // Only the littidx backend honors this. The pebbledb backend is not a gc.PrunableStore, + // so the collector would not prune it and setting this would leave it with no pruner at + // all; newReceiptBackend rejects that combination rather than growing without bound. + ExternalPruning bool `mapstructure:"-"` + // EnableReadWriteMetrics emits simple estimated read/write counters for Pebble-backed receipt storage. // defaults to false EnableReadWriteMetrics bool `mapstructure:"enable-read-write-metrics"` diff --git a/sei-db/config/receipt_config_fuzz_test.go b/sei-db/config/receipt_config_fuzz_test.go index 078cd3ad45..7f1b610e4b 100644 --- a/sei-db/config/receipt_config_fuzz_test.go +++ b/sei-db/config/receipt_config_fuzz_test.go @@ -202,5 +202,11 @@ func TestManifestNamesEveryField(t *testing.T) { // sitting in a config struct that configuration cannot address is exactly the kind of // thing a replacement manager would otherwise try to map a key onto. "KeepRecent", + // ExternalPruning is tagged mapstructure:"-" for a sharper reason than KeepRecent: it is + // only correct when this store is registered with a running StorageGarbageCollector, which + // is a property of how the process was wired and not something an operator can assert from + // app.toml. Exposing a key for it would let a node stand its pruner down with nothing to + // replace it, and the resulting unbounded growth is silent. + "ExternalPruning", ) } diff --git a/sei-db/config/testdata/receipt_store.golden b/sei-db/config/testdata/receipt_store.golden index fb4f0c5ebb..e6891d7a05 100644 --- a/sei-db/config/testdata/receipt_store.golden +++ b/sei-db/config/testdata/receipt_store.golden @@ -3,5 +3,6 @@ Backend = string("pebbledb") AsyncWriteBuffer = int(100) KeepRecent = int(0) PruneIntervalSeconds = int(600) +ExternalPruning = bool(false) EnableReadWriteMetrics = bool(false) LogFilterParallelism = int(16) diff --git a/sei-db/ledger_db/receipt/litt_receipt_gc.go b/sei-db/ledger_db/receipt/litt_receipt_gc.go index b4033b106c..cda5ac11bc 100644 --- a/sei-db/ledger_db/receipt/litt_receipt_gc.go +++ b/sei-db/ledger_db/receipt/litt_receipt_gc.go @@ -13,11 +13,19 @@ func (s *littReceiptStore) Name() string { return "ReceiptDB" } -// ExternalPruning is unconditionally true: the background pruner this store used to run was removed -// when it joined the collector, precisely because it pruned to latest-KeepRecent with no knowledge of -// the shared rollback window. Nothing here prunes on its own any more. +// ExternalPruning reports config.ExternalPruning, and runsLocalPruner asks this same method before +// starting the KeepRecent pruner. One read behind both is what makes "the collector prunes this +// store" and "this store does not prune itself" a single fact, so the local pruner can never race +// the collector to a shallower floor and delete the rollback headroom it is holding. +// +// It is not unconditionally true because this store still runs without a collector — a node with +// rs-backend = "littidx" and KeepRecent from min-retain-blocks depends on the local pruner, and +// standing that down with nothing to replace it grows the tag index without bound. +// +// It is a construction-time value, so the answer is stable for the life of the store and safe to +// read from the collector's goroutine. func (s *littReceiptStore) ExternalPruning() bool { - return true + return s.externalPruning } // PruneBelow advances the retention floor to blockNumber and drops the tag-index entries below diff --git a/sei-db/ledger_db/receipt/litt_receipt_gc_test.go b/sei-db/ledger_db/receipt/litt_receipt_gc_test.go index 957fa78dcd..246ab513be 100644 --- a/sei-db/ledger_db/receipt/litt_receipt_gc_test.go +++ b/sei-db/ledger_db/receipt/litt_receipt_gc_test.go @@ -2,6 +2,7 @@ package receipt_test import ( "testing" + "time" "github.com/ethereum/go-ethereum/common" storetypes "github.com/sei-protocol/sei-chain/sei-cosmos/store/types" @@ -13,8 +14,10 @@ import ( "github.com/stretchr/testify/require" ) -// setupLittIdxForGC opens a littidx store with the given KeepRecent and no background pruner, so -// the only thing that moves the retention floor is the collector call under test. +// setupLittIdxForGC opens a littidx store configured the way a collector-managed one actually is, +// so the only thing that moves the retention floor is the collector call under test. Note that this +// is the real mechanism rather than a test-only dodge: zeroing PruneIntervalSeconds would silence +// the local pruner just as well, and would not exercise the flag the collector depends on. func setupLittIdxForGC(t *testing.T, keepRecent int) (receipt.ReceiptStore, gc.PrunableStore, sdk.Context) { t.Helper() storeKey := storetypes.NewKVStoreKey("evm") @@ -25,7 +28,7 @@ func setupLittIdxForGC(t *testing.T, keepRecent int) (receipt.ReceiptStore, gc.P cfg.Backend = "littidx" cfg.DBDirectory = t.TempDir() cfg.KeepRecent = keepRecent - cfg.PruneIntervalSeconds = 0 + cfg.ExternalPruning = true store, err := receipt.NewReceiptStore(cfg, storeKey) require.NoError(t, err) @@ -36,6 +39,73 @@ func setupLittIdxForGC(t *testing.T, keepRecent int) (receipt.ReceiptStore, gc.P return store, prunable, ctx } +// ExternalPruning is off unless asked for: a store built without a collector must keep pruning +// itself, since standing down with nothing to replace it grows the tag index without bound. +func TestReceiptGCExternalPruningDefaultsOff(t *testing.T) { + storeKey := storetypes.NewKVStoreKey("evm") + cfg := dbconfig.DefaultReceiptStoreConfig() + cfg.Backend = "littidx" + cfg.DBDirectory = t.TempDir() + require.False(t, cfg.ExternalPruning, "the default must leave retention with the store") + + store, err := receipt.NewReceiptStore(cfg, storeKey) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + prunable, ok := store.(gc.PrunableStore) + require.True(t, ok) + require.False(t, prunable.ExternalPruning()) + + _, managed, _ := setupLittIdxForGC(t, 0) + require.True(t, managed.ExternalPruning()) +} + +// The pebble backend is not a gc.PrunableStore, so the collector would never prune it and honoring +// ExternalPruning would leave it with no pruner at all. Refused at startup rather than discovered +// later from a full disk. +func TestReceiptExternalPruningRejectedOnPebbleBackend(t *testing.T) { + cfg := dbconfig.DefaultReceiptStoreConfig() + cfg.DBDirectory = t.TempDir() + cfg.ExternalPruning = true + + _, err := receipt.NewReceiptStore(cfg, storetypes.NewKVStoreKey("evm")) + require.ErrorContains(t, err, "does not support external pruning") +} + +// The standalone shape, and the one a collector-shaped change is most likely to break: with +// ExternalPruning unset there is no collector, and the store's own pruner has to advance the +// retention floor or the tag index grows without bound. Nodes running rs-backend = "littidx" with +// KeepRecent from min-retain-blocks depend on exactly this. +// +// Pays a real wait because the pruner is on a jittered timer and there is no way to observe it +// otherwise; TestRunsLocalPruner covers the decision itself without waiting. +func TestReceiptLocalPrunerAdvancesFloorWithoutCollector(t *testing.T) { + storeKey := storetypes.NewKVStoreKey("evm") + tkey := storetypes.NewTransientStoreKey("evm_transient") + ctx := testutil.DefaultContext(storeKey, tkey).WithBlockHeight(1) + + cfg := dbconfig.DefaultReceiptStoreConfig() + cfg.Backend = "littidx" + cfg.DBDirectory = t.TempDir() + cfg.KeepRecent = 2 + cfg.PruneIntervalSeconds = 1 // the shortest cadence there is; the pruner jitters to 1-2s + + store, err := receipt.NewReceiptStore(cfg, storeKey) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + addr := common.HexToAddress("0xabcd") + topic := common.HexToHash("0x1111") + for block := uint64(1); block <= 5; block++ { + writeLitBlock(t, store, ctx, block, litReceipt(block, 0, addr, topic)) + } + + // KeepRecent 2 at head 5 keeps [4, 5], so the floor lands on 4. + require.Eventually(t, func() bool { + return store.EarliestVersion() == 4 + }, 6*time.Second, 25*time.Millisecond, "the local pruner must advance the floor with no collector present") +} + // KeepRecent and GetRetentionWindow disagree about 0, so the translation is the behavior worth // pinning: KeepRecent 0 means "keep everything" and is the default, while a literal 0 answer // means "keep only the shared rollback window". Returning the field verbatim would prune a store diff --git a/sei-db/ledger_db/receipt/litt_receipt_pruner_internal_test.go b/sei-db/ledger_db/receipt/litt_receipt_pruner_internal_test.go new file mode 100644 index 0000000000..e21161380b --- /dev/null +++ b/sei-db/ledger_db/receipt/litt_receipt_pruner_internal_test.go @@ -0,0 +1,70 @@ +package receipt + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// Which driver enforces retention is a four-way decision, and getting it wrong in either direction +// is a production bug rather than a test nicety: two pruners race to different floors, and none +// lets the tag index grow without bound. Enumerated here rather than observed through the jittered +// ticker so every combination is covered without waiting on any of them. +func TestRunsLocalPruner(t *testing.T) { + for _, tc := range []struct { + name string + externalPruning bool + keepRecent int64 + pruneInterval int64 + want bool + }{ + { + name: "standalone node prunes itself", + keepRecent: 100_000, + pruneInterval: 600, + want: true, + }, + { + name: "under the collector it stands down", + externalPruning: true, + keepRecent: 100_000, + pruneInterval: 600, + want: false, + }, + { + // KeepRecent 0 is the default and means keep everything, so there is nothing for a + // local pruner to do. GetRetentionWindow maps the same 0 to InfiniteRetentionWindow. + name: "keep everything", + keepRecent: 0, + pruneInterval: 600, + want: false, + }, + { + name: "no cadence configured", + keepRecent: 100_000, + pruneInterval: 0, + want: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + s := &littReceiptStore{ + externalPruning: tc.externalPruning, + keepRecent: tc.keepRecent, + pruneInterval: tc.pruneInterval, + } + require.Equal(t, tc.want, s.runsLocalPruner()) + }) + } +} + +// The invariant behind the table: with retention configured at all, the local pruner is exactly the +// negation of ExternalPruning. Both on means two pruners racing to different floors; both off means +// nothing advances the floor. Asserted against the method the collector calls, so it covers the two +// staying wired to each other and not merely today's values. +func TestRunsLocalPrunerIsTheNegationOfExternalPruning(t *testing.T) { + for _, external := range []bool{false, true} { + s := &littReceiptStore{externalPruning: external, keepRecent: 100_000, pruneInterval: 600} + require.Equal(t, !s.ExternalPruning(), s.runsLocalPruner(), + "exactly one of the collector and the local pruner may enforce retention") + } +} diff --git a/sei-db/ledger_db/receipt/litt_receipt_store.go b/sei-db/ledger_db/receipt/litt_receipt_store.go index 4dc2f34c14..c57c69b9db 100644 --- a/sei-db/ledger_db/receipt/litt_receipt_store.go +++ b/sei-db/ledger_db/receipt/litt_receipt_store.go @@ -5,6 +5,7 @@ import ( "encoding/binary" "errors" "fmt" + "math/rand" "os" "path/filepath" "sort" @@ -53,12 +54,27 @@ import ( // range, and reads enforce the retention floor, so visible retention never // exceeds that floor regardless of GC timing. // -// Pruning is driven from outside, by the StorageGarbageCollector through the -// gc.PrunableStore implementation in litt_receipt_gc.go. This store deliberately -// runs no pruner of its own: KeepRecent alone sees neither the shared rollback -// window nor the other stores' floors, so a local pruner would delete the -// rollback headroom the collector exists to preserve. KeepRecent remains what -// this store asks the collector to retain, and what sizes the litt TTL. +// Retention has two possible drivers and exactly one runs at a time, selected by +// cfg.ExternalPruning: +// +// - unset: the background pruner below keeps the last KeepRecent blocks. This +// is the standalone shape, and what a node not running a collector depends +// on. +// - set: the StorageGarbageCollector prunes through the gc.PrunableStore +// implementation in litt_receipt_gc.go, and startPruning stands down. +// +// Both at once would be actively wrong rather than merely redundant: KeepRecent +// sees neither the shared rollback window nor the other stores' floors, so the +// local pruner would delete the rollback headroom the collector exists to +// preserve. Neither at once is the other failure, and the worse one to debug — +// the floor stops advancing and the store grows without bound. runsLocalPruner +// therefore asks ExternalPruning rather than reading the field itself, so the +// collector's view of who prunes this store and the store's own view stay one +// fact. +// +// KeepRecent is what this store asks for either way: the pruner's window when it +// runs, the collector's retention window when it does not, and the litt TTL in +// both cases. type littReceiptStore struct { values litt.DB receipts litt.Table @@ -69,6 +85,8 @@ type littReceiptStore struct { earliestVersion atomic.Int64 keepRecent int64 + pruneInterval int64 + externalPruning bool logFilterParallelism int stopBackground chan struct{} backgroundWg sync.WaitGroup @@ -167,11 +185,14 @@ func newLittReceiptStore(cfg dbconfig.ReceiptStoreConfig, storeKey sdk.StoreKey) index: index, storeKey: storeKey, keepRecent: int64(cfg.KeepRecent), + pruneInterval: int64(cfg.PruneIntervalSeconds), + externalPruning: cfg.ExternalPruning, logFilterParallelism: logFilterParallelism, stopBackground: make(chan struct{}), } s.latestVersion.Store(s.readMeta(receiptLatestVersionKey)) s.earliestVersion.Store(s.readMeta(receiptEarliestVersionKey)) + s.startPruning() s.startFlusher() return s, nil } @@ -401,12 +422,55 @@ func (s *littReceiptStore) Close() error { return err } +// runsLocalPruner reports whether this store drives its own retention. Split out +// from startPruning because it is the whole of the decision and none of the +// timing: a test can enumerate it in full without waiting on a jittered ticker, +// which is what a silently-not-started pruner needs to be caught by. +// +// Asks ExternalPruning rather than reading the field behind it, so the +// collector's view of who prunes this store and the store's own view are the +// same read and cannot drift apart. +func (s *littReceiptStore) runsLocalPruner() bool { + return !s.ExternalPruning() && s.keepRecent > 0 && s.pruneInterval > 0 +} + +// startPruning runs the local retention pruner, which keeps the last keepRecent +// blocks. It is the store's own driver, and stands down entirely under +// externalPruning so it can never race the collector to a different floor — see +// the type doc for why both running is worse than either. +func (s *littReceiptStore) startPruning() { + if !s.runsLocalPruner() { + return + } + s.backgroundWg.Add(1) + go func() { + defer s.backgroundWg.Done() + for { + // Keep exactly keepRecent blocks, [latest-keepRecent+1, latest]; the + // +1 matches the pebble backend (which retains keepRecent, not +1). + pruneBefore := s.latestVersion.Load() - s.keepRecent + 1 + if pruneBefore > 0 { + if err := s.pruneBlocksBelow(uint64(pruneBefore)); err != nil { + logger.Error("failed to prune littdb receipt store", "before-block", pruneBefore, "err", err) + } + } + // Jittered cadence, matching the other receipt pruners. + sleep := time.Duration(float64(s.pruneInterval)*(1+rand.Float64())) * time.Second + select { + case <-s.stopBackground: + return + case <-time.After(sleep): + } + } + }() +} + // pruneBlocksBelow deletes the tag entries in [earliest, cutoff) and advances // the retention floor. Receipt values are reclaimed independently by litt's TTL // GC; the read-time floor keeps them invisible in the meantime. // -// Called by the StorageGarbageCollector via PruneBelow, which is the only thing -// that advances this floor in production — see the type doc. +// Shared by both retention drivers: startPruning above, and the collector via +// PruneBelow. Exactly one of them is live — see the type doc. func (s *littReceiptStore) pruneBlocksBelow(cutoff uint64) error { floor := uint64(0) if earliest := s.earliestVersion.Load(); earliest > 0 { diff --git a/sei-db/ledger_db/receipt/receipt_store.go b/sei-db/ledger_db/receipt/receipt_store.go index 25bdeded24..a46c818925 100644 --- a/sei-db/ledger_db/receipt/receipt_store.go +++ b/sei-db/ledger_db/receipt/receipt_store.go @@ -149,6 +149,13 @@ func newReceiptBackend(config dbconfig.ReceiptStoreConfig, storeKey sdk.StoreKey case receiptBackendLittIdx: return newLittReceiptStore(config, storeKey) case receiptBackendPebble: + // This backend does not implement gc.PrunableStore, so the collector never prunes it and + // its own pruner is the only one there is. Honoring ExternalPruning here would stop that + // pruner and put nothing in its place, so refuse at startup instead: unbounded receipt + // growth is not something to discover from a full disk weeks later. + if config.ExternalPruning { + return nil, fmt.Errorf("receipt store backend %q does not support external pruning; use %q", receiptBackendPebble, receiptBackendLittIdx) + } ssConfig := dbconfig.DefaultStateStoreConfig() ssConfig.DBDirectory = config.DBDirectory ssConfig.AsyncWriteBuffer = config.AsyncWriteBuffer From c9f3de9b6837dd7052e0e8da4480d30a0d52733c Mon Sep 17 00:00:00 2001 From: YimingZang Date: Thu, 6 Aug 2026 11:19:53 -0700 Subject: [PATCH 03/11] Remove retention for wal --- .../state_db/statewal/state_wal_brick_test.go | 2 +- sei-db/state_db/statewal/state_wal_config.go | 36 ------------------- sei-db/state_db/statewal/state_wal_gc.go | 19 +++++----- sei-db/state_db/statewal/state_wal_gc_test.go | 33 ++++------------- sei-db/state_db/statewal/state_wal_impl.go | 11 ++---- 5 files changed, 20 insertions(+), 81 deletions(-) diff --git a/sei-db/state_db/statewal/state_wal_brick_test.go b/sei-db/state_db/statewal/state_wal_brick_test.go index 9282c8f6b3..a047a2d78f 100644 --- a/sei-db/state_db/statewal/state_wal_brick_test.go +++ b/sei-db/state_db/statewal/state_wal_brick_test.go @@ -65,7 +65,7 @@ func (f *fakeWAL) Close() error { func newFakeStateWAL(t *testing.T, f *fakeWAL) StateWAL { t.Helper() - w, err := newStateWAL(f, 0) + w, err := newStateWAL(f) require.NoError(t, err) return w } diff --git a/sei-db/state_db/statewal/state_wal_config.go b/sei-db/state_db/statewal/state_wal_config.go index 334a63b2d7..6aa7eec9bc 100644 --- a/sei-db/state_db/statewal/state_wal_config.go +++ b/sei-db/state_db/statewal/state_wal_config.go @@ -1,10 +1,8 @@ package statewal import ( - "fmt" "time" - "github.com/sei-protocol/sei-chain/sei-db/management/gc" "github.com/sei-protocol/sei-chain/sei-db/seiwal" ) @@ -41,38 +39,9 @@ type Config struct { // The interval at which the underlying WAL samples the buffered depth of its internal channels into the // seiwal_queue_depth gauge. Zero or negative disables sampling. MetricsSampleInterval time.Duration - - // RetentionWindow is how much history this WAL keeps beyond the shared rollback window of the - // StorageGarbageCollector that manages it, in blocks. It is what gc.PrunableStore.GetRetentionWindow - // answers: - // - // > 0 → that many blocks of history beyond the rollback window - // 0 → the rollback window only, plus whatever the other managed stores hold it back for - // -1 → never prune this WAL (gc.InfiniteRetentionWindow) - // - // Zero does NOT mean "keep everything" here, unlike the KeepRecent fields on StateStoreConfig and - // ReceiptStoreConfig, where 0 disables pruning. It is the most aggressive setting this field has; - // "keep everything" is -1. Assigning a KeepRecent value to this field inverts the retention it - // asks for. - // - // Leave this at 0 unless something outside SC/SS needs the history. The WAL is what SC and SS replay - // from, and they already hold it back on their own: each answers its oldest live snapshot as its - // pruning boundary, and the collector prunes every store to the shared minimum. Depth declared here is - // additive on top of that, and — because the minimum is shared — it retains the whole fleet that much - // further back, not just this WAL. Its purpose is a need the snapshot stores do not express, such as - // serving catch-up to a peer further behind than any live snapshot. - // - // -1 is for a WAL that must never be reclaimed at all. It is not a way to protect a replay range: a - // store with no cut line is never asked for a boundary and never pruned, so the WAL simply grows - // without bound. Must be >= gc.InfiniteRetentionWindow. - RetentionWindow int64 } // Constructor for a default state WAL configuration for the WAL at path, identified by name. -// -// RetentionWindow defaults to 0 — no history beyond what SC and SS hold the WAL back for — because that is -// the depth the WAL is actually required to have, and any other default would retain every managed store -// that much further back. See the field for when to raise it. func DefaultConfig(path string, name string) *Config { s := seiwal.DefaultConfig(path, name) return &Config{ @@ -84,16 +53,11 @@ func DefaultConfig(path string, name string) *Config { FsyncOnFlush: s.FsyncOnFlush, IteratorPrefetchSize: s.IteratorPrefetchSize, MetricsSampleInterval: s.MetricsSampleInterval, - RetentionWindow: 0, } } // Validate the configuration, returning nil if valid, or an error describing the problem if invalid. func (c *Config) Validate() error { - if c.RetentionWindow < gc.InfiniteRetentionWindow { - return fmt.Errorf("RetentionWindow must be >= %d (got %d)", - gc.InfiniteRetentionWindow, c.RetentionWindow) - } return c.toSeiwalConfig().Validate() } diff --git a/sei-db/state_db/statewal/state_wal_gc.go b/sei-db/state_db/statewal/state_wal_gc.go index 15a6ab13f5..ba672157ab 100644 --- a/sei-db/state_db/statewal/state_wal_gc.go +++ b/sei-db/state_db/statewal/state_wal_gc.go @@ -47,16 +47,17 @@ func (w *stateWALImpl) PruneBelow(blockNumber uint64) error { return nil } -// GetRetentionWindow reports the configured history beyond the collector's shared rollback window. -// See Config.RetentionWindow for the meaning of each value and why its default of 0 is the depth this -// WAL actually needs: SC and SS hold it back to their oldest live snapshot on their own, by answering -// that snapshot as their pruning boundary, so history declared here is additive on top of that and -// applies to every managed store rather than to this WAL alone. +// GetRetentionWindow reports 0, and this is a property of what the WAL is rather than a default +// something might want to raise. How deep this WAL must go is not its own to declare: it is a replay +// source, and the depth it needs is whatever its consumers need it to be. SC and SS express that by +// answering their oldest live snapshot as a pruning boundary, and the collector prunes every store +// to the shared minimum, so the WAL is already held exactly as far back as they can replay from. +// +// A window here would be additive on top of that, and — because the minimum is shared — it would +// retain every managed store that much further back rather than this WAL alone. That is a fleet-wide +// retention decision wearing a per-store name, which is what RollbackWindow already is. func (w *stateWALImpl) GetRetentionWindow() int64 { - if w.retentionWindow < 0 { - return gc.InfiniteRetentionWindow - } - return w.retentionWindow + return 0 } // GetPruningBoundary returns cutLine, the contract's answer for a contiguous store: the WAL holds diff --git a/sei-db/state_db/statewal/state_wal_gc_test.go b/sei-db/state_db/statewal/state_wal_gc_test.go index fe491296ec..bccedc034d 100644 --- a/sei-db/state_db/statewal/state_wal_gc_test.go +++ b/sei-db/state_db/statewal/state_wal_gc_test.go @@ -65,34 +65,13 @@ func TestGCLatestBlockRecoveredOnOpen(t *testing.T) { require.Equal(t, uint64(5), latest) } -// The window is the configured one, sentinel included: -1 has to survive the trip verbatim or an -// operator asking for "never prune" gets a 1-block window instead. The default is 0, which asks for -// no history of its own — what holds the WAL back then is SC/SS answering their oldest live snapshot, -// which the collector applies to every store as the shared minimum. -func TestGCRetentionWindowComesFromConfig(t *testing.T) { +// The WAL asks for no history of its own, and there is no configuration that changes that. What +// holds it back is SC/SS answering their oldest live snapshot as a boundary, which the collector +// applies to every store as the shared minimum — so its depth tracks its consumers automatically +// (see GetRetentionWindow). +func TestGCRetentionWindowIsZero(t *testing.T) { _, store := openWALForGC(t, testConfig(t.TempDir())) - require.Equal(t, int64(0), store.GetRetentionWindow(), "the default asks for no history of its own") - - for _, retentionWindow := range []int64{gc.InfiniteRetentionWindow, 100_000} { - cfg := testConfig(t.TempDir()) - cfg.RetentionWindow = retentionWindow - _, configured := openWALForGC(t, cfg) - require.Equal(t, retentionWindow, configured.GetRetentionWindow()) - } -} - -func TestConfigValidateRetentionWindow(t *testing.T) { - cfg := testConfig(t.TempDir()) - - for _, retentionWindow := range []int64{gc.InfiniteRetentionWindow, 0, 1} { - cfg.RetentionWindow = retentionWindow - require.NoError(t, cfg.Validate()) - } - - // Below InfiniteRetentionWindow there is no meaning left to assign, and GetRetentionWindow reads any - // negative value as infinite retention — so a typo'd -2 would silently disable pruning. - cfg.RetentionWindow = -2 - require.ErrorContains(t, cfg.Validate(), "RetentionWindow") + require.Equal(t, int64(0), store.GetRetentionWindow()) } // A contiguous store answers the cut line it was given whatever it holds, including on an empty WAL diff --git a/sei-db/state_db/statewal/state_wal_impl.go b/sei-db/state_db/statewal/state_wal_impl.go index 23fe2e604a..d2aedecd12 100644 --- a/sei-db/state_db/statewal/state_wal_impl.go +++ b/sei-db/state_db/statewal/state_wal_impl.go @@ -22,11 +22,6 @@ type stateWALImpl struct { // The underlying generic WAL, keyed by block number, whose payload is a block's changesets. wal seiwal.WAL[[]*proto.NamedChangeSet] - // Config.RetentionWindow, the history this WAL asks the garbage collector to keep beyond the shared - // rollback window. Immutable after construction, so the collector reads it off-goroutine without - // synchronization (GetRetentionWindow). - retentionWindow int64 - // Set by Close() so subsequent calls fail fast. A plain field: like the write-ordering state below, it // is only ever touched by the single caller, which must not invoke methods concurrently. closed bool @@ -68,7 +63,7 @@ func New(config *Config) (StateWAL, error) { if err != nil { return nil, fmt.Errorf("failed to open state WAL: %w", err) } - return newStateWAL(wal, config.RetentionWindow) + return newStateWAL(wal) } // GetRange reports the range of block numbers stored in the state WAL directory configured by config, @@ -119,8 +114,8 @@ func VerifyIntegrity(config *Config) error { return nil } -func newStateWAL(wal seiwal.WAL[[]*proto.NamedChangeSet], retentionWindow int64) (StateWAL, error) { - w := &stateWALImpl{wal: wal, retentionWindow: retentionWindow} +func newStateWAL(wal seiwal.WAL[[]*proto.NamedChangeSet]) (StateWAL, error) { + w := &stateWALImpl{wal: wal} // Recover the write-ordering position from the highest block already on disk. ok, _, last, err := wal.Bounds() From 2a10f31b5cc4a0d5fd8b39c0ddb8c69168e3bbc5 Mon Sep 17 00:00:00 2001 From: YimingZang Date: Thu, 6 Aug 2026 11:47:51 -0700 Subject: [PATCH 04/11] Address comments --- .../ledger_db/receipt/litt_receipt_gc_test.go | 30 +++++++++++++++++++ .../ledger_db/receipt/litt_receipt_store.go | 18 +++++++++++ .../internal/p2p/giga_router_fullnode_test.go | 2 +- .../p2p/giga_router_validator_test.go | 4 +-- 4 files changed, 51 insertions(+), 3 deletions(-) diff --git a/sei-db/ledger_db/receipt/litt_receipt_gc_test.go b/sei-db/ledger_db/receipt/litt_receipt_gc_test.go index 246ab513be..6ac65ed895 100644 --- a/sei-db/ledger_db/receipt/litt_receipt_gc_test.go +++ b/sei-db/ledger_db/receipt/litt_receipt_gc_test.go @@ -179,3 +179,33 @@ func TestReceiptGCPruningBoundaryAndPruneBelow(t *testing.T) { require.NoError(t, prunable.PruneBelow(1)) require.Equal(t, int64(3), store.EarliestVersion()) } + +// PruneBelow carries a minimum taken across every managed store, so it can arrive above this +// store's head whenever this one lags or has ingested nothing. Both cases are the store's own to +// survive: the collector's head minimum makes them unlikely, but that is a property of the caller. +func TestReceiptGCPruneBelowAboveHead(t *testing.T) { + addr := common.HexToAddress("0xabcd") + topic := common.HexToHash("0x1111") + + t.Run("empty store keeps its floor at zero", func(t *testing.T) { + store, prunable, _ := setupLittIdxForGC(t, 0) + + require.NoError(t, prunable.PruneBelow(1_000)) + require.Equal(t, int64(0), store.EarliestVersion(), + "a floor above an empty store would have to be walked back once blocks arrive") + }) + + t.Run("lagging store is capped at its head", func(t *testing.T) { + store, prunable, ctx := setupLittIdxForGC(t, 0) + for block := uint64(1); block <= 3; block++ { + writeLitBlock(t, store, ctx, block, litReceipt(block, 0, addr, topic)) + } + + require.NoError(t, prunable.PruneBelow(1_000)) + require.Equal(t, int64(3), store.EarliestVersion(), "the floor stops at the head, not the request") + + kept, err := store.GetReceiptFromStore(ctx, litTxHash(3, 0)) + require.NoError(t, err, "honoring the request literally would drop every block the store holds") + require.Equal(t, uint64(3), kept.BlockNumber) + }) +} diff --git a/sei-db/ledger_db/receipt/litt_receipt_store.go b/sei-db/ledger_db/receipt/litt_receipt_store.go index c57c69b9db..a448767bf7 100644 --- a/sei-db/ledger_db/receipt/litt_receipt_store.go +++ b/sei-db/ledger_db/receipt/litt_receipt_store.go @@ -471,7 +471,25 @@ func (s *littReceiptStore) startPruning() { // // Shared by both retention drivers: startPruning above, and the collector via // PruneBelow. Exactly one of them is live — see the type doc. +// +// cutoff may sit above this store's head, because the collector's PruneBelow +// carries a minimum taken across every managed store and this one can lag or be +// empty. Such a request is capped at the head rather than honored: taking it +// literally would drop every tag entry the store holds and leave the floor above +// anything it could serve. The collector's own head minimum makes that request +// unlikely, but that is a property of the caller, and the floor here should not +// depend on reasoning about one. func (s *littReceiptStore) pruneBlocksBelow(cutoff uint64) error { + head := s.latestVersion.Load() + if head <= 0 { + // Nothing ingested, so nothing to drop, and a floor above an empty store + // would only have to be walked back when blocks finally arrive. + return nil + } + if cutoff > uint64(head) { //nolint:gosec // guarded positive above + cutoff = uint64(head) + } + floor := uint64(0) if earliest := s.earliestVersion.Load(); earliest > 0 { floor = uint64(earliest) //nolint:gosec // earliest is non-negative diff --git a/sei-tendermint/internal/p2p/giga_router_fullnode_test.go b/sei-tendermint/internal/p2p/giga_router_fullnode_test.go index 5c3ecb0b56..4c1c846b51 100644 --- a/sei-tendermint/internal/p2p/giga_router_fullnode_test.go +++ b/sei-tendermint/internal/p2p/giga_router_fullnode_test.go @@ -60,7 +60,7 @@ func TestGigaRouter_Fullnode(t *testing.T) { proxyApp := proxy.New(app) dir := t.TempDir() - // Same resolve path as config.AutobahnBlockDBConfig{}.BlockDBConfig + // Same resolve path as config.AutobahnBlockDBConfig{}.LittBlockConfig // (zero overrides); p2p can't import config (import cycle). littCfg, err := littblock.DefaultConfig(filepath.Join(dir, "blockdb")) require.NoError(t, err) diff --git a/sei-tendermint/internal/p2p/giga_router_validator_test.go b/sei-tendermint/internal/p2p/giga_router_validator_test.go index f831e966b4..90c641a4dc 100644 --- a/sei-tendermint/internal/p2p/giga_router_validator_test.go +++ b/sei-tendermint/internal/p2p/giga_router_validator_test.go @@ -68,7 +68,7 @@ func TestGigaRouter_FinalizeBlocks(t *testing.T) { // In giga mode the CometBFT handshaker is skipped; the router's // runExecute calls InitChain itself on fresh start. dir := t.TempDir() - // Same resolve path as config.AutobahnBlockDBConfig{}.BlockDBConfig + // Same resolve path as config.AutobahnBlockDBConfig{}.LittBlockConfig // (zero overrides); p2p can't import config (import cycle). littCfg, err := littblock.DefaultConfig(filepath.Join(dir, "blockdb")) require.NoError(t, err, "littblock.DefaultConfig[%v]", i) @@ -237,7 +237,7 @@ func TestGigaRouter_EvmProxy(t *testing.T) { require.NoError(t, genDoc.ValidateAndComplete()) dir := t.TempDir() - // Same resolve path as config.AutobahnBlockDBConfig{}.BlockDBConfig + // Same resolve path as config.AutobahnBlockDBConfig{}.LittBlockConfig // (zero overrides); p2p can't import config (import cycle). littCfg, err := littblock.DefaultConfig(filepath.Join(dir, "blockdb")) require.NoError(t, err) From f43b8b31567fd7217b15cce84fe28816ae5ea4d6 Mon Sep 17 00:00:00 2001 From: YimingZang Date: Thu, 6 Aug 2026 12:20:18 -0700 Subject: [PATCH 05/11] Address comments --- .../ledger_db/block/littblock/litt_block_db.go | 15 +++++++++------ sei-db/state_db/statewal/state_wal_gc.go | 17 +++++++++++++---- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index 97a677362f..252e0a240c 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -12,11 +12,14 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" ) -// TableName is the single table holding both blocks and QCs. They share -// one table so a crash leaves a contiguous write-order prefix spanning both -// record kinds (see NewBlockDB), which is what guarantees a persisted block is -// always covered by a persisted QC. -const TableName = "blocks" +// tableName is the single table holding blocks and QCs both, despite the name. They share one +// table so a crash leaves a contiguous write-order prefix spanning both record kinds (see +// NewBlockDB), which is what guarantees a persisted block is always covered by a persisted QC. +// +// This value is persisted layout, not just an identifier: littdb puts a table's data at +// //segments, so changing it makes NewBlockDB open a fresh empty table while the +// old data sits untouched under the previous name — neither served nor reclaimed. +const tableName = "blocks" var _ types.BlockDB = (*blockDB)(nil) @@ -97,7 +100,7 @@ func NewBlockDB(config *BlockDBConfig) (types.BlockDB, error) { // guarantees a persisted block is always covered by a persisted QC. It also // backs the write-order cursors and contiguous-QC recovery. ShardingFactor // > 1, or splitting blocks and QCs across two tables, would void this. - tableConfig := littdb.DefaultTableConfig(TableName) + tableConfig := littdb.DefaultTableConfig(tableName) tableConfig.TTL = config.Retention tableConfig.GCFilter = s.gcFilter tableConfig.ShardingFactor = 1 // DO NOT CHANGE!! diff --git a/sei-db/state_db/statewal/state_wal_gc.go b/sei-db/state_db/statewal/state_wal_gc.go index ba672157ab..d3097dae7b 100644 --- a/sei-db/state_db/statewal/state_wal_gc.go +++ b/sei-db/state_db/statewal/state_wal_gc.go @@ -23,11 +23,20 @@ func (w *stateWALImpl) Name() string { return "StateWAL" } -// ExternalPruning is unconditionally true: the WAL prunes only when told to, by Prune or PruneBelow. +// ExternalPruning is unconditionally true: the WAL prunes only when told to, by Prune or PruneBelow, +// so there is never a pruner inside it for the collector to collide with. // -// Note that its owner may still prune it directly — FlatKV does, in tryTruncateWAL — which this -// cannot report on, since it is a property of the caller rather than of the WAL. FlatKV stands that -// down under its own ExternalPruning for the same reason this exists. +// FlatKV does prune this WAL directly today, in tryTruncateWAL, which can look like the store +// enforcing its own retention and so like a reason to answer false. It is not. This method is read +// only from a prune cycle, so it has an effect only where a collector exists — and FlatKV stands +// tryTruncateWAL down under its own config.ExternalPruning precisely so the collector can take the +// WAL over. False would leave that handover with nothing on either side of it: tryTruncateWAL +// stopped, the collector declining, and the WAL growing without bound. +// +// Where FlatKV has not handed over, both prune, and the shared minimum makes that safe rather than +// merely redundant: a store is asked for its boundary whatever it answers here, so FlatKV holds the +// collector back to the oldest snapshot it still replays from. That relies on FlatKV being +// registered at all, which is the precondition on the type above. func (w *stateWALImpl) ExternalPruning() bool { return true } From 47a6dfd7008e480e004000ac5d9a51ca6fb2e9aa Mon Sep 17 00:00:00 2001 From: YimingZang Date: Thu, 6 Aug 2026 14:27:07 -0700 Subject: [PATCH 06/11] Fix config --- .../server/config/testdata/server_config.golden | 1 + sei-db/config/receipt_config.go | 2 +- sei-db/state_db/sc/flatkv/config/config.go | 16 ++++++++++++---- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/sei-cosmos/server/config/testdata/server_config.golden b/sei-cosmos/server/config/testdata/server_config.golden index a2ce09015b..a8b4777fa6 100644 --- a/sei-cosmos/server/config/testdata/server_config.golden +++ b/sei-cosmos/server/config/testdata/server_config.golden @@ -67,6 +67,7 @@ StateCommit.FlatKVConfig.Fsync = bool(false) StateCommit.FlatKVConfig.AsyncWriteBuffer = int(0) StateCommit.FlatKVConfig.SnapshotInterval = uint32(10000) StateCommit.FlatKVConfig.SnapshotKeepRecent = uint32(1) +StateCommit.FlatKVConfig.ExternalPruning = bool(false) StateCommit.FlatKVConfig.EnablePebbleMetrics = bool(true) StateCommit.FlatKVConfig.EnableReadWriteMetrics = bool(false) StateCommit.FlatKVConfig.AccountDBConfig.DataDir = string("") diff --git a/sei-db/config/receipt_config.go b/sei-db/config/receipt_config.go index a2dd8c18f0..b4b90e7d6f 100644 --- a/sei-db/config/receipt_config.go +++ b/sei-db/config/receipt_config.go @@ -71,7 +71,7 @@ type ReceiptStoreConfig struct { // Only the littidx backend honors this. The pebbledb backend is not a gc.PrunableStore, // so the collector would not prune it and setting this would leave it with no pruner at // all; newReceiptBackend rejects that combination rather than growing without bound. - ExternalPruning bool `mapstructure:"-"` + ExternalPruning bool `mapstructure:"external-pruning"` // EnableReadWriteMetrics emits simple estimated read/write counters for Pebble-backed receipt storage. // defaults to false diff --git a/sei-db/state_db/sc/flatkv/config/config.go b/sei-db/state_db/sc/flatkv/config/config.go index f75a42d785..18852ba9cf 100644 --- a/sei-db/state_db/sc/flatkv/config/config.go +++ b/sei-db/state_db/sc/flatkv/config/config.go @@ -47,10 +47,18 @@ type Config struct { // own snapshots (SnapshotKeepRecent) and stops truncating the state WAL, and answers // gc.PrunableStore.ExternalPruning with this value so the collector takes both over. // - // Set this only when the store is actually registered with a running collector. Nothing here - // can check that, and the failure is silent in the expensive direction: snapshots and the WAL - // then grow without bound, because the machinery that used to bound them has stood down and - // nothing replaced it. + // Like ReceiptStoreConfig.ExternalPruning this is not read from app.toml, and for the same + // reason: it is only correct when this store is registered with a running collector, which is + // a property of how the process was wired and not something an operator can assert. It is set + // by whatever constructs the collector. The stakes are higher here than on the receipt side — + // this one flag stands down two mechanisms, pruneSnapshots and tryTruncateWAL, so a stray key + // would leave both the snapshots and the state WAL with nothing bounding them, and the failure + // is silent in the expensive direction. + // + // Unlike the receipt path, which rejects pebble + ExternalPruning in newReceiptBackend, there + // is no equivalent check to make here: "a collector exists" is not knowable from this package. + // Keeping the field unreachable from config is what replaces that guard; the collector-exists + // check belongs wherever the collector is eventually constructed. // // It is one field rather than two so the count-based pruner and the collector can never both // be enforcing retention — they would disagree, and SnapshotKeepRecent would win by deleting From 29e7d7dec7bdb0d4a3b8c93ccbce687fcd42989f Mon Sep 17 00:00:00 2001 From: YimingZang Date: Sun, 9 Aug 2026 15:12:10 -0700 Subject: [PATCH 07/11] More config change --- app/seidb_test.go | 6 +- app/testdata/state-commit.golden | 1 + sei-db/config/receipt_config.go | 25 ++- sei-db/config/testdata/receipt-store.golden | 2 +- sei-db/ledger_db/block/block_db_test.go | 2 +- sei-db/ledger_db/block/blocksim/blocksim.go | 2 +- .../block/littblock/litt_block_config.go | 24 ++- .../block/littblock/litt_block_db.go | 53 ++++- .../block/littblock/litt_block_gc.go | 7 +- .../block/littblock/litt_block_gc_test.go | 2 +- .../littblock/litt_block_legacy_table_test.go | 63 ++++++ .../littblock/litt_block_stranding_test.go | 2 +- sei-db/ledger_db/receipt/litt_receipt_gc.go | 7 +- .../ledger_db/receipt/litt_receipt_gc_test.go | 3 +- .../litt_receipt_gcfilter_internal_test.go | 189 ++++++++++++++++++ .../ledger_db/receipt/litt_receipt_store.go | 121 +++++++---- sei-db/state_db/sc/flatkv/config/config.go | 2 +- sei-db/state_db/sc/flatkv/store_gc.go | 7 +- sei-tendermint/config/autobahn.go | 2 +- sei-tendermint/config/autobahn_test.go | 2 +- .../autobahn/data/state_recovery_test.go | 4 +- .../internal/autobahn/data/state_test.go | 2 +- sei-tendermint/node/setup_test.go | 4 +- 23 files changed, 453 insertions(+), 79 deletions(-) create mode 100644 sei-db/ledger_db/block/littblock/litt_block_legacy_table_test.go create mode 100644 sei-db/ledger_db/receipt/litt_receipt_gcfilter_internal_test.go diff --git a/app/seidb_test.go b/app/seidb_test.go index f089826f2b..48cbcb0617 100644 --- a/app/seidb_test.go +++ b/app/seidb_test.go @@ -252,7 +252,9 @@ func TestParseReceiptConfigs_UsesConfiguredBackend(t *testing.T) { assert.NoError(t, err) assert.Equal(t, "pebbledb", receiptConfig.Backend) assert.Equal(t, config.DefaultReceiptStoreConfig().AsyncWriteBuffer, receiptConfig.AsyncWriteBuffer) - assert.Equal(t, 0, receiptConfig.KeepRecent) + // Left at the default: no [receipt-store] key reaches KeepRecent. What a node retains is + // set later, by readReceiptStoreConfig from min-retain-blocks. + assert.Equal(t, config.DefaultReceiptKeepRecent, receiptConfig.KeepRecent) } func TestParseReceiptConfigs_UsesConfiguredValues(t *testing.T) { @@ -267,7 +269,7 @@ func TestParseReceiptConfigs_UsesConfiguredValues(t *testing.T) { assert.Equal(t, "/tmp/custom-receipt-db", receiptConfig.DBDirectory) assert.Equal(t, "pebbledb", receiptConfig.Backend) assert.Equal(t, 7, receiptConfig.AsyncWriteBuffer) - assert.Equal(t, 0, receiptConfig.KeepRecent) + assert.Equal(t, config.DefaultReceiptKeepRecent, receiptConfig.KeepRecent) assert.Equal(t, 9, receiptConfig.PruneIntervalSeconds) assert.True(t, receiptConfig.EnableReadWriteMetrics) } diff --git a/app/testdata/state-commit.golden b/app/testdata/state-commit.golden index 58522b9721..bd9be14299 100644 --- a/app/testdata/state-commit.golden +++ b/app/testdata/state-commit.golden @@ -15,6 +15,7 @@ FlatKVConfig.Fsync = bool(false) FlatKVConfig.AsyncWriteBuffer = int(0) FlatKVConfig.SnapshotInterval = uint32(10000) FlatKVConfig.SnapshotKeepRecent = uint32(1) +FlatKVConfig.ExternalPruning = bool(false) FlatKVConfig.EnablePebbleMetrics = bool(true) FlatKVConfig.EnableReadWriteMetrics = bool(false) FlatKVConfig.AccountDBConfig.DataDir = string("") diff --git a/sei-db/config/receipt_config.go b/sei-db/config/receipt_config.go index b4b90e7d6f..f178ca0f72 100644 --- a/sei-db/config/receipt_config.go +++ b/sei-db/config/receipt_config.go @@ -27,6 +27,12 @@ const ( // littidx eth_getLogs (see ReceiptStoreConfig.LogFilterParallelism). const DefaultReceiptLogFilterParallelism = 16 +// DefaultReceiptKeepRecent is the default retention window in blocks for callers that +// build a ReceiptStoreConfig directly. A seid node never uses it — see +// ReceiptStoreConfig.KeepRecent for why — so it is sized for a tool or test that wants +// bounded growth, not to express what a node should retain. +const DefaultReceiptKeepRecent = 10000 + // ReceiptStoreConfig defines configuration for the receipt store database. type ReceiptStoreConfig struct { // DBDirectory defines the directory to store the receipt store db files @@ -47,8 +53,13 @@ type ReceiptStoreConfig struct { // KeepRecent defines the number of versions to keep in receipt store. // Setting it to 0 means keep everything (no pruning). - // This is NOT read from receipt-store config; it is always derived from - // the global min-retain-blocks flag at the app layer. + // + // This is NOT read from receipt-store config, and on a seid node it is not read + // from DefaultReceiptStoreConfig either: readReceiptStoreConfig overwrites it + // unconditionally from the global min-retain-blocks flag, including with 0 when + // that flag is unset. The default below is therefore only reachable by callers + // that build this config directly — tools and tests — and changing it does not + // change what any node retains. KeepRecent int `mapstructure:"-"` // PruneIntervalSeconds defines the interval in seconds to trigger pruning @@ -71,7 +82,7 @@ type ReceiptStoreConfig struct { // Only the littidx backend honors this. The pebbledb backend is not a gc.PrunableStore, // so the collector would not prune it and setting this would leave it with no pruner at // all; newReceiptBackend rejects that combination rather than growing without bound. - ExternalPruning bool `mapstructure:"external-pruning"` + ExternalPruning bool `mapstructure:"-"` // EnableReadWriteMetrics emits simple estimated read/write counters for Pebble-backed receipt storage. // defaults to false @@ -86,13 +97,15 @@ type ReceiptStoreConfig struct { } // DefaultReceiptStoreConfig returns the default ReceiptStoreConfig. -// KeepRecent defaults to 0 (no pruning). The app layer is responsible -// for setting KeepRecent from the global min-retain-blocks flag. +// +// KeepRecent is a bounded default rather than 0 so that a caller building this config +// directly gets a store that prunes. It is not what a node retains: the app layer +// replaces it with min-retain-blocks before the store is opened (see KeepRecent). func DefaultReceiptStoreConfig() ReceiptStoreConfig { return ReceiptStoreConfig{ Backend: "pebbledb", AsyncWriteBuffer: DefaultSSAsyncBuffer, - KeepRecent: 0, + KeepRecent: DefaultReceiptKeepRecent, PruneIntervalSeconds: DefaultSSPruneInterval, LogFilterParallelism: DefaultReceiptLogFilterParallelism, } diff --git a/sei-db/config/testdata/receipt-store.golden b/sei-db/config/testdata/receipt-store.golden index e6891d7a05..2abac0169b 100644 --- a/sei-db/config/testdata/receipt-store.golden +++ b/sei-db/config/testdata/receipt-store.golden @@ -1,7 +1,7 @@ DBDirectory = string("") Backend = string("pebbledb") AsyncWriteBuffer = int(100) -KeepRecent = int(0) +KeepRecent = int(10000) PruneIntervalSeconds = int(600) ExternalPruning = bool(false) EnableReadWriteMetrics = bool(false) diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index acd57e0878..1c07f8e703 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -1405,7 +1405,7 @@ func TestMemblockPruneIntoCohortRoundsDown(t *testing.T) { func littConfig(t *testing.T, dir string) *littblock.BlockDBConfig { cfg, err := littblock.DefaultConfig(dir) require.NoError(t, err) - cfg.Retention = time.Nanosecond + cfg.RetentionTime = time.Nanosecond return cfg } diff --git a/sei-db/ledger_db/block/blocksim/blocksim.go b/sei-db/ledger_db/block/blocksim/blocksim.go index e70e8d4a87..2e74e261bf 100644 --- a/sei-db/ledger_db/block/blocksim/blocksim.go +++ b/sei-db/ledger_db/block/blocksim/blocksim.go @@ -506,7 +506,7 @@ func openBlockDB(config *BlocksimConfig) (types.BlockDB, error) { if err != nil { return nil, fmt.Errorf("failed to build litt block db config: %w", err) } - littConfig.Retention = time.Duration(config.LittRetentionSeconds) * time.Second + littConfig.RetentionTime = time.Duration(config.LittRetentionSeconds) * time.Second // Record litt_* metrics into blocksim's already-configured global OTel MeterProvider (set up in // main before the DB is opened). MetricsServeEndpoint stays false so LittDB does not stand up its // own registry/server; the metrics surface on blocksim's single /metrics endpoint. diff --git a/sei-db/ledger_db/block/littblock/litt_block_config.go b/sei-db/ledger_db/block/littblock/litt_block_config.go index c4e8d57655..5473a9dd07 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_config.go +++ b/sei-db/ledger_db/block/littblock/litt_block_config.go @@ -11,16 +11,22 @@ import ( // BlockDBConfig configures a LittDB-backed types.BlockDB. type BlockDBConfig struct { // Litt is the underlying LittDB configuration, including the data directory - // paths. The block store builds its two tables (blocks, qcs) on top of this - // DB. Required; use DefaultConfig to obtain one with sane defaults, then - // override fields as needed (e.g. Litt.Fsync, Litt.GCPeriod). + // paths. The block store builds its single table (see tableName, which holds + // blocks and QCs both) on top of this DB. Required; use DefaultConfig to obtain + // one with sane defaults, then override fields as needed (e.g. Litt.Fsync, + // Litt.GCPeriod). Litt *littdb.Config - // Retention is the failsafe minimum age before any pruned record may be + // RetentionTime is the failsafe minimum age before any pruned record may be // reclaimed. Reclamation requires BOTH this age to elapse AND the prune // watermark to advance past the record, so even an over-eager watermark - // cannot delete data younger than Retention. Must be positive. - Retention time.Duration + // cannot delete data younger than RetentionTime. Must be positive. + // + // It is an age floor, not a retention policy: how much history this store + // keeps is RetentionWindow below. Raising this only delays reclaiming what + // the watermark has already released, which costs disk and buys nothing the + // window does not already express. + RetentionTime time.Duration // RetentionWindow is how much history this store keeps beyond the shared rollback // window of the StorageGarbageCollector that manages it, in blocks. It is what @@ -53,7 +59,7 @@ func DefaultConfig(dir string) (*BlockDBConfig, error) { } return &BlockDBConfig{ Litt: littConfig, - Retention: 24 * time.Hour, + RetentionTime: time.Hour, RetentionWindow: 10000, }, nil } @@ -66,8 +72,8 @@ func (c *BlockDBConfig) Validate() error { if c.Litt == nil { return fmt.Errorf("config.Litt is required") } - if c.Retention <= 0 { - return fmt.Errorf("config.Retention must be positive (got %s)", c.Retention) + if c.RetentionTime <= 0 { + return fmt.Errorf("config.RetentionTime must be positive (got %s)", c.RetentionTime) } if c.RetentionWindow < gc.InfiniteRetentionWindow { return fmt.Errorf("config.RetentionWindow must be >= %d (got %d)", diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index 252e0a240c..267d2ebb22 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -2,6 +2,8 @@ package littblock import ( "fmt" + "os" + "path/filepath" "sync" "sync/atomic" @@ -18,9 +20,14 @@ import ( // // This value is persisted layout, not just an identifier: littdb puts a table's data at // //segments, so changing it makes NewBlockDB open a fresh empty table while the -// old data sits untouched under the previous name — neither served nor reclaimed. +// old data sits untouched under the previous name — neither served nor reclaimed. refuseLegacyTable +// turns that into a startup error rather than a store that looks healthy and empty. const tableName = "blocks" +// legacyTableName is what tableName was called before the rename. Nothing opens it; it exists only +// so refuseLegacyTable can recognize a directory written before the rename. +const legacyTableName = "ledger" + var _ types.BlockDB = (*blockDB)(nil) // blockDB is a durable types.BlockDB backed by LittDB @@ -78,13 +85,18 @@ type blockDB struct { } // NewBlockDB opens (or creates) a LittDB-backed types.BlockDB from config. The -// underlying LittDB is built from config.Litt, and the two tables apply -// config.Retention as a TTL failsafe (pruning never reclaims data younger than -// that even once the watermark has advanced past it). +// underlying LittDB is built from config.Litt, and the table applies +// config.RetentionTime as a TTL failsafe (pruning never reclaims data younger +// than that even once the watermark has advanced past it). func NewBlockDB(config *BlockDBConfig) (types.BlockDB, error) { if err := config.Validate(); err != nil { return nil, fmt.Errorf("invalid block db config: %w", err) } + // Before littbuilder.NewDB, so a refused open leaves the directory exactly as it found it + // rather than adding an empty table beside the one it is complaining about. + if err := refuseLegacyTable(config.Litt.Paths); err != nil { + return nil, err + } db, err := littbuilder.NewDB(config.Litt) if err != nil { return nil, fmt.Errorf("failed to open litt db: %w", err) @@ -101,7 +113,7 @@ func NewBlockDB(config *BlockDBConfig) (types.BlockDB, error) { // backs the write-order cursors and contiguous-QC recovery. ShardingFactor // > 1, or splitting blocks and QCs across two tables, would void this. tableConfig := littdb.DefaultTableConfig(tableName) - tableConfig.TTL = config.Retention + tableConfig.TTL = config.RetentionTime tableConfig.GCFilter = s.gcFilter tableConfig.ShardingFactor = 1 // DO NOT CHANGE!! table, err := db.BuildTable(tableConfig) @@ -124,6 +136,37 @@ func NewBlockDB(config *BlockDBConfig) (types.BlockDB, error) { return s, nil } +// refuseLegacyTable fails the open when any root path holds a table directory under +// legacyTableName. Such a directory is blocks and QCs that this process cannot reach: littdb +// resolves a table to //segments, so the data is neither served nor reclaimed, and +// the store would otherwise present itself as healthy and empty. An empty store is indistinguishable +// from a correct one until something asks for history that is no longer there, which makes it the +// worst shape this failure could take. +// +// No deployment can reach this — nothing has ever run littblock with the old name persisted — so it +// exists for dev, CI, and devnet homes written before the rename. The operator action is to delete +// the directory (or move it aside); this check can be deleted once no such directory remains. +// +// A root that cannot be stat'd is refused for the same reason it is refused when the directory is +// present: an unreadable root cannot rule out data hiding under the old name. +func refuseLegacyTable(paths []string) error { + for _, root := range paths { + legacy := filepath.Join(root, legacyTableName) + switch _, err := os.Stat(legacy); { + case err == nil: + return fmt.Errorf( + "block db: found a pre-rename %q table at %s; the table is now named %q, so those "+ + "blocks and QCs would be neither served nor reclaimed. Delete or move the "+ + "directory aside to start from an empty store", + legacyTableName, legacy, tableName) + case !os.IsNotExist(err): + return fmt.Errorf("block db: check for a pre-rename %q table at %s: %w", + legacyTableName, legacy, err) + } + } + return nil +} + // recoverCursors reloads the write-order cursors (lastBlockNumber, lastQCNext, // and their presence flags) from on-disk state. Without this, a reopened DB // would treat itself as empty and let WriteBlock/WriteQC silently accept diff --git a/sei-db/ledger_db/block/littblock/litt_block_gc.go b/sei-db/ledger_db/block/littblock/litt_block_gc.go index 73a514fd9e..3d23c31f7b 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_gc.go +++ b/sei-db/ledger_db/block/littblock/litt_block_gc.go @@ -16,14 +16,15 @@ func (s *blockDB) Name() string { } // ExternalPruning is unconditionally true: this store has no pruner of its own for the collector to -// collide with. LittDB's GC reclaims what PruneBefore has already released, on a config.Retention -// timer, so it enforces no retention policy — it only carries out one this store has recorded. +// collide with. LittDB's GC reclaims what PruneBefore has already released, on a +// config.RetentionTime timer, so it enforces no retention policy — it only carries out one this +// store has recorded. func (s *blockDB) ExternalPruning() bool { return true } // PruneBelow advances the retention watermark to blockNumber. It only records the watermark; -// reclamation happens on LittDB's own GC schedule and no earlier than config.Retention (see +// reclamation happens on LittDB's own GC schedule and no earlier than config.RetentionTime (see // PruneBefore). // // blockNumber is a minimum shared across every managed store, so it may sit above this store's diff --git a/sei-db/ledger_db/block/littblock/litt_block_gc_test.go b/sei-db/ledger_db/block/littblock/litt_block_gc_test.go index 450a466699..c6079bfef7 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_gc_test.go +++ b/sei-db/ledger_db/block/littblock/litt_block_gc_test.go @@ -18,7 +18,7 @@ import ( func gcConfig(t *testing.T, dir string) *BlockDBConfig { cfg, err := DefaultConfig(dir) require.NoError(t, err) - cfg.Retention = time.Nanosecond + cfg.RetentionTime = time.Nanosecond cfg.Litt.GCPeriod = time.Hour cfg.Litt.Fsync = false return cfg diff --git a/sei-db/ledger_db/block/littblock/litt_block_legacy_table_test.go b/sei-db/ledger_db/block/littblock/litt_block_legacy_table_test.go new file mode 100644 index 0000000000..2559f7327d --- /dev/null +++ b/sei-db/ledger_db/block/littblock/litt_block_legacy_table_test.go @@ -0,0 +1,63 @@ +package littblock + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// A home written before the table rename must not open as an empty store. The data under the old +// name is unreachable either way; the only question is whether the operator is told, and a healthy +// empty store tells them nothing until the history they wanted is already gone. +func TestOpenRefusesAPreRenameTable(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, legacyTableName, "segments"), 0o700)) + + _, err := NewBlockDB(gcConfig(t, dir)) + require.ErrorContains(t, err, legacyTableName) + require.ErrorContains(t, err, "Delete or move the directory aside") +} + +// The refusal happens before littdb builds anything, so a refused open leaves the directory as it +// found it. Otherwise the operator inspecting the failure would find a new empty table sitting +// next to the old one, and could not tell which of the two the error was about. +func TestRefusedOpenLeavesTheDirectoryUntouched(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, legacyTableName, "segments"), 0o700)) + + _, err := NewBlockDB(gcConfig(t, dir)) + require.Error(t, err) + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Len(t, entries, 1, "nothing may be created beside the legacy table") + require.Equal(t, legacyTableName, entries[0].Name()) +} + +// The guard keys on the old name specifically: a normal home has no such directory and must open, +// and the check must not be satisfied by the store's own table. +func TestOpenAcceptsAHomeWithoutAPreRenameTable(t *testing.T) { + dir := t.TempDir() + + db, err := NewBlockDB(gcConfig(t, dir)) + require.NoError(t, err) + require.NoError(t, db.Close()) + + require.DirExists(t, filepath.Join(dir, tableName), "the store builds its table under the current name") + + db2, err := NewBlockDB(gcConfig(t, dir)) + require.NoError(t, err, "reopening a store it just wrote must not trip the guard") + require.NoError(t, db2.Close()) +} + +// Every root is checked, not just the first: littdb spreads a table across all of them, so data +// under the old name in any one of them is data this store cannot reach. +func TestOpenRefusesAPreRenameTableInANonFirstRoot(t *testing.T) { + first, second := t.TempDir(), t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(second, legacyTableName), 0o700)) + + require.NoError(t, refuseLegacyTable([]string{first})) + require.ErrorContains(t, refuseLegacyTable([]string{first, second}), second) +} diff --git a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go index ec070534ff..002433a083 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go +++ b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go @@ -18,7 +18,7 @@ import ( func strandingConfig(t *testing.T, dir string, maxSegmentKeyCount uint32) *BlockDBConfig { cfg, err := DefaultConfig(dir) require.NoError(t, err) - cfg.Retention = time.Nanosecond + cfg.RetentionTime = time.Nanosecond cfg.Litt.TargetSegmentFileSize = math.MaxUint32 cfg.Litt.MaxSegmentKeyCount = maxSegmentKeyCount cfg.Litt.GCPeriod = time.Hour diff --git a/sei-db/ledger_db/receipt/litt_receipt_gc.go b/sei-db/ledger_db/receipt/litt_receipt_gc.go index cda5ac11bc..9c42fd5011 100644 --- a/sei-db/ledger_db/receipt/litt_receipt_gc.go +++ b/sei-db/ledger_db/receipt/litt_receipt_gc.go @@ -29,9 +29,10 @@ func (s *littReceiptStore) ExternalPruning() bool { } // PruneBelow advances the retention floor to blockNumber and drops the tag-index entries below -// it. Receipt bodies are not deleted here: litt expires them by TTL, and reads below the floor -// return not-found in the meantime (see belowRetentionFloor), so visible retention follows this -// call even when reclamation lags it. +// it. Receipt bodies are not deleted here: advancing the floor is what releases them to litt's GC, +// which reclaims them once they are also past the TTL (see gcFilter). Reads below the floor return +// not-found in the meantime (see belowRetentionFloor), so visible retention follows this call even +// when reclamation lags it — and because the floor gates reclamation, it can never lead it. func (s *littReceiptStore) PruneBelow(blockNumber uint64) error { return s.pruneBlocksBelow(blockNumber) } diff --git a/sei-db/ledger_db/receipt/litt_receipt_gc_test.go b/sei-db/ledger_db/receipt/litt_receipt_gc_test.go index 6ac65ed895..3a61d18000 100644 --- a/sei-db/ledger_db/receipt/litt_receipt_gc_test.go +++ b/sei-db/ledger_db/receipt/litt_receipt_gc_test.go @@ -149,7 +149,8 @@ func TestReceiptGCLatestBlock(t *testing.T) { // A contiguous store answers the cut line it was given whatever it holds, and the prune that // follows moves the retention floor to it — which is what makes the receipts below it stop being -// served, even though litt reclaims their bodies later on its own TTL schedule. +// served. Reclaiming their bodies lags that, since litt also waits for the TTL, but it can no +// longer lead it (see TestGCFilterMakesLittReclamationFollowTheBlockFloor). func TestReceiptGCPruningBoundaryAndPruneBelow(t *testing.T) { store, prunable, ctx := setupLittIdxForGC(t, 0) addr := common.HexToAddress("0xabcd") diff --git a/sei-db/ledger_db/receipt/litt_receipt_gcfilter_internal_test.go b/sei-db/ledger_db/receipt/litt_receipt_gcfilter_internal_test.go new file mode 100644 index 0000000000..2b98ce7d16 --- /dev/null +++ b/sei-db/ledger_db/receipt/litt_receipt_gcfilter_internal_test.go @@ -0,0 +1,189 @@ +package receipt + +import ( + "testing" + "time" + + "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt" + "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/littbuilder" + "github.com/stretchr/testify/require" +) + +// floorStore returns a bare store whose retention floor is set, which is all gcFilter reads. +// Reclamation decisions are a pure function of that floor and the key, so nothing else is opened. +func floorStore(floor int64) *littReceiptStore { + s := &littReceiptStore{} + s.earliestVersion.Store(floor) + return s +} + +func requireFilter(t *testing.T, s *littReceiptStore, key []byte, isPrimary, want bool) { + t.Helper() + got, err := s.gcFilter(key, isPrimary) + require.NoError(t, err) + require.Equal(t, want, got) +} + +// The point of the filter: a body at or above the floor is not reclaimable however old it is, so a +// TTL sized for the wrong window cannot delete what reads still serve. +func TestGCFilterHoldsBodiesAtOrAboveTheFloor(t *testing.T) { + s := floorStore(100) + + requireFilter(t, s, littPartKey(99, 0), true, true) + requireFilter(t, s, littPartKey(100, 0), true, false) + requireFilter(t, s, littPartKey(101, 0), true, false) +} + +// Every part of a block shares its fate, since the floor is a block boundary and a partially +// reclaimed block would serve some of its receipts and not others. +func TestGCFilterTreatsEveryPartOfABlockAlike(t *testing.T) { + s := floorStore(100) + + for _, part := range []uint32{0, 1, 7} { + requireFilter(t, s, littPartKey(99, part), true, true) + requireFilter(t, s, littPartKey(100, part), true, false) + } +} + +// Tx-hash secondaries alias a body in their own segment, so the body's part key is what gates the +// segment. A secondary that blocked on its own would pin segments by tx hash, which carries no +// block number to compare against the floor. +func TestGCFilterPassesSecondaryKeys(t *testing.T) { + s := floorStore(100) + requireFilter(t, s, make([]byte, 32), false, true) +} + +// An unestablished floor must block rather than permit: 0 means nothing has been pruned yet, or +// the index has not been read, and reclaiming on that reading is not recoverable. +func TestGCFilterBlocksEverythingWithoutAFloor(t *testing.T) { + for _, floor := range []int64{0, -1} { + s := floorStore(floor) + requireFilter(t, s, littPartKey(0, 0), true, false) + requireFilter(t, s, littPartKey(1_000_000, 0), true, false) + } +} + +// litt requires the filter to be monotonic: once true for a key, always true. The floor only +// advances, so this holds — assert it across an advancing floor rather than trusting the field. +func TestGCFilterIsMonotonicAsTheFloorAdvances(t *testing.T) { + s := floorStore(0) + key := littPartKey(50, 0) + + var everTrue bool + for _, floor := range []int64{0, 10, 50, 51, 500} { + s.earliestVersion.Store(floor) + got, err := s.gcFilter(key, true) + require.NoError(t, err) + if everTrue { + require.True(t, got, "the filter went back on a key it already released at floor %d", floor) + } + everTrue = everTrue || got + } + require.True(t, everTrue, "an advancing floor must eventually release the key") +} + +// gcFilterTable builds a real litt table wired to s.gcFilter, sized so reclamation is observable in +// a unit test: segments seal on every write, and the TTL is already expired the moment a segment is +// sealed. That leaves the retention floor as the only thing deciding what may be reclaimed, which +// is the property under test. +// +// The receipt store's own segment thresholds (512MB) never seal in a test, so this drives litt +// directly rather than through newLittReceiptStore. What it cannot prove is that the constructor +// attaches the filter; the tests above and the wiring at the BuildTable call cover that separately. +func gcFilterTable(t *testing.T, s *littReceiptStore) litt.ManagedTable { + t.Helper() + + cfg, err := litt.DefaultConfig(t.TempDir()) + require.NoError(t, err) + cfg.TargetSegmentFileSize = 1 // seal on every write, so each block lands in a collectable segment + cfg.GCPeriod = time.Hour // only the explicit RunGC below collects, so nothing races a timer + cfg.Fsync = false + + db, err := littbuilder.NewDB(cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + + tableConfig := litt.DefaultTableConfig(littReceiptTableName) + tableConfig.ShardingFactor = 1 + tableConfig.TTL = time.Nanosecond // the age gate is satisfied at once; only the floor is left + tableConfig.GCFilter = s.gcFilter + table, err := db.BuildTable(tableConfig) + require.NoError(t, err) + + // RunGC lives on ManagedTable; without it a pass only happens on GCPeriod, which this test + // deliberately pushes out of reach so the assertions are not racing a timer. + managed, ok := table.(litt.ManagedTable) + require.True(t, ok, "forcing a GC pass requires a ManagedTable") + return managed +} + +func requireBlockRetained(t *testing.T, table litt.Table, block uint64) { + t.Helper() + ok, err := table.Exists(littPartKey(block, 0)) + require.NoError(t, err) + require.True(t, ok, "block %d was reclaimed while the retention floor still covered it", block) +} + +// Reclamation is not synchronous with a GC pass: the pass schedules keymap deletes and the control +// loop drops the files once they are durable. Poll rather than assert straight after RunGC. +func requireBlockReclaimed(t *testing.T, table litt.Table, block uint64) { + t.Helper() + require.Eventually(t, func() bool { + ok, err := table.Exists(littPartKey(block, 0)) + require.NoError(t, err) + return !ok + }, 5*time.Second, 10*time.Millisecond, "block %d stayed readable after the floor passed it", block) +} + +// The end of the chain this change exists to close: pruning by block number is what releases receipt +// bodies to litt, and an expired TTL alone does not. Without the filter every block here would be +// reclaimable from the very first pass, since the TTL expires the moment a segment seals. +// +// The two directions are not symmetric, and the test asserts them at different strengths on purpose. +// Retention is exact — a block at or above the floor must never be reclaimed, and that is the +// guarantee the collector depends on. Reclamation is only eventual and segment-granular: litt frees +// whole segments, so a segment holding one block above the floor keeps every block below it in that +// same segment. Blocks just under the floor may therefore survive a pass, which is why the +// reclaimed-side assertion leaves a margin rather than pinning the block after the floor. +func TestGCFilterMakesLittReclamationFollowTheBlockFloor(t *testing.T) { + const ( + head = 12 + floor = 10 + margin = 5 // comfortably below any segment that could straddle the floor + ) + + s := floorStore(0) + table := gcFilterTable(t, s) + + for block := uint64(1); block <= head; block++ { + require.NoError(t, table.Put(littPartKey(block, 0), []byte{byte(block)})) + } + require.NoError(t, table.Flush()) + + // No floor established and the TTL long expired: nothing may go. This is the case the filter + // exists for — every one of these is TTL-reclaimable and only the floor is holding it. + require.NoError(t, table.RunGC()) + for block := uint64(1); block <= head; block++ { + requireBlockRetained(t, table, block) + } + + // Pruning by block number is what releases them. + s.earliestVersion.Store(floor) + require.NoError(t, table.RunGC()) + + for block := uint64(1); block <= margin; block++ { + requireBlockReclaimed(t, table, block) + } + for block := uint64(floor); block <= head; block++ { + requireBlockRetained(t, table, block) + } +} + +// A primary key that is not a part key means the table's key layout changed without this filter +// being updated. Guessing a block number out of it would reclaim by coincidence, so litt's contract +// (an error crashes the DB loudly) is the right response. +func TestGCFilterRejectsAnUnknownPrimaryKey(t *testing.T) { + s := floorStore(100) + _, err := s.gcFilter(make([]byte, 32), true) + require.ErrorContains(t, err, "unexpected primary receipt key length") +} diff --git a/sei-db/ledger_db/receipt/litt_receipt_store.go b/sei-db/ledger_db/receipt/litt_receipt_store.go index a448767bf7..f8a079eb36 100644 --- a/sei-db/ledger_db/receipt/litt_receipt_store.go +++ b/sei-db/ledger_db/receipt/litt_receipt_store.go @@ -49,10 +49,11 @@ import ( // the most recent receipt bodies while the index still lists them — tolerable // for auxiliary, non-consensus RPC data, since reads return not-found for a // missing body. The tight interval is only affordable because litt flushes its -// keymap asynchronously off the control loop. Retention: receipt values -// expire via litt's per-table TTL (time based), tag keys are pruned by block -// range, and reads enforce the retention floor, so visible retention never -// exceeds that floor regardless of GC timing. +// keymap asynchronously off the control loop. Retention: tag keys are pruned by +// block range, reads enforce the retention floor, and receipt bodies are +// reclaimed once they are both below that floor and older than litt's per-table +// TTL (see gcFilter). Visible retention therefore never exceeds the floor, and +// reclamation never runs ahead of it. // // Retention has two possible drivers and exactly one runs at a time, selected by // cfg.ExternalPruning: @@ -73,8 +74,9 @@ import ( // fact. // // KeepRecent is what this store asks for either way: the pruner's window when it -// runs, the collector's retention window when it does not, and the litt TTL in -// both cases. +// runs, and the collector's retention window when it does not. It does not reach +// litt's TTL, which is a flat age floor (littRetentionTime) — how much history is +// kept is the floor's business, and only gcFilter releases a body to be reclaimed. type littReceiptStore struct { values litt.DB receipts litt.Table @@ -110,12 +112,18 @@ const ( // control loop; without that, a per-block flush regresses write throughput // badly (≈-48% observed before the async keymap landed). littFlushInterval = 5 * time.Millisecond - // littTTLPerBlock converts the KeepRecent block count into litt's wall-clock - // TTL (KeepRecent * littTTLPerBlock). Set above Giga block times so the TTL - // over-retains; only a sustained block time above this would expire a body - // still inside the height-based KeepRecent window, which reads mask as - // not-found (the earliest-version floor is authoritative). - littTTLPerBlock = 2 * time.Second + // littRetentionTime is the failsafe minimum age before a receipt body may be + // reclaimed, litt's per-table TTL. It is the same role BlockDBConfig.RetentionTime + // plays, and is deliberately a flat duration rather than a function of any block + // count: how much history this store keeps is decided by the retention floor, and + // gcFilter will not release a body until the floor has passed it. + // + // It was previously KeepRecent × 2s, which made the TTL a second, disagreeing + // answer to a question the floor already answers — and one that came out short + // under ExternalPruning, where the enforced retention is RollbackWindow + + // KeepRecent. An age floor cannot be wrong that way, because it does not claim to + // know how many blocks anything is. + littRetentionTime = time.Hour littPartCountLen = 4 ) @@ -129,6 +137,39 @@ func littPartKey(blockNumber uint64, part uint32) []byte { return key } +// gcFilter makes the retention floor a precondition for reclaiming a receipt body, so litt's TTL +// can only ever reclaim what the floor has already released. +// +// Without it the TTL is the sole reclaimer (a nil filter leaves TTL as the only condition), and it +// is derived from KeepRecent alone — which is not the retention this store owes anyone. Under +// ExternalPruning the collector holds the floor at RollbackWindow + KeepRecent blocks back, so a +// TTL sized for KeepRecent can expire bodies the floor still calls live, and a read inside the +// rollback window returns not-found. That is the cross-store guarantee the collector exists to +// provide, so the two are joined here rather than kept in agreement by arithmetic: what reads +// enforce and what GC reclaims are now the same fact, whatever the block time or the window. +// +// Only primary part keys gate. Tx-hash secondaries alias a body in their own segment, so the +// body's part key is what holds the segment back — the same division littblock draws between its +// number keys and its header-hash aliases. +// +// A floor of 0 blocks everything: it means no floor has been established (nothing pruned yet, or +// the index has not been read), and refusing to reclaim is the recoverable direction. Monotonic as +// the filter contract requires, because the floor only ever advances. +func (s *littReceiptStore) gcFilter(key []byte, isPrimaryKey bool) (bool, error) { + if !isPrimaryKey { + return true, nil + } + if len(key) != blockNumLen+littPartCountLen { + return false, fmt.Errorf("unexpected primary receipt key length %d (want %d)", + len(key), blockNumLen+littPartCountLen) + } + floor := s.earliestVersion.Load() + if floor <= 0 { + return false, nil + } + return binary.BigEndian.Uint64(key[:blockNumLen]) < uint64(floor), nil //nolint:gosec // guarded positive above +} + func newLittReceiptStore(cfg dbconfig.ReceiptStoreConfig, storeKey sdk.StoreKey) (ReceiptStore, error) { if err := os.MkdirAll(cfg.DBDirectory, 0o750); err != nil { return nil, fmt.Errorf("failed to create receipt store directory: %w", err) @@ -151,18 +192,40 @@ func newLittReceiptStore(cfg dbconfig.ReceiptStoreConfig, storeKey sdk.StoreKey) if err != nil { return nil, fmt.Errorf("failed to open littdb: %w", err) } + // getLogs per-query block fan-out; non-positive config falls back to the default. + logFilterParallelism := cfg.LogFilterParallelism + if logFilterParallelism <= 0 { + logFilterParallelism = dbconfig.DefaultReceiptLogFilterParallelism + } + + // The store is built before its table because the table's GC filter is a method on it. Until + // the floor is loaded from the index below, gcFilter reads 0 and blocks reclamation, which is + // the safe direction for a filter that does not yet know its floor. + s := &littReceiptStore{ + values: values, + storeKey: storeKey, + keepRecent: int64(cfg.KeepRecent), + pruneInterval: int64(cfg.PruneIntervalSeconds), + externalPruning: cfg.ExternalPruning, + logFilterParallelism: logFilterParallelism, + stopBackground: make(chan struct{}), + } + tableConfig := litt.DefaultTableConfig(littReceiptTableName) tableConfig.ShardingFactor = 1 // single shard: flushing one file is cheaper; sharding mainly helps across multiple disks + tableConfig.GCFilter = s.gcFilter receipts, err := values.BuildTable(tableConfig) if err != nil { _ = values.Close() return nil, fmt.Errorf("failed to open littdb receipts table: %w", err) } - if cfg.KeepRecent > 0 { - if err := receipts.SetTTL(time.Duration(cfg.KeepRecent) * littTTLPerBlock); err != nil { - _ = values.Close() - return nil, fmt.Errorf("failed to set littdb ttl: %w", err) - } + s.receipts = receipts + // Set unconditionally, including when nothing prunes this store. A TTL is necessary for litt to + // collect at all but no longer sufficient, so on a store whose floor never advances gcFilter + // refuses every key and nothing is reclaimed regardless of age. + if err := receipts.SetTTL(littRetentionTime); err != nil { + _ = values.Close() + return nil, fmt.Errorf("failed to set littdb ttl: %w", err) } indexCfg := pebbledb.DefaultConfig() @@ -172,24 +235,8 @@ func newLittReceiptStore(cfg dbconfig.ReceiptStoreConfig, storeKey sdk.StoreKey) _ = values.Close() return nil, fmt.Errorf("failed to open receipt log index: %w", err) } + s.index = index - // getLogs per-query block fan-out; non-positive config falls back to the default. - logFilterParallelism := cfg.LogFilterParallelism - if logFilterParallelism <= 0 { - logFilterParallelism = dbconfig.DefaultReceiptLogFilterParallelism - } - - s := &littReceiptStore{ - values: values, - receipts: receipts, - index: index, - storeKey: storeKey, - keepRecent: int64(cfg.KeepRecent), - pruneInterval: int64(cfg.PruneIntervalSeconds), - externalPruning: cfg.ExternalPruning, - logFilterParallelism: logFilterParallelism, - stopBackground: make(chan struct{}), - } s.latestVersion.Store(s.readMeta(receiptLatestVersionKey)) s.earliestVersion.Store(s.readMeta(receiptEarliestVersionKey)) s.startPruning() @@ -466,8 +513,10 @@ func (s *littReceiptStore) startPruning() { } // pruneBlocksBelow deletes the tag entries in [earliest, cutoff) and advances -// the retention floor. Receipt values are reclaimed independently by litt's TTL -// GC; the read-time floor keeps them invisible in the meantime. +// the retention floor. Receipt bodies are not deleted here: advancing the floor +// is what releases them to litt's GC, which reclaims a segment once every body +// in it is below the floor and past the TTL (see gcFilter). The read-time floor +// keeps them invisible in the meantime. // // Shared by both retention drivers: startPruning above, and the collector via // PruneBelow. Exactly one of them is live — see the type doc. diff --git a/sei-db/state_db/sc/flatkv/config/config.go b/sei-db/state_db/sc/flatkv/config/config.go index 18852ba9cf..3a2d56bb1a 100644 --- a/sei-db/state_db/sc/flatkv/config/config.go +++ b/sei-db/state_db/sc/flatkv/config/config.go @@ -70,7 +70,7 @@ type Config struct { // more snapshots than the count-based default. // // Default: false - ExternalPruning bool `mapstructure:"external-pruning"` + ExternalPruning bool `mapstructure:"-"` // EnablePebbleMetrics defines if the Pebble metrics should be enabled. // Default: true diff --git a/sei-db/state_db/sc/flatkv/store_gc.go b/sei-db/state_db/sc/flatkv/store_gc.go index c9e8310e6b..dc3631a869 100644 --- a/sei-db/state_db/sc/flatkv/store_gc.go +++ b/sei-db/state_db/sc/flatkv/store_gc.go @@ -88,12 +88,17 @@ func (s *CommitStore) PruneBelow(blockNumber uint64) error { return false, nil } pruned++ - logger.Info("pruned snapshot below retention floor", "version", version, "floor", blockNumber) return false, nil }) if scanErr != nil { errs = errors.Join(errs, fmt.Errorf("scan snapshots: %w", scanErr)) } + // One line per cycle rather than one per snapshot: the count is the whole story here, since the + // set pruned is always "everything below the floor". A cycle that prunes nothing is the common + // case and says nothing, so it stays silent. + if pruned > 0 { + logger.Info("pruned snapshots below retention floor", "count", pruned, "floor", blockNumber) + } return errs } diff --git a/sei-tendermint/config/autobahn.go b/sei-tendermint/config/autobahn.go index 27b9590191..b4c7b97042 100644 --- a/sei-tendermint/config/autobahn.go +++ b/sei-tendermint/config/autobahn.go @@ -134,7 +134,7 @@ func (c AutobahnBlockDBConfig) LittBlockConfig(dir string) (littblock.BlockDBCon return littblock.BlockDBConfig{}, fmt.Errorf("littblock.DefaultConfig: %w", err) } if r, ok := c.Retention.Get(); ok { - cfg.Retention = r.Duration() + cfg.RetentionTime = r.Duration() } if p, ok := c.GCPeriod.Get(); ok { cfg.Litt.GCPeriod = p.Duration() diff --git a/sei-tendermint/config/autobahn_test.go b/sei-tendermint/config/autobahn_test.go index b6e4b815a7..d9bf43f679 100644 --- a/sei-tendermint/config/autobahn_test.go +++ b/sei-tendermint/config/autobahn_test.go @@ -38,7 +38,7 @@ func TestAutobahnBlockDBConfig_LittBlockConfig(t *testing.T) { GCPeriod: utils.Some(utils.Duration(wantGCPeriod)), }).LittBlockConfig(dir) require.NoError(t, err) - require.Equal(t, wantRetention, cfg.Retention) + require.Equal(t, wantRetention, cfg.RetentionTime) require.Equal(t, wantGCPeriod, cfg.Litt.GCPeriod) require.True(t, cfg.Litt.Fsync) } diff --git a/sei-tendermint/internal/autobahn/data/state_recovery_test.go b/sei-tendermint/internal/autobahn/data/state_recovery_test.go index 6e7ebe9e31..251c1b51c0 100644 --- a/sei-tendermint/internal/autobahn/data/state_recovery_test.go +++ b/sei-tendermint/internal/autobahn/data/state_recovery_test.go @@ -440,7 +440,7 @@ func TestRecoveryAfterPruneNoGC(t *testing.T) { // Write both QCs and all their blocks to the DB. cfg1 := utils.OrPanic1(littblock.DefaultConfig(dir)) - cfg1.Retention = time.Nanosecond + cfg1.RetentionTime = time.Nanosecond db1 := utils.OrPanic1(littblock.NewBlockDB(cfg1)) writeToBlockDB(t, db1, []*types.FullCommitQC{qc1, qc2}, [][]*types.Block{blocks1, blocks2}) @@ -450,7 +450,7 @@ func TestRecoveryAfterPruneNoGC(t *testing.T) { // Reopen the same dir without ForceGC — pruned entries may still be present. cfg2 := utils.OrPanic1(littblock.DefaultConfig(dir)) - cfg2.Retention = time.Nanosecond + cfg2.RetentionTime = time.Nanosecond db2 := utils.OrPanic1(littblock.NewBlockDB(cfg2)) t.Cleanup(func() { _ = db2.Close() }) diff --git a/sei-tendermint/internal/autobahn/data/state_test.go b/sei-tendermint/internal/autobahn/data/state_test.go index 23ae335fb7..d1893ef689 100644 --- a/sei-tendermint/internal/autobahn/data/state_test.go +++ b/sei-tendermint/internal/autobahn/data/state_test.go @@ -52,7 +52,7 @@ func snapshot(s *State) Snapshot { func newTestBlockDB(t *testing.T, dir string) types.BlockDB { t.Helper() cfg := utils.OrPanic1(littblock.DefaultConfig(dir)) - cfg.Retention = time.Nanosecond + cfg.RetentionTime = time.Nanosecond db := utils.OrPanic1(littblock.NewBlockDB(cfg)) t.Cleanup(func() { _ = db.Close() }) return db diff --git a/sei-tendermint/node/setup_test.go b/sei-tendermint/node/setup_test.go index 62668485a0..8aed49ae95 100644 --- a/sei-tendermint/node/setup_test.go +++ b/sei-tendermint/node/setup_test.go @@ -119,7 +119,7 @@ func TestBuildGigaConfig_BlockDBOverrides(t *testing.T) { littCfg, err := fc.BlockDB.LittBlockConfig(filepath.Join(dir, "blockdb")) require.NoError(t, err) require.NotNil(t, littCfg.Litt) - assert.Equal(t, 30*time.Second, littCfg.Retention) + assert.Equal(t, 30*time.Second, littCfg.RetentionTime) assert.Equal(t, 5*time.Second, littCfg.Litt.GCPeriod) assert.True(t, littCfg.Litt.Fsync) } @@ -141,7 +141,7 @@ func TestBuildGigaConfig_BlockDBOmittedKeepsDefaults(t *testing.T) { littCfg, err := config.AutobahnBlockDBConfig{}.LittBlockConfig(filepath.Join(dir, "blockdb")) require.NoError(t, err) require.NotNil(t, littCfg.Litt) - assert.Equal(t, 24*time.Hour, littCfg.Retention) + assert.Equal(t, time.Hour, littCfg.RetentionTime) assert.True(t, littCfg.Litt.Fsync) } From 9b96d16e8588df5b8005ec2cb14c735f96eac487 Mon Sep 17 00:00:00 2001 From: YimingZang Date: Mon, 10 Aug 2026 07:28:58 -0700 Subject: [PATCH 08/11] Address comments --- app/seidb_test.go | 6 +- sei-db/config/receipt_config.go | 23 ++---- sei-db/config/testdata/receipt-store.golden | 2 +- .../block/littblock/litt_block_config.go | 20 +++++- .../gc/storage_garbage_collector_config.go | 16 +++++ sei-db/seiwal/seiwal.go | 6 +- sei-db/state_db/sc/flatkv/store_gc.go | 71 ++++++++++++++----- 7 files changed, 98 insertions(+), 46 deletions(-) diff --git a/app/seidb_test.go b/app/seidb_test.go index 48cbcb0617..f089826f2b 100644 --- a/app/seidb_test.go +++ b/app/seidb_test.go @@ -252,9 +252,7 @@ func TestParseReceiptConfigs_UsesConfiguredBackend(t *testing.T) { assert.NoError(t, err) assert.Equal(t, "pebbledb", receiptConfig.Backend) assert.Equal(t, config.DefaultReceiptStoreConfig().AsyncWriteBuffer, receiptConfig.AsyncWriteBuffer) - // Left at the default: no [receipt-store] key reaches KeepRecent. What a node retains is - // set later, by readReceiptStoreConfig from min-retain-blocks. - assert.Equal(t, config.DefaultReceiptKeepRecent, receiptConfig.KeepRecent) + assert.Equal(t, 0, receiptConfig.KeepRecent) } func TestParseReceiptConfigs_UsesConfiguredValues(t *testing.T) { @@ -269,7 +267,7 @@ func TestParseReceiptConfigs_UsesConfiguredValues(t *testing.T) { assert.Equal(t, "/tmp/custom-receipt-db", receiptConfig.DBDirectory) assert.Equal(t, "pebbledb", receiptConfig.Backend) assert.Equal(t, 7, receiptConfig.AsyncWriteBuffer) - assert.Equal(t, config.DefaultReceiptKeepRecent, receiptConfig.KeepRecent) + assert.Equal(t, 0, receiptConfig.KeepRecent) assert.Equal(t, 9, receiptConfig.PruneIntervalSeconds) assert.True(t, receiptConfig.EnableReadWriteMetrics) } diff --git a/sei-db/config/receipt_config.go b/sei-db/config/receipt_config.go index f178ca0f72..a2dd8c18f0 100644 --- a/sei-db/config/receipt_config.go +++ b/sei-db/config/receipt_config.go @@ -27,12 +27,6 @@ const ( // littidx eth_getLogs (see ReceiptStoreConfig.LogFilterParallelism). const DefaultReceiptLogFilterParallelism = 16 -// DefaultReceiptKeepRecent is the default retention window in blocks for callers that -// build a ReceiptStoreConfig directly. A seid node never uses it — see -// ReceiptStoreConfig.KeepRecent for why — so it is sized for a tool or test that wants -// bounded growth, not to express what a node should retain. -const DefaultReceiptKeepRecent = 10000 - // ReceiptStoreConfig defines configuration for the receipt store database. type ReceiptStoreConfig struct { // DBDirectory defines the directory to store the receipt store db files @@ -53,13 +47,8 @@ type ReceiptStoreConfig struct { // KeepRecent defines the number of versions to keep in receipt store. // Setting it to 0 means keep everything (no pruning). - // - // This is NOT read from receipt-store config, and on a seid node it is not read - // from DefaultReceiptStoreConfig either: readReceiptStoreConfig overwrites it - // unconditionally from the global min-retain-blocks flag, including with 0 when - // that flag is unset. The default below is therefore only reachable by callers - // that build this config directly — tools and tests — and changing it does not - // change what any node retains. + // This is NOT read from receipt-store config; it is always derived from + // the global min-retain-blocks flag at the app layer. KeepRecent int `mapstructure:"-"` // PruneIntervalSeconds defines the interval in seconds to trigger pruning @@ -97,15 +86,13 @@ type ReceiptStoreConfig struct { } // DefaultReceiptStoreConfig returns the default ReceiptStoreConfig. -// -// KeepRecent is a bounded default rather than 0 so that a caller building this config -// directly gets a store that prunes. It is not what a node retains: the app layer -// replaces it with min-retain-blocks before the store is opened (see KeepRecent). +// KeepRecent defaults to 0 (no pruning). The app layer is responsible +// for setting KeepRecent from the global min-retain-blocks flag. func DefaultReceiptStoreConfig() ReceiptStoreConfig { return ReceiptStoreConfig{ Backend: "pebbledb", AsyncWriteBuffer: DefaultSSAsyncBuffer, - KeepRecent: DefaultReceiptKeepRecent, + KeepRecent: 0, PruneIntervalSeconds: DefaultSSPruneInterval, LogFilterParallelism: DefaultReceiptLogFilterParallelism, } diff --git a/sei-db/config/testdata/receipt-store.golden b/sei-db/config/testdata/receipt-store.golden index 2abac0169b..e6891d7a05 100644 --- a/sei-db/config/testdata/receipt-store.golden +++ b/sei-db/config/testdata/receipt-store.golden @@ -1,7 +1,7 @@ DBDirectory = string("") Backend = string("pebbledb") AsyncWriteBuffer = int(100) -KeepRecent = int(10000) +KeepRecent = int(0) PruneIntervalSeconds = int(600) ExternalPruning = bool(false) EnableReadWriteMetrics = bool(false) diff --git a/sei-db/ledger_db/block/littblock/litt_block_config.go b/sei-db/ledger_db/block/littblock/litt_block_config.go index 5473a9dd07..038e90e2ed 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_config.go +++ b/sei-db/ledger_db/block/littblock/litt_block_config.go @@ -45,13 +45,29 @@ type BlockDBConfig struct { // to this store alone: a deep window here also holds back receiptDB and the SC/SS // snapshots. Must be >= gc.InfiniteRetentionWindow. // - // Independent of Retention, which is a wall-clock TTL failsafe underneath the watermark. + // With F = LatestBlock - RollbackWindow - RetentionWindow, garbage collection guarantees: + // + // 1. Nothing needed to roll back to any block in + // [LatestBlock - RollbackWindow, LatestBlock] is deleted. + // 2. No block or QC at or above F is deleted. So even after rolling back to + // LatestBlock - RollbackWindow, any of the last RetentionWindow blocks is still readable. + // 3. Blocks and QCs below F are eventually deleted — eventually, because reclamation also + // waits for RetentionTime and for LittDB's own GC to come round. + // + // Independent of RetentionTime, which is a wall-clock TTL failsafe underneath the watermark. // Both must permit reclamation before any record is dropped. RetentionWindow int64 } // DefaultConfig returns a BlockDBConfig preloaded with all defaults, rooted at // dir. Override fields as needed, then pass it to NewBlockDB (which validates). +// +// RetentionWindow defaults to 0 — no extra history beyond the collector's shared rollback +// window — because it is not a blockDB-local knob. The collector prunes every store to one +// minimum, so a non-zero default here would hold receiptDB, the state WAL and the SC snapshots +// that much further back too, on blockDB's say-so. Every other store in the fleet reports 0. +// A deployment that wants deeper block history sets this at the call site, where the fleet-wide +// cost of the choice is visible. func DefaultConfig(dir string) (*BlockDBConfig, error) { littConfig, err := littdb.DefaultConfig(dir) if err != nil { @@ -60,7 +76,7 @@ func DefaultConfig(dir string) (*BlockDBConfig, error) { return &BlockDBConfig{ Litt: littConfig, RetentionTime: time.Hour, - RetentionWindow: 10000, + RetentionWindow: 0, }, nil } diff --git a/sei-db/management/gc/storage_garbage_collector_config.go b/sei-db/management/gc/storage_garbage_collector_config.go index 0346bf8f3a..9b1e732fb3 100644 --- a/sei-db/management/gc/storage_garbage_collector_config.go +++ b/sei-db/management/gc/storage_garbage_collector_config.go @@ -11,6 +11,22 @@ type StorageGarbageCollectorConfig struct { // roll back to. Shared by every managed store so they stay mutually consistent; // per-store extras beyond this window use PrunableStore.GetRetentionWindow. // + // With R = the store's own GetRetentionWindow and F = LatestBlock - RollbackWindow - R, + // collection guarantees, for each managed store: + // + // 1. Nothing needed to roll back to any block in + // [LatestBlock - RollbackWindow, LatestBlock] is deleted. + // 2. No data at or above F is deleted. So even after rolling back to + // LatestBlock - RollbackWindow, the most recent R blocks are still readable. + // 3. Data below F is eventually deleted — eventually, because each store reclaims on its + // own schedule once the collector has released the range. + // + // A snapshot store reads these in terms of restore points rather than blocks: what it must + // hold is the newest snapshot at or below the boundary, which is what GetPruningBoundary + // answers. Guarantee 2 is also why pruning is to the shared minimum rather than to each + // store's own boundary — a retained snapshot is only restorable if the blocks that follow + // it survive in the contiguous stores. + // // 0 is allowed and waives the guarantee: cutLine then equals head for any store with // retention 0, so those stores are pruned to their head. It is safe only when every // participating store leaves enough history on its own — a positive GetRetentionWindow, diff --git a/sei-db/seiwal/seiwal.go b/sei-db/seiwal/seiwal.go index f1efaa2ab0..bb6c4aa1ad 100644 --- a/sei-db/seiwal/seiwal.go +++ b/sei-db/seiwal/seiwal.go @@ -65,9 +65,9 @@ type WAL[T any] interface { // sealed files only, so records may survive above the requested threshold until their containing file // is fully below it. // - // Unlike every other method here, PruneBefore may be called from a goroutine other than the WAL's - // owner, concurrently with any method including Append and Close, and implementations must support - // that without external serialization. Retention is driven by a garbage collector on its own + // Most methods here belong to the WAL's owner. PruneBefore does not: it may be called from another + // goroutine, concurrently with any method including Append and Close, and implementations must + // support that without external serialization. Retention is driven by a garbage collector on its own // goroutine, and requiring it to take the writer's turn would mean either blocking the writer or // deferring the prune until the writer next runs — the latter stalling reclamation indefinitely on a // WAL that has stopped receiving appends. diff --git a/sei-db/state_db/sc/flatkv/store_gc.go b/sei-db/state_db/sc/flatkv/store_gc.go index dc3631a869..3ab691f0e7 100644 --- a/sei-db/state_db/sc/flatkv/store_gc.go +++ b/sei-db/state_db/sc/flatkv/store_gc.go @@ -61,38 +61,39 @@ func (s *CommitStore) PruneBelow(blockNumber uint64) error { return nil } - dir := s.flatkvDir() - // The active snapshot is what the next open resolves to, so it is never a candidate however deep the // request. Nothing in the contract should produce such a request — a boundary is never above the // snapshot it names — but the cost of the check is one readlink against making a wrong answer // elsewhere unbootable here. - _, activeVersion, err := currentSnapshotDir(dir) + _, activeVersion, err := currentSnapshotDir(s.flatkvDir()) if err != nil { return fmt.Errorf("resolve active snapshot before pruning: %w", err) } + blocks, err := s.snapshotBlocks() + if err != nil { + return fmt.Errorf("scan snapshots: %w", err) + } + var errs error pruned := 0 - scanErr := traverseSnapshots(dir, true, func(version int64) (bool, error) { - if uint64(version) >= blockNumber { //nolint:gosec // snapshot versions are non-negative - return true, nil // ascending, so nothing further is a candidate + for _, block := range blocks { + if block >= blockNumber { + break // ascending, so nothing further is a candidate } - if version == activeVersion { - return false, nil + if block == uint64(activeVersion) { //nolint:gosec // snapshot versions are non-negative + continue } - if err := atomicRemoveDir(filepath.Join(dir, snapshotName(version))); err != nil { - if !os.IsNotExist(err) { - errs = errors.Join(errs, fmt.Errorf("remove snapshot %d: %w", version, err)) - } - return false, nil + removed, err := s.deleteSnapshot(block) + if err != nil { + errs = errors.Join(errs, err) + continue + } + if removed { + pruned++ } - pruned++ - return false, nil - }) - if scanErr != nil { - errs = errors.Join(errs, fmt.Errorf("scan snapshots: %w", scanErr)) } + // One line per cycle rather than one per snapshot: the count is the whole story here, since the // set pruned is always "everything below the floor". A cycle that prunes nothing is the common // case and says nothing, so it stays silent. @@ -102,6 +103,40 @@ func (s *CommitStore) PruneBelow(blockNumber uint64) error { return errs } +// snapshotBlocks returns the block number of every snapshot on disk, ascending. A missing snapshot +// directory yields no blocks rather than an error, matching traverseSnapshots: a store that has not +// snapshotted yet has nothing to prune, which is not a failure. +func (s *CommitStore) snapshotBlocks() ([]uint64, error) { + var blocks []uint64 + err := traverseSnapshots(s.flatkvDir(), true, func(version int64) (bool, error) { + if version >= 0 { + blocks = append(blocks, uint64(version)) + } + return false, nil + }) + if err != nil { + return nil, err + } + return blocks, nil +} + +// deleteSnapshot removes the snapshot directory for block, reporting whether this call is the one +// that removed it. +// +// An already-gone snapshot is success rather than an error: the SnapshotKeepRecent pruner in +// WriteSnapshot deletes from the same set, and losing that race means the work is done. It reports +// false in that case, so the cycle's count stays the number of snapshots this call reclaimed. +func (s *CommitStore) deleteSnapshot(block uint64) (bool, error) { + path := filepath.Join(s.flatkvDir(), snapshotName(int64(block))) //nolint:gosec // block numbers are bounded well below 2^63 + if err := atomicRemoveDir(path); err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, fmt.Errorf("remove snapshot %d: %w", block, err) + } + return true, nil +} + // GetRetentionWindow reports 0: this store asks for no history of its own beyond the collector's shared // rollback window, which is the contract for a snapshot store (see gc.PrunableStore). // From a682ec399fde9559ecb1989471ef1b565dfdf928 Mon Sep 17 00:00:00 2001 From: YimingZang Date: Mon, 10 Aug 2026 19:34:07 -0700 Subject: [PATCH 09/11] Refactor for gc api --- sei-db/config/giga_config.go | 5 +- .../block/littblock/litt_block_config.go | 56 +- .../block/littblock/litt_block_gc.go | 67 +- .../block/littblock/litt_block_gc_test.go | 82 +-- sei-db/ledger_db/receipt/litt_receipt_gc.go | 62 +- .../ledger_db/receipt/litt_receipt_gc_test.go | 84 ++- .../litt_receipt_pruner_internal_test.go | 2 +- .../ledger_db/receipt/litt_receipt_store.go | 23 +- sei-db/management/gc/api.go | 195 ++---- .../gc/storage_garbage_collector.go | 242 +++---- .../gc/storage_garbage_collector_config.go | 62 +- .../gc/storage_garbage_collector_test.go | 662 +++++++++--------- sei-db/state_db/sc/flatkv/snapshot.go | 14 +- sei-db/state_db/sc/flatkv/store_gc.go | 267 ++++--- sei-db/state_db/sc/flatkv/store_gc_test.go | 326 ++++++--- sei-db/state_db/statewal/state_wal_gc.go | 66 +- sei-db/state_db/statewal/state_wal_gc_test.go | 76 +- sei-db/state_db/statewal/state_wal_impl.go | 3 +- 18 files changed, 1177 insertions(+), 1117 deletions(-) diff --git a/sei-db/config/giga_config.go b/sei-db/config/giga_config.go index ae70cb095b..e7451d363f 100644 --- a/sei-db/config/giga_config.go +++ b/sei-db/config/giga_config.go @@ -14,9 +14,8 @@ import ( // callers build it via DefaultGigaStorageConfig. Nested knobs live on the store and // collector configs they already own (StateStoreConfig, ReceiptStoreConfig, // gc.StorageGarbageCollectorConfig) rather than being redeclared here — in particular -// RollbackWindow has a single source of truth in -// gc.DefaultStorageGarbageCollectorConfig; per-store extras live on each -// PrunableStore via GetRetentionWindow. +// RollbackWindow and LookbackWindow have a single source of truth in +// gc.DefaultStorageGarbageCollectorConfig, and cover every managed store at once. type GigaStorageConfig struct { HomePath string FlatKVConfig *flatkvConfig.Config diff --git a/sei-db/ledger_db/block/littblock/litt_block_config.go b/sei-db/ledger_db/block/littblock/litt_block_config.go index 038e90e2ed..06acdf328b 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_config.go +++ b/sei-db/ledger_db/block/littblock/litt_block_config.go @@ -5,7 +5,6 @@ import ( "time" littdb "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt" - "github.com/sei-protocol/sei-chain/sei-db/management/gc" ) // BlockDBConfig configures a LittDB-backed types.BlockDB. @@ -22,61 +21,24 @@ type BlockDBConfig struct { // watermark to advance past the record, so even an over-eager watermark // cannot delete data younger than RetentionTime. Must be positive. // - // It is an age floor, not a retention policy: how much history this store - // keeps is RetentionWindow below. Raising this only delays reclaiming what - // the watermark has already released, which costs disk and buys nothing the - // window does not already express. + // It is an age floor, not a retention policy: how much history this store keeps + // is the RollbackWindow and LookbackWindow on + // gc.StorageGarbageCollectorConfig, which cover every managed store at once. + // Raising this only delays reclaiming what the watermark has already released, + // which costs disk and buys nothing those windows do not already express. RetentionTime time.Duration - - // RetentionWindow is how much history this store keeps beyond the shared rollback - // window of the StorageGarbageCollector that manages it, in blocks. It is what - // gc.PrunableStore.GetRetentionWindow answers: - // - // > 0 → that many blocks of history beyond the rollback window - // 0 → keep history to serve rollback window only - // -1 → never prune this store (gc.InfiniteRetentionWindow) - // - // Zero does NOT mean "keep everything" here, unlike the KeepRecent fields on - // StateStoreConfig and ReceiptStoreConfig, where 0 disables pruning. It is the most - // aggressive setting this field has; "keep everything" is -1. Assigning a KeepRecent - // value to this field inverts the retention it asks for. - // - // This is an input to a minimum shared across every managed store, not a policy applied - // to this store alone: a deep window here also holds back receiptDB and the SC/SS - // snapshots. Must be >= gc.InfiniteRetentionWindow. - // - // With F = LatestBlock - RollbackWindow - RetentionWindow, garbage collection guarantees: - // - // 1. Nothing needed to roll back to any block in - // [LatestBlock - RollbackWindow, LatestBlock] is deleted. - // 2. No block or QC at or above F is deleted. So even after rolling back to - // LatestBlock - RollbackWindow, any of the last RetentionWindow blocks is still readable. - // 3. Blocks and QCs below F are eventually deleted — eventually, because reclamation also - // waits for RetentionTime and for LittDB's own GC to come round. - // - // Independent of RetentionTime, which is a wall-clock TTL failsafe underneath the watermark. - // Both must permit reclamation before any record is dropped. - RetentionWindow int64 } // DefaultConfig returns a BlockDBConfig preloaded with all defaults, rooted at // dir. Override fields as needed, then pass it to NewBlockDB (which validates). -// -// RetentionWindow defaults to 0 — no extra history beyond the collector's shared rollback -// window — because it is not a blockDB-local knob. The collector prunes every store to one -// minimum, so a non-zero default here would hold receiptDB, the state WAL and the SC snapshots -// that much further back too, on blockDB's say-so. Every other store in the fleet reports 0. -// A deployment that wants deeper block history sets this at the call site, where the fleet-wide -// cost of the choice is visible. func DefaultConfig(dir string) (*BlockDBConfig, error) { littConfig, err := littdb.DefaultConfig(dir) if err != nil { return nil, fmt.Errorf("failed to build litt config: %w", err) } return &BlockDBConfig{ - Litt: littConfig, - RetentionTime: time.Hour, - RetentionWindow: 0, + Litt: littConfig, + RetentionTime: time.Hour, }, nil } @@ -91,9 +53,5 @@ func (c *BlockDBConfig) Validate() error { if c.RetentionTime <= 0 { return fmt.Errorf("config.RetentionTime must be positive (got %s)", c.RetentionTime) } - if c.RetentionWindow < gc.InfiniteRetentionWindow { - return fmt.Errorf("config.RetentionWindow must be >= %d (got %d)", - gc.InfiniteRetentionWindow, c.RetentionWindow) - } return nil } diff --git a/sei-db/ledger_db/block/littblock/litt_block_gc.go b/sei-db/ledger_db/block/littblock/litt_block_gc.go index 3d23c31f7b..2bed9ef2fb 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_gc.go +++ b/sei-db/ledger_db/block/littblock/litt_block_gc.go @@ -5,10 +5,19 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" ) -// blockDB joins the shared prune cycle as a contiguous store: everything from its retention -// floor up to its head is retained, so it can serve a rollback to any height in between. The -// collector owns the decision of how deep to prune; PruneBefore stays the direct entry point -// for callers holding a types.BlockDB. +// In terms of the collector's RollbackWindow and LookbackWindow, garbage collection guarantees: +// +// 1. Garbage collection will not delete any data that is necessary to roll back to any block +// between LatestBlock and (LatestBlock - RollbackWindow), inclusive. +// 2. Garbage collection will not delete block DB data that is at or after +// (LatestBlock - RollbackWindow - LookbackWindow). This ensures that however far the system +// rolls back inside the rollback window, it is still possible to read at least LookbackWindow +// blocks of history below wherever it landed. +// 3. Garbage collection will eventually delete block data older than +// (LatestBlock - RollbackWindow - LookbackWindow). +// +// Eventually, in guarantee 3, because PruneHistory only records a watermark: reclamation also +// waits for BlockDBConfig.RetentionTime and for LittDB's own GC to come round. var _ gc.PrunableStore = (*blockDB)(nil) func (s *blockDB) Name() string { @@ -23,7 +32,7 @@ func (s *blockDB) ExternalPruning() bool { return true } -// PruneBelow advances the retention watermark to blockNumber. It only records the watermark; +// PruneHistory advances the retention watermark to blockNumber. It only records the watermark; // reclamation happens on LittDB's own GC schedule and no earlier than config.RetentionTime (see // PruneBefore). // @@ -31,39 +40,43 @@ func (s *blockDB) ExternalPruning() bool { // own head — a store that ingests ahead of blockDB pulls the head up, and a QC boundary can // leave the newest retained cohort below it. PruneBefore caps the request at the newest // retained (block, QC) pair, which is what keeps the store from emptying itself here. -func (s *blockDB) PruneBelow(blockNumber uint64) error { +func (s *blockDB) PruneHistory(blockNumber uint64) error { return s.PruneBefore(types.GlobalBlockNumber(blockNumber)) } -// GetRetentionWindow reports the configured history beyond the collector's shared rollback -// window. See BlockDBConfig.RetentionWindow for the meaning of each value, and note that it is -// an input to a fleet-wide minimum rather than a policy applied to this store alone. -func (s *blockDB) GetRetentionWindow() int64 { - if s.config.RetentionWindow < 0 { - return gc.InfiniteRetentionWindow - } - return s.config.RetentionWindow +// PruneSnapshots does nothing: blockDB keeps no snapshots. It restores by reading the blocks it +// holds, so its whole retention story is the watermark PruneHistory moves. +func (s *blockDB) PruneSnapshots(uint64) error { + return nil } -// GetPruningBoundary returns cutLine, the contract's answer for a contiguous store: every block -// at or above the watermark is retained, so cutLine itself is restorable and nothing below it -// has to be held back. +// GetRollbackFloor returns head - rollbackWindow, the contract's answer for a contiguous store: +// every block from its floor to its head is retained, so that height is restorable directly and +// nothing below it has to be held back. It keeps no snapshots, so there is nothing to resolve the +// window against beyond its own head. // -// Unconditional on purpose. A store whose floor already sits above cutLine — bootstrapped -// mid-chain, or pruned there by an earlier cycle — still answers cutLine, because the -// PruneBefore that follows is a no-op on it while a lower answer would hold back every other -// store. CannotServeRollback is never right here: this store fills from its own ingest path, -// so it has no replay range for another store's data to protect. -func (s *blockDB) GetPruningBoundary(cutLine uint64) uint64 { - return cutLine +// It reports against its own head even when that runs ahead of the fleet's, because the collector +// takes a minimum across stores. A lagging store therefore sets the depth, and answering high here +// cannot prune anything out from under it. +// +// 0 when the window is deeper than the whole history — including the empty store, whose head is 0. +// Nothing here is eligible for pruning yet: the rollback owed reaches past genesis, so no part of the +// history can be given up until the head clears the window. Nothing is logged from here; the +// collector logs every store's answer each cycle. +func (s *blockDB) GetRollbackFloor(rollbackWindow uint64) uint64 { + head, err := s.GetLatestBlock() + if err != nil || head <= rollbackWindow { + return 0 // cannot say what it holds, so nothing may be dropped anywhere + } + return head - rollbackWindow } // GetLatestBlock returns the newest block number written, or 0 when none has been. // // Global block numbers start at genesis block 0, so a store holding only that block is -// indistinguishable from an empty one and is excluded from the collector's head. That is the -// safe direction: it drops out of the head minimum rather than dragging every store's cut line -// to 0, and the prune it then receives is capped by PruneBefore to a no-op. +// indistinguishable from an empty one. That is the safe direction: GetRollbackFloor then answers 0, +// which holds the fleet's history where it is, and the prune it receives is capped by PruneBefore +// to a no-op. // // Reports the written cursor, not the flushed one. A block that a crash would lose still counts // as ingested — recovery re-derives this cursor from what survived, so the head can only move diff --git a/sei-db/ledger_db/block/littblock/litt_block_gc_test.go b/sei-db/ledger_db/block/littblock/litt_block_gc_test.go index c6079bfef7..73f8382247 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_gc_test.go +++ b/sei-db/ledger_db/block/littblock/litt_block_gc_test.go @@ -38,7 +38,7 @@ func openForGC(t *testing.T, dir string) (types.BlockDB, gc.PrunableStore) { // The head the collector reads is the newest block written — not the newest QC's coverage, since // a QC is written before the blocks it covers — and 0 while nothing has been ingested, so an -// empty store drops out of the head minimum instead of dragging every cut line to 0. The reopen +// empty store drops out of the head minimum instead of dragging the lookback floor to 0. The reopen // at the end covers recovery: a store that reported 0 after a restart would let the other stores // prune past a height this one still holds. func TestGCLatestBlock(t *testing.T) { @@ -71,62 +71,68 @@ func TestGCLatestBlock(t *testing.T) { require.Equal(t, uint64(20), latest, "the head must be re-derived on open") } -// The window the collector applies is the configured one, sentinel included: -1 has to survive -// the trip verbatim or an operator asking for "never prune" gets a 1-block window instead. -func TestGCRetentionWindowComesFromConfig(t *testing.T) { - for _, retentionWindow := range []int64{gc.InfiniteRetentionWindow, 100_000} { - cfg := gcConfig(t, t.TempDir()) - cfg.RetentionWindow = retentionWindow +// blockDB keeps no snapshots, so its half of the split contract is a no-op. It is still called +// every cycle, and answering with an error would fail a cycle that has nothing to do with it. +func TestGCPruneSnapshotsIsANoOp(t *testing.T) { + db, store := openForGC(t, t.TempDir()) + rng := utils.TestRngFromSeed(4) + writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19 - db, err := NewBlockDB(cfg) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, db.Close()) }) + require.NoError(t, store.PruneSnapshots(10)) + require.Equal(t, uint64(0), db.(*blockDB).watermark.Load(), + "pruning snapshots must not move the history watermark") - require.Equal(t, retentionWindow, db.(gc.PrunableStore).GetRetentionWindow()) - } + blk, err := db.ReadBlockByNumber(0) + require.NoError(t, err) + require.True(t, blk.IsPresent(), "no block may be dropped by a snapshot prune") } -// A contiguous store answers the cut line it was given whatever it holds. The states below are -// the ones where a snapshot store would answer differently, and answering under cutLine in any -// of them would hold every other store back to this store's floor for no benefit. +// A contiguous store resolves the rollback window against its own head: head - rollbackWindow, or 0 +// where that head has not cleared the window. The states below are the ones where a snapshot store +// would answer differently. // // The prune assertions ride along on the same store because both methods are answers about the // same three states, and opening a second store to re-reach them is the slow part of this file. -func TestGCPruningBoundaryAndPruneBelow(t *testing.T) { +func TestGCRollbackFloorAndPruneHistory(t *testing.T) { db, store := openForGC(t, t.TempDir()) rng := utils.TestRngFromSeed(2) impl := db.(*blockDB) - // Empty. The prune that follows is a no-op rather than an error — a store still filling is - // pruned like any other — and the boundary is still cutLine, since CannotServeRollback here - // would stall every other store. - require.Equal(t, uint64(42), store.GetPruningBoundary(42)) - require.NoError(t, store.PruneBelow(1_000)) + // Empty, so the head is 0 and every window reports 0 — "keep everything", which holds the + // fleet's history where it is until this store fills. The prune that follows is a no-op rather + // than an error. + require.Equal(t, uint64(0), store.GetRollbackFloor(0)) + require.Equal(t, uint64(0), store.GetRollbackFloor(42)) + require.NoError(t, store.PruneHistory(1_000)) require.Equal(t, uint64(0), impl.watermark.Load()) writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19, QCs [0,5),[5,10),[10,15),[15,20) - require.Equal(t, uint64(7), store.GetPruningBoundary(7)) - // Above the head, which happens whenever another store's data puts the head above this one's. - require.Equal(t, uint64(1_000), store.GetPruningBoundary(1_000)) + require.Equal(t, uint64(19), store.GetRollbackFloor(0), "the whole store is inside a window of 0") + require.Equal(t, uint64(7), store.GetRollbackFloor(12)) + // A window deeper than the store's own head is a rollback promise reaching past genesis, so + // nothing here is eligible for pruning yet. + require.Equal(t, uint64(0), store.GetRollbackFloor(1_000)) - require.NoError(t, store.PruneBelow(7)) + require.NoError(t, store.PruneHistory(7)) require.Equal(t, uint64(5), impl.watermark.Load(), "a prune inside QC[5,10) rounds down to its start") - require.NoError(t, store.PruneBelow(3)) + require.NoError(t, store.PruneHistory(3)) require.Equal(t, uint64(5), impl.watermark.Load(), "the watermark must not move backwards") - // Pruned above the cut line: nothing below it survives to protect, so cutLine still stands. - require.Equal(t, uint64(4), store.GetPruningBoundary(4)) + // Already pruned above the answer: the floor is a function of the head, not of the retention + // floor, so a store pruned higher by an earlier cycle does not hold the others back to where it + // started. + require.Equal(t, uint64(15), store.GetRollbackFloor(4)) } // The collector prunes every store to a shared minimum, so a store lagging the head is asked to // prune past everything it holds. The never-empty cap turns that into a prune to the newest // cohort rather than a store that can serve nothing. -func TestGCPruneBelowAboveHeadIsCapped(t *testing.T) { +func TestGCPruneHistoryAboveHeadIsCapped(t *testing.T) { db, store := openForGC(t, t.TempDir()) rng := utils.TestRngFromSeed(3) writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19, newest cohort QC[15,20) - require.NoError(t, store.PruneBelow(1_000)) + require.NoError(t, store.PruneHistory(1_000)) require.Equal(t, uint64(15), db.(*blockDB).watermark.Load(), "a prune past the head is capped to the newest cohort") for n := types.GlobalBlockNumber(15); n < 20; n++ { @@ -136,17 +142,15 @@ func TestGCPruneBelowAboveHeadIsCapped(t *testing.T) { } } -func TestConfigValidateRetentionWindow(t *testing.T) { +func TestConfigValidateRetentionTime(t *testing.T) { cfg, err := DefaultConfig(t.TempDir()) require.NoError(t, err) + require.NoError(t, cfg.Validate()) - for _, retentionWindow := range []int64{gc.InfiniteRetentionWindow, 0, 1} { - cfg.RetentionWindow = retentionWindow - require.NoError(t, cfg.Validate()) + // RetentionTime gates reclamation underneath the watermark, so a non-positive value would + // release records the moment the watermark passed them, removing the failsafe entirely. + for _, retentionTime := range []time.Duration{0, -time.Second} { + cfg.RetentionTime = retentionTime + require.ErrorContains(t, cfg.Validate(), "RetentionTime") } - - // Below InfiniteRetentionWindow there is no meaning left to assign, and the collector reads - // any negative value as infinite retention — so a typo'd -2 would silently disable pruning. - cfg.RetentionWindow = -2 - require.ErrorContains(t, cfg.Validate(), "RetentionWindow") } diff --git a/sei-db/ledger_db/receipt/litt_receipt_gc.go b/sei-db/ledger_db/receipt/litt_receipt_gc.go index 9c42fd5011..501d0d8ba4 100644 --- a/sei-db/ledger_db/receipt/litt_receipt_gc.go +++ b/sei-db/ledger_db/receipt/litt_receipt_gc.go @@ -28,52 +28,48 @@ func (s *littReceiptStore) ExternalPruning() bool { return s.externalPruning } -// PruneBelow advances the retention floor to blockNumber and drops the tag-index entries below +// PruneHistory advances the retention floor to blockNumber and drops the tag-index entries below // it. Receipt bodies are not deleted here: advancing the floor is what releases them to litt's GC, // which reclaims them once they are also past the TTL (see gcFilter). Reads below the floor return // not-found in the meantime (see belowRetentionFloor), so visible retention follows this call even // when reclamation lags it — and because the floor gates reclamation, it can never lead it. -func (s *littReceiptStore) PruneBelow(blockNumber uint64) error { +// +// KeepRecent does not enter into this. It is the window the store's own pruner uses when it runs, +// and the collector calling here means that pruner stood down (see ExternalPruning); how deep to +// prune is then the collector's fleet-wide RollbackWindow and LookbackWindow, not a per-store +// setting. +func (s *littReceiptStore) PruneHistory(blockNumber uint64) error { return s.pruneBlocksBelow(blockNumber) } -// GetRetentionWindow translates KeepRecent into the collector's window. -// -// The two disagree on what 0 means, and the disagreement is the whole reason this is not a plain -// field read: KeepRecent 0 means "keep everything, never prune" (it is derived from -// min-retain-blocks, and 0 is the default), while a 0 here means "keep nothing beyond the shared -// RollbackWindow" — the most aggressive answer available. Returning it verbatim would prune a -// store configured to retain forever back to the rollback window. gc.InfiniteRetentionWindow is -// the sentinel that carries the intended meaning, and it is a legitimate answer for a contiguous -// store, which holds no replay range another store depends on. -// -// Negative values are folded into the same case: KeepRecent is never negative in practice, and -// this store has historically read <= 0 as "pruning off", so folding them keeps that reading -// intact rather than passing an out-of-contract value to the collector. -func (s *littReceiptStore) GetRetentionWindow() int64 { - if s.keepRecent <= 0 { - return gc.InfiniteRetentionWindow - } - return s.keepRecent +// PruneSnapshots does nothing: this store keeps no snapshots. Receipts are read from the blocks +// themselves, so the retention floor PruneHistory moves is the whole story. +func (s *littReceiptStore) PruneSnapshots(uint64) error { + return nil } -// GetPruningBoundary returns cutLine, the contract's answer for a contiguous store: every block -// at or above the floor is retained, so cutLine itself is restorable and nothing below it has to -// be held back. +// GetRollbackFloor returns head - rollbackWindow, the contract's answer for a contiguous store: +// every block from the floor to the head is retained, so that height is restorable directly and +// nothing below it has to be held back. This store keeps no snapshots, so there is nothing to resolve +// the window against beyond its own head. +// +// It reports against its own head even when that runs ahead of the fleet's, because the collector +// takes a minimum across stores. // -// Unconditional on purpose. A store whose floor already sits above cutLine — pruned there by an -// earlier cycle, or still backfilling — also answers cutLine, because the PruneBelow that follows -// is a no-op on it while a lower answer would hold every other store back to this store's floor. -// CannotServeRollback is never right here: receipts are written from this node's own execution -// path, so no other store replays out of them. -func (s *littReceiptStore) GetPruningBoundary(cutLine uint64) uint64 { - return cutLine +// 0 when the window is deeper than the whole history — including a store still backfilling, whose +// head is 0. Nothing here is eligible for pruning until the head clears the window. +func (s *littReceiptStore) GetRollbackFloor(rollbackWindow uint64) uint64 { + head, err := s.GetLatestBlock() + if err != nil || head <= rollbackWindow { + return 0 // cannot say what it holds, so nothing may be dropped anywhere + } + return head - rollbackWindow } // GetLatestBlock returns the newest block whose receipts have been written, or 0 when none have. -// 0 keeps the store out of the collector's head minimum rather than dragging every store's cut -// line down to it — the right trade while a store is still filling, since the prune it then -// receives only moves a floor that has no data under it. +// A store still filling then answers a rollback floor of 0, which holds the fleet's history where it +// is — the right trade, since the prune it would otherwise receive only moves a floor that has no +// data under it. func (s *littReceiptStore) GetLatestBlock() (uint64, error) { latest := s.latestVersion.Load() if latest <= 0 { diff --git a/sei-db/ledger_db/receipt/litt_receipt_gc_test.go b/sei-db/ledger_db/receipt/litt_receipt_gc_test.go index 3a61d18000..2476b7f198 100644 --- a/sei-db/ledger_db/receipt/litt_receipt_gc_test.go +++ b/sei-db/ledger_db/receipt/litt_receipt_gc_test.go @@ -106,28 +106,37 @@ func TestReceiptLocalPrunerAdvancesFloorWithoutCollector(t *testing.T) { }, 6*time.Second, 25*time.Millisecond, "the local pruner must advance the floor with no collector present") } -// KeepRecent and GetRetentionWindow disagree about 0, so the translation is the behavior worth -// pinning: KeepRecent 0 means "keep everything" and is the default, while a literal 0 answer -// means "keep only the shared rollback window". Returning the field verbatim would prune a store -// configured to retain forever back to ~1_000 blocks. -func TestReceiptGCRetentionWindowMapsKeepRecent(t *testing.T) { - for _, tc := range []struct { - name string - keepRecent int - want int64 - }{ - {name: "keep everything", keepRecent: 0, want: gc.InfiniteRetentionWindow}, - {name: "bounded retention", keepRecent: 100_000, want: 100_000}, - } { - t.Run(tc.name, func(t *testing.T) { - _, prunable, _ := setupLittIdxForGC(t, tc.keepRecent) - require.Equal(t, tc.want, prunable.GetRetentionWindow()) - }) +// KeepRecent belongs to the local pruner and must not reach the collector's answers: how deep the +// collector prunes is its own fleet-wide window. Pinned across both readings of KeepRecent, since 0 +// used to mean "keep everything" here and would once have suppressed pruning entirely. +func TestReceiptGCAnswersDoNotDependOnKeepRecent(t *testing.T) { + for _, keepRecent := range []int{0, 100_000} { + _, prunable, _ := setupLittIdxForGC(t, keepRecent) + require.Equal(t, uint64(0), prunable.GetRollbackFloor(10_000), + "keepRecent %d must not change the floor", keepRecent) } } -// The head is 0 until receipts land, which keeps a store that is still filling out of the -// collector's head minimum instead of dragging every store's cut line down to it. +// This store keeps no snapshots, so its half of the split contract is a no-op that must leave the +// retention floor where it is. +func TestReceiptGCPruneSnapshotsIsANoOp(t *testing.T) { + store, prunable, ctx := setupLittIdxForGC(t, 0) + addr := common.HexToAddress("0xabcd") + topic := common.HexToHash("0x1111") + for block := uint64(1); block <= 3; block++ { + writeLitBlock(t, store, ctx, block, litReceipt(block, 0, addr, topic)) + } + + require.NoError(t, prunable.PruneSnapshots(3)) + require.Equal(t, int64(0), store.EarliestVersion(), "a snapshot prune must not move the floor") + + kept, err := store.GetReceiptFromStore(ctx, litTxHash(1, 0)) + require.NoError(t, err) + require.Equal(t, uint64(1), kept.BlockNumber) +} + +// The head is 0 until receipts land, so a store that is still filling reports a floor of 0 and the +// fleet holds its history where it is rather than pruning against a head this store does not have. func TestReceiptGCLatestBlock(t *testing.T) { store, prunable, ctx := setupLittIdxForGC(t, 0) @@ -144,31 +153,32 @@ func TestReceiptGCLatestBlock(t *testing.T) { latest, err = prunable.GetLatestBlock() require.NoError(t, err) require.Equal(t, uint64(3), latest) - require.Equal(t, int64(3), store.LatestVersion(), "the collector's head must agree with the store's own version") + require.Equal(t, int64(3), store.LatestVersion(), "the head reported to the collector must agree with the store's own version") } -// A contiguous store answers the cut line it was given whatever it holds, and the prune that -// follows moves the retention floor to it — which is what makes the receipts below it stop being +// A contiguous store resolves the window against its own head, and the prune that follows moves the +// retention floor to the height it is given — which is what makes the receipts below it stop being // served. Reclaiming their bodies lags that, since litt also waits for the TTL, but it can no // longer lead it (see TestGCFilterMakesLittReclamationFollowTheBlockFloor). -func TestReceiptGCPruningBoundaryAndPruneBelow(t *testing.T) { +func TestReceiptGCRollbackFloorAndPruneHistory(t *testing.T) { store, prunable, ctx := setupLittIdxForGC(t, 0) addr := common.HexToAddress("0xabcd") topic := common.HexToHash("0x1111") - // Holding nothing: the boundary is still cutLine, since CannotServeRollback here would stall - // every other store rather than protect anything. - require.Equal(t, uint64(42), prunable.GetPruningBoundary(42)) + // Holding nothing: the head is 0, so every window reports 0 — keep everything — which holds the + // fleet's history where it is until this store fills. + require.Equal(t, uint64(0), prunable.GetRollbackFloor(42)) for block := uint64(1); block <= 3; block++ { writeLitBlock(t, store, ctx, block, litReceipt(block, 0, addr, topic)) } - require.Equal(t, uint64(2), prunable.GetPruningBoundary(2)) - // Above the head, which happens whenever another store's data puts the head above this one's. - require.Equal(t, uint64(1_000), prunable.GetPruningBoundary(1_000)) + require.Equal(t, uint64(1), prunable.GetRollbackFloor(2)) + // A window deeper than its own head is a rollback promise reaching past genesis, so nothing here + // is eligible for pruning yet. + require.Equal(t, uint64(0), prunable.GetRollbackFloor(1_000)) - require.NoError(t, prunable.PruneBelow(3)) - require.Equal(t, int64(3), store.EarliestVersion(), "PruneBelow must advance the retention floor") + require.NoError(t, prunable.PruneHistory(3)) + require.Equal(t, int64(3), store.EarliestVersion(), "PruneHistory must advance the retention floor") _, err := store.GetReceiptFromStore(ctx, litTxHash(1, 0)) require.ErrorIs(t, err, receipt.ErrNotFound, "a receipt below the floor must not be served") @@ -177,21 +187,21 @@ func TestReceiptGCPruningBoundaryAndPruneBelow(t *testing.T) { require.Equal(t, uint64(3), kept.BlockNumber) // The floor only advances: a later, lower cycle must not re-expose what was pruned. - require.NoError(t, prunable.PruneBelow(1)) + require.NoError(t, prunable.PruneHistory(1)) require.Equal(t, int64(3), store.EarliestVersion()) } -// PruneBelow carries a minimum taken across every managed store, so it can arrive above this +// PruneHistory carries a minimum taken across every managed store, so it can arrive above this // store's head whenever this one lags or has ingested nothing. Both cases are the store's own to -// survive: the collector's head minimum makes them unlikely, but that is a property of the caller. -func TestReceiptGCPruneBelowAboveHead(t *testing.T) { +// survive: this store's own floor of 0 makes them unlikely, but that is a property of the caller. +func TestReceiptGCPruneHistoryAboveHead(t *testing.T) { addr := common.HexToAddress("0xabcd") topic := common.HexToHash("0x1111") t.Run("empty store keeps its floor at zero", func(t *testing.T) { store, prunable, _ := setupLittIdxForGC(t, 0) - require.NoError(t, prunable.PruneBelow(1_000)) + require.NoError(t, prunable.PruneHistory(1_000)) require.Equal(t, int64(0), store.EarliestVersion(), "a floor above an empty store would have to be walked back once blocks arrive") }) @@ -202,7 +212,7 @@ func TestReceiptGCPruneBelowAboveHead(t *testing.T) { writeLitBlock(t, store, ctx, block, litReceipt(block, 0, addr, topic)) } - require.NoError(t, prunable.PruneBelow(1_000)) + require.NoError(t, prunable.PruneHistory(1_000)) require.Equal(t, int64(3), store.EarliestVersion(), "the floor stops at the head, not the request") kept, err := store.GetReceiptFromStore(ctx, litTxHash(3, 0)) diff --git a/sei-db/ledger_db/receipt/litt_receipt_pruner_internal_test.go b/sei-db/ledger_db/receipt/litt_receipt_pruner_internal_test.go index e21161380b..f684e41a7a 100644 --- a/sei-db/ledger_db/receipt/litt_receipt_pruner_internal_test.go +++ b/sei-db/ledger_db/receipt/litt_receipt_pruner_internal_test.go @@ -33,7 +33,7 @@ func TestRunsLocalPruner(t *testing.T) { }, { // KeepRecent 0 is the default and means keep everything, so there is nothing for a - // local pruner to do. GetRetentionWindow maps the same 0 to InfiniteRetentionWindow. + // local pruner to do. It reaches none of the collector's answers either. name: "keep everything", keepRecent: 0, pruneInterval: 600, diff --git a/sei-db/ledger_db/receipt/litt_receipt_store.go b/sei-db/ledger_db/receipt/litt_receipt_store.go index f8a079eb36..08c3359dda 100644 --- a/sei-db/ledger_db/receipt/litt_receipt_store.go +++ b/sei-db/ledger_db/receipt/litt_receipt_store.go @@ -73,10 +73,10 @@ import ( // collector's view of who prunes this store and the store's own view stay one // fact. // -// KeepRecent is what this store asks for either way: the pruner's window when it -// runs, and the collector's retention window when it does not. It does not reach -// litt's TTL, which is a flat age floor (littRetentionTime) — how much history is -// kept is the floor's business, and only gcFilter releases a body to be reclaimed. +// KeepRecent is only the local pruner's window. Under the collector it is ignored: +// how deep to keep is the shared RollbackWindow + LookbackWindow, not this per-store +// setting. Either way it does not reach litt's TTL, a flat age floor (littRetentionTime) +// — how much history is kept is the floor's business, and only gcFilter releases a body. type littReceiptStore struct { values litt.DB receipts litt.Table @@ -121,8 +121,8 @@ const ( // It was previously KeepRecent × 2s, which made the TTL a second, disagreeing // answer to a question the floor already answers — and one that came out short // under ExternalPruning, where the enforced retention is RollbackWindow + - // KeepRecent. An age floor cannot be wrong that way, because it does not claim to - // know how many blocks anything is. + // LookbackWindow. An age floor cannot be wrong that way, because it does not claim + // to know how many blocks anything is. littRetentionTime = time.Hour littPartCountLen = 4 @@ -140,12 +140,11 @@ func littPartKey(blockNumber uint64, part uint32) []byte { // gcFilter makes the retention floor a precondition for reclaiming a receipt body, so litt's TTL // can only ever reclaim what the floor has already released. // -// Without it the TTL is the sole reclaimer (a nil filter leaves TTL as the only condition), and it -// is derived from KeepRecent alone — which is not the retention this store owes anyone. Under -// ExternalPruning the collector holds the floor at RollbackWindow + KeepRecent blocks back, so a -// TTL sized for KeepRecent can expire bodies the floor still calls live, and a read inside the -// rollback window returns not-found. That is the cross-store guarantee the collector exists to -// provide, so the two are joined here rather than kept in agreement by arithmetic: what reads +// Without it the TTL is the sole reclaimer (a nil filter leaves TTL as the only condition). Under +// ExternalPruning the collector holds the floor at RollbackWindow + LookbackWindow blocks back, so +// a TTL sized to any block count could expire bodies the floor still calls live, and a read inside +// the rollback window would return not-found. That is the cross-store guarantee the collector exists +// to provide, so the two are joined here rather than kept in agreement by arithmetic: what reads // enforce and what GC reclaims are now the same fact, whatever the block time or the window. // // Only primary part keys gate. Tx-hash secondaries alias a body in their own segment, so the diff --git a/sei-db/management/gc/api.go b/sei-db/management/gc/api.go index 803124b7cd..138301e284 100644 --- a/sei-db/management/gc/api.go +++ b/sei-db/management/gc/api.go @@ -1,145 +1,102 @@ package gc -// InfiniteRetentionWindow is the GetRetentionWindow value meaning "never prune this store": its -// cut line is forced to 0, so it is never asked for a boundary and never passed to PruneBelow. -// -// Note that 0 here means "no extra beyond the shared RollbackWindow", not "keep forever" as -// KeepRecent / MinRetainBlocks do elsewhere in this repo — hence the negative sentinel. -const InfiniteRetentionWindow int64 = -1 - -// CannotServeRollback is the GetPruningBoundary answer meaning "I hold nothing that can serve a -// rollback to any height": in practice, a snapshot store with no completed snapshot. A store that -// answers this is not idle — it will replay forward once its first snapshot lands, so pruning the -// others now can delete the range it will replay from. The collector therefore stops: +// PrunableStore is a store whose old data may be dropped by the StorageGarbageCollector. // -// RollbackWindow > 0 → the whole cycle is abandoned; nothing is pruned -// RollbackWindow == 0 → this store is ignored and the others are pruned +// Deletion is split in two because the two halves are pruned to different depths. Snapshots go down +// to the deepest rollback the fleet still owes, since a restore point below that is one nothing can +// ask for. History goes one lookback window deeper, because the floor snapshot is only restorable if +// the blocks above it survive, and because that history is still readable where the snapshot is not. // -// 0 is a safe value for this because block heights start at 1 and a store is only asked when its -// own cut line is above 0, so a real boundary is always positive. It also puts the conservative -// answer on Go's zero value: a store that returns nothing meaningful stops pruning rather than -// silently dropping out of the decision. +// Every store implements both halves. Most hold only one kind of data and return nil from the +// other, which is a cheaper contract than an optional interface: a store cannot drop out of a +// prune by failing to implement a method, so "this store keeps no snapshots" is a claim written +// down in the store rather than inferred from a type assertion in the collector. // -// Two limits are worth knowing. RollbackWindow == 0 puts the cut line at the head itself, so -// contiguous stores are pruned to their head and a store that has not snapshotted yet loses that -// replay range outright — do not run it on a node whose snapshot stores are still bootstrapping. -// And this signal only reaches the collector if the store is asked, which happens only when its -// own cut line is above 0: a snapshot store with InfiniteRetentionWindow, or with a retention -// window deep enough to zero its cut line, is never asked and so cannot stop a cycle. The SC/SS -// retention contract below pins retention at 0 to keep them out of that case. -const CannotServeRollback uint64 = 0 - -// PrunableStore is a store whose old data may be dropped by the StorageGarbageCollector. +// The store, not the collector, turns the rollback window into a height. It is the only party that +// knows what it actually holds — where its snapshots sit, how far its ingest has got — so it reports +// a floor and the collector reduces those to the cut lines it hands back. type PrunableStore interface { // Name identifies the store for logs and errors. Duplicate names are allowed; the // collector keys answers by index, not by name. Name() string - // PruneBelow may drop data for all blocks below blockNumber. The store may perform the - // deletion asynchronously. Only called when ExternalPruning reports true. - PruneBelow(blockNumber uint64) error + // PruneHistory may drop the per-block history below blockNumber, leaving snapshots to + // PruneSnapshots. The store may perform the deletion asynchronously. Only called when + // ExternalPruning reports true. + // + // blockNumber is calculated with the consideration of both lookback and rollback window. + // For example, WAL can only prune the block number up till the last snapshot height. + // + // It is the minimum across every participating store less the lookback window, so it can sit + // below what this store alone would need — and above this store's head, since a store may lag + // the ones that set the minimum. Both are the store's to absorb: a request outside what it + // holds must clamp to a no-op rather than empty the store. + PruneHistory(blockNumber uint64) error + + // PruneSnapshots may drop every snapshot strictly below blockNumber, leaving the per-block + // history to PruneHistory. Only called when ExternalPruning reports true, and never with 0. + // + // blockNumber is the minimum of every store's GetRollbackFloor, so it sits at or below what + // this store itself reported. A store that reported a snapshot therefore always keeps that + // snapshot: it is at or above this height by construction, and so is never a candidate here. + // That is what makes the collector's minimum meaningful — history is held at a floor the + // store still has the restore point for. + // + // It stops one lookback window short of where PruneHistory goes, and the asymmetry is the + // point. Restoring to any height inside the rollback window starts from the floor snapshot + // and replays history forward, so a snapshot below that floor is a restore point nothing can + // ask for, while the history below it is still readable and is what the lookback window + // buys. + // + // A store that keeps no snapshots returns nil. + PruneSnapshots(blockNumber uint64) error // ExternalPruning reports whether this store's retention is the collector's to enforce. // // true → the collector prunes it; any pruner inside the store must stand down - // false → the store prunes itself; the collector never calls PruneBelow on it + // false → the store prunes itself; it receives neither PruneHistory nor PruneSnapshots // // This exists so that "the collector manages this store" and "the store's own pruner is // off" are one fact rather than two settings that can disagree. A store with an internal // pruner answers from the same field that pruner consults, which is what makes the two // mutually exclusive by construction instead of by wiring discipline. A store with no // pruner of its own returns true unconditionally. - // - // false does NOT withdraw the store from the decision. It is still asked for - // GetLatestBlock and GetPruningBoundary, and still holds the shared minimum down, because - // what a self-pruning store retains can be exactly what another store must replay from — - // SC's snapshots are useless if the WAL beneath them has been pruned away. Dropping it - // from the vote would prune that range out from under it. Opting out of being pruned and - // opting out of protecting others are different things, and only the first is on offer. - // - // Read once per cycle, so a store may change its answer between cycles but not within one. ExternalPruning() bool - // GetRetentionWindow is how many extra blocks beyond the shared RollbackWindow this store - // needs to keep servable, where head is the min non-zero GetLatestBlock. Three answers: - // - // > 0 → extra history on top; cutLine = head - RollbackWindow - retention - // 0 → no extra history; cutLine = head - RollbackWindow - // -1 → never prune this store at all (see InfiniteRetentionWindow) - // - // RollbackWindow is shared so every store stays consistent under rollback. Retention is - // optional history on top, so a store can still serve queries after a rollback has consumed - // the whole window. - // - // This is an input to a shared minimum, not an independently applied per-store policy: - // because every participating store is pruned to the lowest boundary, one store's deep - // retention extends retention for all of them (see StorageGarbageCollector). - // - // By store kind: - // - Contiguous (blockDB, receiptDB, state WAL): -1 / 0 / positive as configured. - // - SC: 0. It only needs the shared rollback window for its snapshots. - // - SS: 0, including on an archive node. What the collector manages for SS is snapshot - // pruning, not state pruning, and those snapshots only need to cover RollbackWindow. SS - // keeps its own version history via KeepRecent, so making a node archival belongs there, - // not here. Returning -1 would freeze SS snapshots forever without retaining any extra - // state — the opposite trade from the one intended — and would also drop SS out of the - // shared minimum, per the warning below. - // - // Do not give a snapshot store InfiniteRetentionWindow expecting deeper history — it does the - // opposite. A store with no cut line is never asked for a boundary, so it drops out of the - // minimum and the contiguous store holding its replay range is pruned to its own cut line. - // With retention 0 the same store answers its oldest needed snapshot and pulls that range down - // with it. Concretely, with head 100_000, RollbackWindow 1_000, and a lone snapshot at 20_000: - // retention 0 holds the WAL at 20_000, while InfiniteRetentionWindow lets it go to 99_000 and - // leaves the snapshot unable to replay forward. Only the snapshot itself survives, so this is - // safe solely for a store that never needs another store's history — which is what the - // StorageGarbageCollector precondition requires. TestPruneInfiniteRetentionSnapshotStore pins - // both directions. - GetRetentionWindow() int64 - - // GetPruningBoundary returns the oldest block this store must keep in order to serve cutLine: - // - // > 0 → that boundary, which must never exceed cutLine - // CannotServeRollback → no snapshots available; abandons the cycle - // - // There is no third answer: a store either has a boundary to report or cannot serve a - // rollback at all. - // - // Contiguous stores can restore to any height they hold, so they return cutLine. Snapshot - // stores can restore only at a snapshot, so they return the newest completed snapshot at or - // below cutLine; when every snapshot sits above cutLine none of them can be dropped, and they - // return cutLine rather than their oldest snapshot. With no completed snapshot at all they - // return CannotServeRollback. - // - // A store holding nothing at or below cutLine — empty, already pruned to a higher floor, or - // restored by state sync above it — still returns cutLine. The PruneBelow that follows is a - // no-op, which is the point: it has nothing to delete and nothing to protect, since a - // contiguous store fills from its own ingest path rather than by replaying another store. - // CannotServeRollback would be wrong there, stalling the fleet until the head advanced a full - // RollbackWindow past that floor. - // - // Never answering above cutLine is what makes pruneHeight <= head - RollbackWindow hold by - // construction, since the collector takes a minimum across stores and does not clamp. A higher - // answer would raise that minimum; with no correctly answering retention-0 store to cap it, it - // can exceed the head and drop the store outright. - // - // Nothing in the collector enforces this — honoring it is the implementor's job. The collector - // cannot repair a bad answer, because cutLine is all it knows: substituting it would still prune - // past the snapshot a snapshot store needed, and refusing to prune would let one faulty store - // stall every other store's pruning. So this bound is load-bearing and unchecked. - // - // A snapshot write in flight need not be reserved, as snapshot creation is assumed to finish - // quickly. Never having produced one is the separate CannotServeRollback case. - GetPruningBoundary(cutLine uint64) uint64 + // GetRollbackFloor returns the earliest height a rollback may target: given rollbackWindow, + // the deepest height this store could still be asked to restore to, and so the oldest block + // it needs the fleet to keep. The collector takes the minimum across stores to get the + // snapshot cut line, then subtracts LookbackWindow from that for the history cut line. + // + // The window is measured against the store's own head rather than a height handed down, so a + // store ahead of the fleet answers from where it actually is: + // + // contiguous store → head - rollbackWindow, every height in between being restorable + // snapshot store → the newest snapshot at or below head - rollbackWindow, restoring + // starting there and replaying forward; or its oldest snapshot when + // every one of them is above that height, that being the deepest it + // can reach + // + // 0 means nothing here is eligible for pruning, and it is a height rather than a sentinel: + // keep everything from block 0 up. Because both cut lines are derived from the minimum of + // these answers, one store answering 0 holds the whole fleet where it is for the cycle. + // Three situations reach it, and all three want that outcome: + // + // head <= rollbackWindow → the promised rollback is deeper than this store's whole + // history, so no part of it can be given up yet + // nothing ingested yet → the same case with a head of 0 + // cannot tell what it holds → an unreadable head or snapshot listing; the store must not + // let history be dropped on the strength of a guess + // + // Answering high is the damaging direction, since nothing above clamps it: the collector + // derives its cut lines from these answers rather than capping them. + GetRollbackFloor(rollbackWindow uint64) uint64 - // GetLatestBlock returns the highest block this store has ingested. - // - // 0 means "nothing ingested yet" and is excluded when computing the head, so a store still - // filling cannot drag the head — and every cut line with it — down to 0. That exclusion says - // nothing about whether the store may be pruned around: one that cannot serve a rollback - // still stops the cycle, via GetPruningBoundary. + // GetLatestBlock returns the highest block this store has ingested, 0 when it has ingested + // nothing. // - // 0 does not mean "disabled". A store disabled for this node is not instantiated and never - // reaches the collector. + // It is the head GetRollbackFloor measures the rollback window against. The collector does + // not call it — every height it acts on comes from GetRollbackFloor — so this is here as the + // store's ingest position for operators and tests. GetLatestBlock() (uint64, error) } diff --git a/sei-db/management/gc/storage_garbage_collector.go b/sei-db/management/gc/storage_garbage_collector.go index 8e70d2349e..290ff54d5e 100644 --- a/sei-db/management/gc/storage_garbage_collector.go +++ b/sei-db/management/gc/storage_garbage_collector.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "math" "strings" "sync" "time" @@ -15,49 +16,20 @@ var logger = seilog.NewLogger("db", "gc") // StorageGarbageCollector periodically prunes a set of PrunableStores. // -// Every store shares config.RollbackWindow and may request extra history via -// GetRetentionWindow. Each cycle: +// One config.RollbackWindow and one config.LookbackWindow cover every store. Each cycle: // -// 1. head = min non-zero GetLatestBlock across stores -// 2. per store: cutLine = head - RollbackWindow - GetRetentionWindow, skipping the store -// when that is 0 (infinite retention, or head still inside its own window) -// 3. ask GetPruningBoundary(cutLine): a positive answer votes, and CannotServeRollback -// abandons the cycle while RollbackWindow > 0 -// 4. pruneHeight = min of the positive answers; PruneBelow(pruneHeight) on every store -// that answered positively and reports ExternalPruning +// 1. ask every store GetRollbackFloor(RollbackWindow) — the earliest height it could still be +// asked to roll back to, which it resolves against its own head +// 2. snapshotCutLine = min of those answers, the deepest rollback the fleet still owes +// 3. historyCutLine = snapshotCutLine - LookbackWindow, so the lookback window sits entirely below +// the deepest promised rollback point rather than overlapping it +// 4. on every store reporting ExternalPruning, PruneSnapshots(snapshotCutLine) then +// PruneHistory(historyCutLine) // -// Step 4 prunes to the shared minimum rather than to each store's own boundary so that a -// retained snapshot stays restorable: contiguous stores must still hold the blocks that follow -// it. The side effect is one effective retention across the fleet — receiptDB retention 100_000 -// also keeps SC/SS snapshots that far back, even though those stores report retention 0. That -// uniform prune is an intentional Giga tradeoff, not independent per-store retention. -// -// Two properties do the safety work. -// -// Answers are bounded by the cut line they were given, so pruneHeight <= head - RollbackWindow -// holds by construction and the collector does not clamp. That yields the invariant: a prune cannot -// take away a rollback that was possible before it. Full headroom is not promised after a rollback -// has already consumed part of the window. -// -// The bound is trusted rather than checked. A check could only substitute cutLine for a bad answer, -// which for a snapshot store is itself too aggressive — the correct answer is the newest snapshot at -// or below cutLine, normally well below it — and refusing to prune instead would let one faulty -// store stall the fleet and grow disk without limit. Both remedies are worse than the bug they -// guard, so the bound stays a contract requirement on GetPruningBoundary. -// -// And a store that can serve no rollback at all is not merely skipped, it abandons the cycle. It -// will replay forward once its first snapshot lands, so pruning the others now could delete the -// range it replays from, and a store contributing no boundary is not covered by the minimum in -// step 4. See CannotServeRollback for the limits of that signal. -// -// Precondition: every store passed in is live. A store disabled for this node is not -// instantiated and never reaches the collector, so the set holds no permanently-empty member. -// -// PruneBelow failures are joined and do not skip later stores: permission-to-drop is not -// transactional, and one unhealthy store must not block pruning of the others. -// -// The type is a ticker around prune; the decision logic lives in prune for unit testing. -// Not safe for concurrent use (Close must be called exactly once). +// The two windows stack rather than share: RollbackWindow buys the ability to rewind, LookbackWindow +// buys history that is still readable after rewinding as far as that promise allows. Deriving the +// history cut line by subtracting the second from the first is what makes the lookback guarantee +// independent of how deep a rollback actually goes. type StorageGarbageCollector struct { config *StorageGarbageCollectorConfig stores []PrunableStore @@ -125,161 +97,107 @@ func prune(config *StorageGarbageCollectorConfig, stores []PrunableStore) error return nil } - globalLatestBlock, err := getGlobalLatestBlock(stores) - if err != nil { - return err - } - if globalLatestBlock == 0 { - logger.Info("skipping pruning: no store has a latest block") - return nil - } - // Answers are positional so duplicate Name() values cannot mis-attribute one. decisions := make([]storeDecision, len(stores)) - var pruneHeight uint64 - blocked := false + snapshotCutLine := uint64(math.MaxUint64) for i, store := range stores { - retention := store.GetRetentionWindow() - cutLine := getCutLine(globalLatestBlock, config.RollbackWindow, retention) - if cutLine == 0 { - // Infinite retention, or head still inside this store's retain window: never asked. - continue - } - - decisions[i].cutLine = cutLine decisions[i].externalPruning = store.ExternalPruning() - decisions[i].boundary = store.GetPruningBoundary(cutLine) - if decisions[i].boundary == CannotServeRollback { - // One blocker is enough to abandon the cycle, but the rest are still asked rather than - // broken out of, so the blocked cycle logs every store's decision. These are the cycles - // an operator has to explain, and every blocker surfaces in the first one rather than - // one per cycle. RollbackWindow 0 waives the guarantee, so the blocker is ignored. - if config.RollbackWindow > 0 { - blocked = true - } - continue - } - if pruneHeight == 0 || decisions[i].boundary < pruneHeight { - pruneHeight = decisions[i].boundary - } + decisions[i].floor = store.GetRollbackFloor(config.RollbackWindow) + snapshotCutLine = min(snapshotCutLine, decisions[i].floor) } - if blocked { - // Ahead of the pruneHeight check on purpose: every non-blocking store has already voted, so - // a usable pruneHeight is sitting here and must not be issued. decisionByStore names the - // blockers, rendering each as cannotServeRollback. - logger.Info("pruning blocked: a store cannot serve the rollback window yet", - "globalLatestBlock", globalLatestBlock, - "decisionByStore", describeDecisions(stores, decisions), - ) - return nil - } - if pruneHeight == 0 { - logger.Info("skipping pruning: no store reported a pruning boundary", - "globalLatestBlock", globalLatestBlock, - "decisionByStore", describeDecisions(stores, decisions), - ) - return nil - } + historyCutLine := getHistoryCutLine(snapshotCutLine, config.LookbackWindow) logger.Info("pruning stores", - "globalLatestBlock", globalLatestBlock, - "pruneHeight", pruneHeight, + "rollbackWindow", config.RollbackWindow, + "lookbackWindow", config.LookbackWindow, + "snapshotCutLine", snapshotCutLine, + "historyCutLine", historyCutLine, "decisionByStore", describeDecisions(stores, decisions), ) - var pruneErrs error + return pruneStores(stores, decisions, snapshotCutLine, historyCutLine) +} + +// pruneStores issues the deletions the cycle decided on: snapshots below snapshotHeight, history +// below the deeper historyHeight. See StorageGarbageCollector for why the two depths differ. +// +// A cut line of 0 is skipped rather than passed down. It means nothing is eligible, which every +// store would absorb as a no-op anyway; not making the call keeps a cycle that decided to delete +// nothing from reaching the deletion paths at all. +// +// A store is skipped only when it prunes itself — it still answered above, and is still protected by +// the minimum its answer produced, but its own pruner is the one enforcing its retention (see +// PrunableStore.ExternalPruning). +func pruneStores( + stores []PrunableStore, + decisions []storeDecision, + snapshotHeight uint64, + historyHeight uint64, +) error { + var errs error for i, store := range stores { - // Covers both a never-asked store and one that cannot serve a rollback. - if decisions[i].boundary == 0 { - continue - } - // A self-pruning store still voted above, and is still protected by the minimum that - // vote produced. What it does not get is a PruneBelow, because its own pruner is the - // one enforcing its retention (see PrunableStore.ExternalPruning). if !decisions[i].externalPruning { continue } - if err := store.PruneBelow(pruneHeight); err != nil { - pruneErrs = errors.Join(pruneErrs, fmt.Errorf("failed to prune %s below %d: %w", store.Name(), pruneHeight, err)) + if snapshotHeight > 0 { + if err := store.PruneSnapshots(snapshotHeight); err != nil { + errs = errors.Join(errs, fmt.Errorf("failed to prune %s snapshots below %d: %w", + store.Name(), snapshotHeight, err)) + } + } + if historyHeight > 0 { + if err := store.PruneHistory(historyHeight); err != nil { + errs = errors.Join(errs, fmt.Errorf("failed to prune %s history below %d: %w", + store.Name(), historyHeight, err)) + } } } - return pruneErrs + return errs } -// storeDecision records what the collector asked one store and what it answered, for the prune -// log. cutLine == 0 means the store was never asked, which is what separates it from a store that -// was asked and answered 0. +// storeDecision records what one store answered, for the prune log. The windows and the resulting +// heights are shared by every store and so are logged once alongside these rather than repeated +// per entry. type storeDecision struct { - cutLine uint64 - boundary uint64 + floor uint64 externalPruning bool } -// describeDecisions renders one entry per store, in store order, for the prune log. The three -// outcomes render differently on purpose — never asked, cannot serve a rollback, and reported a -// boundary — since collapsing them would cost exactly the distinction wanted when auditing a -// deletion after the fact. Each asked store carries the cut line its answer is a function of. +// describeDecisions renders one entry per store, in store order, for the prune log. It is the stated +// mechanism for reconstructing a deletion after the fact, which is why every store appears whatever +// it answered: a floor of 0 is the entry that explains a cycle that pruned nothing, and naming the +// store that produced it is the whole value of the line. // -// A store that voted but prunes itself is tagged selfPruned, because its boundary is in the -// minimum below while no deletion follows on it — without the tag that reads like a prune that -// silently did nothing. +// A store that answered but prunes itself is tagged selfPruned, because its floor is in the minimum +// below while no deletion follows on it — without the tag that reads like a prune that silently did +// nothing. func describeDecisions(stores []PrunableStore, decisions []storeDecision) string { var sb strings.Builder for i, store := range stores { if i > 0 { sb.WriteString(" ") } - switch { - case decisions[i].cutLine == 0: - fmt.Fprintf(&sb, "%s=notAsked", store.Name()) - case decisions[i].boundary == CannotServeRollback: - fmt.Fprintf(&sb, "%s=cannotServeRollback(cutLine=%d)", store.Name(), decisions[i].cutLine) - case !decisions[i].externalPruning: - fmt.Fprintf(&sb, "%s=%d(cutLine=%d,selfPruned)", store.Name(), decisions[i].boundary, decisions[i].cutLine) - default: - fmt.Fprintf(&sb, "%s=%d(cutLine=%d)", store.Name(), decisions[i].boundary, decisions[i].cutLine) + if decisions[i].externalPruning { + fmt.Fprintf(&sb, "%s=%d", store.Name(), decisions[i].floor) + continue } + fmt.Fprintf(&sb, "%s=%d(selfPruned)", store.Name(), decisions[i].floor) } return sb.String() } -// getCutLine returns head - RollbackWindow - retention for retention >= 0. -// Returns 0 when retention < 0, when the combined window overflows uint64, or when -// head is still inside that window (unsigned subtraction must not wrap). -func getCutLine(globalLatestBlock uint64, rollbackWindow uint64, retention int64) uint64 { - if retention < 0 { - return 0 - } - totalRetainWindow := rollbackWindow + uint64(retention) - if totalRetainWindow < rollbackWindow { - // Addition wrapped; treat as an unsatisfiable retain window (skip pruning). - return 0 - } - if globalLatestBlock <= totalRetainWindow { - return 0 - } - return globalLatestBlock - totalRetainWindow -} - -// getGlobalLatestBlock returns the smallest non-zero GetLatestBlock among stores, or 0 when no -// store has data. The minimum keeps a lagging store from being pruned past blocks it still needs. +// getHistoryCutLine returns the height history may be pruned below: snapshotCutLine less +// lookbackWindow, which places the lookback window entirely beneath the deepest rollback the fleet +// still owes rather than overlapping it. // -// Heads of 0 are skipped rather than treated as "unknown". That necessarily excludes the store -// holding nothing from the minimum, so the minimum cannot protect it; what protects it is -// CannotServeRollback, plus the store-set precondition on StorageGarbageCollector. -func getGlobalLatestBlock(stores []PrunableStore) (uint64, error) { - var blockNum uint64 - for _, store := range stores { - storeHeight, err := store.GetLatestBlock() - if err != nil { - return 0, fmt.Errorf("failed to read latest block from %s: %w", store.Name(), err) - } - if storeHeight == 0 { - continue - } - if blockNum == 0 || storeHeight < blockNum { - blockNum = storeHeight - } +// Subtracting from the minimum of the stores' answers is what makes the result safe without clamping +// any of them: it can only sit at or below every store's own floor. +// +// 0 when the window reaches below genesis, and it needs no special handling — it is the literal +// height "keep everything from block 0 up", which every store's PruneHistory absorbs as a no-op. +func getHistoryCutLine(snapshotCutLine uint64, lookbackWindow uint64) uint64 { + if snapshotCutLine <= lookbackWindow { + return 0 } - return blockNum, nil + return snapshotCutLine - lookbackWindow } diff --git a/sei-db/management/gc/storage_garbage_collector_config.go b/sei-db/management/gc/storage_garbage_collector_config.go index 9b1e732fb3..f57eebb6ea 100644 --- a/sei-db/management/gc/storage_garbage_collector_config.go +++ b/sei-db/management/gc/storage_garbage_collector_config.go @@ -7,36 +7,32 @@ import ( // StorageGarbageCollectorConfig configures a StorageGarbageCollector. type StorageGarbageCollectorConfig struct { - // RollbackWindow is how many blocks behind head the system must remain able to - // roll back to. Shared by every managed store so they stay mutually consistent; - // per-store extras beyond this window use PrunableStore.GetRetentionWindow. + // RollbackWindow is how many blocks behind head the system must remain able to roll back + // to. It buys the ability to rewind; LookbackWindow buys the ability to still read history + // afterwards, and the two are separate knobs because they answer to different needs. // - // With R = the store's own GetRetentionWindow and F = LatestBlock - RollbackWindow - R, - // collection guarantees, for each managed store: + // 0 is allowed and waives the guarantee: each store then reports its own head as the + // earliest height a rollback may target, and a snapshot store reports its newest snapshot. + RollbackWindow uint64 + + // LookbackWindow is how much queryable history is kept behind the rollback window, in blocks. + // It is extra on top of RollbackWindow rather than a total that includes it, which is what + // makes the promise independent of rollback depth: however far the node rewinds inside the + // rollback window, at least LookbackWindow blocks below the new head stay readable. + // + // One window covers every managed store. With F = LatestBlock - RollbackWindow - + // LookbackWindow, collection guarantees: // // 1. Nothing needed to roll back to any block in // [LatestBlock - RollbackWindow, LatestBlock] is deleted. - // 2. No data at or above F is deleted. So even after rolling back to - // LatestBlock - RollbackWindow, the most recent R blocks are still readable. - // 3. Data below F is eventually deleted — eventually, because each store reclaims on its - // own schedule once the collector has released the range. - // - // A snapshot store reads these in terms of restore points rather than blocks: what it must - // hold is the newest snapshot at or below the boundary, which is what GetPruningBoundary - // answers. Guarantee 2 is also why pruning is to the shared minimum rather than to each - // store's own boundary — a retained snapshot is only restorable if the blocks that follow - // it survive in the contiguous stores. + // 2. No data at or above F is deleted. + // 3. Data below F is eventually deleted, on each store's own reclamation schedule. // - // 0 is allowed and waives the guarantee: cutLine then equals head for any store with - // retention 0, so those stores are pruned to their head. It is safe only when every - // participating store leaves enough history on its own — a positive GetRetentionWindow, - // or snapshot answers that hold the contiguous stores above genesis. It also disables - // the CannotServeRollback stop signal, so a store that has not snapshotted yet loses the - // range it would have replayed from. See that constant. - // - // After a rollback that consumes part of the window, full headroom is not promised - // (e.g. window 10_000, roll back 5_000 → only ~5_000 of headroom remain). - RollbackWindow uint64 + // Snapshots stop one window short of history, at the lowest rollback floor the stores report, + // since they are restore points and this window buys history to read rather than history to + // restore to. History goes LookbackWindow below that same floor, because a retained snapshot + // is only restorable if the blocks above it survive. See PrunableStore. + LookbackWindow uint64 // PruneInterval is how often the collector runs a prune cycle. Must be > 0. PruneInterval time.Duration @@ -44,24 +40,20 @@ type StorageGarbageCollectorConfig struct { // DefaultStorageGarbageCollectorConfig returns the default collector config. // -// Both values are deliberately lower than the pre-unification defaults (RollbackWindow 10_000, -// interval 60s), to match how these are actually used: -// -// - RollbackWindow 1_000. Real rollbacks are a few blocks deep, so 1_000 is already ample -// headroom, and this is only the correctness floor. The production shape is a short -// rollback window paired with much longer history, and that history now belongs on each -// store via GetRetentionWindow rather than being folded into this shared window — keeping -// it short here avoids charging every store for retention only some of them need. -// - PruneInterval 5m. Pruning reclaims whole files lazily; running it every minute buys -// nothing and costs I/O against a live node. Anything in the 5-10 minute range is fine. +// LookbackWindow is 0: extra history costs disk on every managed store at once, so it belongs at +// the call site where that cost is visible rather than in a default. func DefaultStorageGarbageCollectorConfig() *StorageGarbageCollectorConfig { return &StorageGarbageCollectorConfig{ RollbackWindow: 1_000, + LookbackWindow: 0, PruneInterval: 5 * time.Minute, } } // Validate checks that required fields are set to usable values. +// +// The windows are additive, so every combination of them is meaningful and neither is constrained +// against the other. A sum that overflows uint64 is handled in getHistoryCutLine. func (c *StorageGarbageCollectorConfig) Validate() error { if c == nil { return fmt.Errorf("config is required") diff --git a/sei-db/management/gc/storage_garbage_collector_test.go b/sei-db/management/gc/storage_garbage_collector_test.go index 2ecfbcc6eb..9f5af6e3b8 100644 --- a/sei-db/management/gc/storage_garbage_collector_test.go +++ b/sei-db/management/gc/storage_garbage_collector_test.go @@ -15,40 +15,40 @@ import ( type mockStore struct { name string latestHeight uint64 - // retentionWindow is returned by GetRetentionWindow (0 = shared RollbackWindow only). - retentionWindow int64 - pruningBoundary func(cutLine uint64) uint64 - getErr error - pruneErr error + floor func(rollbackWindow uint64) uint64 + pruneErr error // selfPruned inverts ExternalPruning: the constructors below build collector-managed stores, // which is the common case, so opting out is what a test has to say explicitly. selfPruned bool - pruneBelowCalled atomic.Bool - prunedBelow atomic.Uint64 - pruneBelowCalls atomic.Uint64 - boundaryCalls atomic.Uint64 + historyPruned atomic.Bool + prunedHistoryBelow atomic.Uint64 + historyPruneCalls atomic.Uint64 + + snapshotsPruned atomic.Bool + prunedSnapshotsBelow atomic.Uint64 + + floorCalls atomic.Uint64 } -// snapshotStore models SC/SS (retention 0). GetPruningBoundary returns the newest snapshot -// ≤ cutLine, or cutLine when every snapshot is above it (nothing can be dropped, and the -// contract forbids answering above cutLine). No snapshots at all → CannotServeRollback. -// Snapshots must be ascending. +// snapshotStore models SC/SS, mirroring flatkv's snapshotFloor: the newest snapshot at or below its +// own head less the window, or its oldest snapshot when every one of them is above that height, and 0 +// when it holds no snapshot or the window is deeper than its head. Snapshots must be ascending. func snapshotStore(name string, latestHeight uint64, snapshots ...uint64) *mockStore { return &mockStore{ - name: name, - latestHeight: latestHeight, - retentionWindow: 0, - pruningBoundary: func(cutLine uint64) uint64 { - if len(snapshots) == 0 { - return CannotServeRollback + name: name, + latestHeight: latestHeight, + floor: func(rollbackWindow uint64) uint64 { + if len(snapshots) == 0 || latestHeight <= rollbackWindow { + return 0 } - if snapshots[0] > cutLine { - return cutLine + target := latestHeight - rollbackWindow + if snapshots[0] > target { + return snapshots[0] } newest := snapshots[0] for _, snapshot := range snapshots { - if snapshot > cutLine { + if snapshot > target { break } newest = snapshot @@ -58,26 +58,24 @@ func snapshotStore(name string, latestHeight uint64, snapshots ...uint64) *mockS } } -// contiguousStore models blockDB / receiptDB / WAL (retention 0 by default): it can restore to -// any height it holds, so it always answers cutLine — including when it holds nothing at or below -// cutLine, where the resulting PruneBelow is simply a no-op. Use withRetentionWindow for extra -// retention or InfiniteRetentionWindow. +// contiguousStore models blockDB / receiptDB / WAL: it can restore to any height it holds, so it +// answers its own head less the window — and 0 where the window is deeper than that head, which +// includes the empty store and holds the prune height at 0. func contiguousStore(name string, latestHeight uint64) *mockStore { return &mockStore{ - name: name, - latestHeight: latestHeight, - retentionWindow: 0, - pruningBoundary: func(cutLine uint64) uint64 { return cutLine }, + name: name, + latestHeight: latestHeight, + floor: func(rollbackWindow uint64) uint64 { + if latestHeight <= rollbackWindow { + return 0 + } + return latestHeight - rollbackWindow + }, } } -func withRetentionWindow(store *mockStore, retention int64) *mockStore { - store.retentionWindow = retention - return store -} - -// withSelfPruning marks a store as enforcing its own retention, so the collector must vote it in but -// never call PruneBelow on it. +// withSelfPruning marks a store as enforcing its own retention, so the collector must count its +// answer but never prune it. func withSelfPruning(store *mockStore) *mockStore { store.selfPruned = true return store @@ -91,23 +89,25 @@ func (m *mockStore) Name() string { return m.name } -func (m *mockStore) GetRetentionWindow() int64 { - return m.retentionWindow +func (m *mockStore) GetLatestBlock() (uint64, error) { + return m.latestHeight, nil } -func (m *mockStore) GetLatestBlock() (uint64, error) { - return m.latestHeight, m.getErr +func (m *mockStore) GetRollbackFloor(rollbackWindow uint64) uint64 { + m.floorCalls.Add(1) + return m.floor(rollbackWindow) } -func (m *mockStore) GetPruningBoundary(cutLine uint64) uint64 { - m.boundaryCalls.Add(1) - return m.pruningBoundary(cutLine) +func (m *mockStore) PruneHistory(blockNumber uint64) error { + m.historyPruned.Store(true) + m.prunedHistoryBelow.Store(blockNumber) + m.historyPruneCalls.Add(1) + return m.pruneErr } -func (m *mockStore) PruneBelow(blockNumber uint64) error { - m.pruneBelowCalled.Store(true) - m.prunedBelow.Store(blockNumber) - m.pruneBelowCalls.Add(1) +func (m *mockStore) PruneSnapshots(blockNumber uint64) error { + m.snapshotsPruned.Store(true) + m.prunedSnapshotsBelow.Store(blockNumber) return m.pruneErr } @@ -119,41 +119,47 @@ func prunableStores(list ...*mockStore) []PrunableStore { return result } -func testConfig(t *testing.T, rollbackWindow uint64) *StorageGarbageCollectorConfig { +func testConfig(t *testing.T, rollbackWindow, lookbackWindow uint64) *StorageGarbageCollectorConfig { t.Helper() config := &StorageGarbageCollectorConfig{ RollbackWindow: rollbackWindow, + LookbackWindow: lookbackWindow, PruneInterval: time.Minute, } require.NoError(t, config.Validate()) return config } -// TestPruneDecisions covers one prune cycle. wantPruneBelow == nil means no store is pruned. +// TestPruneDecisions covers one prune cycle. wantHistoryBelow == nil means no store is touched at +// all; otherwise every collector-managed store is pruned to that height. The two are separate +// because a cycle can reach one cut line and not the other: the lookback window can take the history +// cut line below genesis while snapshots are still prunable. // -// wantPruned is hardcoded per store rather than derived from getCutLine / GetPruningBoundary: deriving it from the -// helpers under test would let a sign error or off-by-one shift the expectation in lockstep with the bug. +// Expectations are hardcoded per case rather than derived from getHistoryCutLine / +// GetRollbackFloor: deriving them from the code under test would let a sign error or off-by-one +// shift the expectation in lockstep with the bug. func TestPruneDecisions(t *testing.T) { cases := []struct { - name string - rollbackWindow uint64 - stores []*mockStore - wantPruneBelow *uint64 - wantPruned []bool + name string + rollbackWindow uint64 + lookbackWindow uint64 + stores []*mockStore + wantSnapshotsBelow *uint64 + wantHistoryBelow *uint64 }{ { - name: "SC and WAL both retention 0: min boundary wins", + name: "SC and WAL: min floor wins", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 100_000, 80_000, 85_000, 92_000), contiguousStore("stateWAL", 100_000), }, - // cutLine 90_000; sc keeps snapshot 85_000; WAL answers 90_000 → min 85_000. - wantPruneBelow: ptr(85_000), - wantPruned: []bool{true, true}, + // sc keeps snapshot 85_000; WAL answers 90_000; min 85_000, no lookback to subtract. + wantSnapshotsBelow: ptr(85_000), + wantHistoryBelow: ptr(85_000), }, { - name: "lowest boundary across many stores wins", + name: "lowest floor across many stores wins", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 100_000, 80_000, 85_000, 92_000), @@ -162,185 +168,170 @@ func TestPruneDecisions(t *testing.T) { contiguousStore("stateWAL", 100_000), }, // sc 85_000, ss 88_000, flatKV 90_000, WAL 90_000 → min 85_000. - wantPruneBelow: ptr(85_000), - wantPruned: []bool{true, true, true, true}, + wantSnapshotsBelow: ptr(85_000), + wantHistoryBelow: ptr(85_000), }, { - name: "RollbackWindow of 1 still leaves one block of margin", - rollbackWindow: 1, + // The lookback window deepens history without deepening the snapshot depth: restore + // points are only needed as far back as the rollback promise. + name: "lookback window deepens history only", + rollbackWindow: 10_000, + lookbackWindow: 5_000, stores: []*mockStore{ - snapshotStore("sc", 100_000, 80_000, 99_999), + snapshotStore("sc", 100_000, 95_000), contiguousStore("stateWAL", 100_000), }, - wantPruneBelow: ptr(99_999), - wantPruned: []bool{true, true}, + // sc's only snapshot is above head - 10_000, so it answers that snapshot, 95_000; WAL + // 90_000; min 90_000, and history goes 5_000 deeper. + wantSnapshotsBelow: ptr(90_000), + wantHistoryBelow: ptr(85_000), }, { - name: "RollbackWindow 0: cutLine equals head", - rollbackWindow: 0, + // The lookback window is subtracted from the minimum however deep that minimum already + // is, so it stacks below a snapshot floor rather than being capped by the head. + name: "lookback window stacks below the lowest floor", + rollbackWindow: 10_000, + lookbackWindow: 5_000, stores: []*mockStore{ - snapshotStore("sc", 100_000, 80_000, 90_000), + snapshotStore("sc", 100_000, 60_000), contiguousStore("stateWAL", 100_000), }, - // cutLine 100_000; sc 90_000; WAL 100_000 → min 90_000. - wantPruneBelow: ptr(90_000), - wantPruned: []bool{true, true}, + // sc holds a snapshot at 60_000, which binds; 5_000 blocks of lookback go below it. + wantSnapshotsBelow: ptr(60_000), + wantHistoryBelow: ptr(55_000), }, { - name: "positive contiguous retention deepens that store's cut line", - rollbackWindow: 10_000, + name: "lookback window alone bounds a contiguous fleet", + rollbackWindow: 1_000, + lookbackWindow: 50_000, stores: []*mockStore{ - snapshotStore("sc", 100_000, 80_000, 90_000), - withRetentionWindow(contiguousStore("stateWAL", 100_000), 5_000), + contiguousStore("blockDB", 100_000), + contiguousStore("receiptDB", 100_000), }, - // sc cutLine 90_000 → 90_000; WAL cutLine 85_000 → 85_000 → min 85_000. - wantPruneBelow: ptr(85_000), - wantPruned: []bool{true, true}, + // Both answer 99_000; history goes 50_000 deeper. + wantSnapshotsBelow: ptr(99_000), + wantHistoryBelow: ptr(49_000), }, { - name: "SS retention 0 behaves like SC", - rollbackWindow: 10_000, + name: "RollbackWindow of 1 still leaves one block of margin", + rollbackWindow: 1, stores: []*mockStore{ - snapshotStore("ss", 100_000, 80_000, 90_000), + snapshotStore("sc", 100_000, 80_000, 99_999), contiguousStore("stateWAL", 100_000), }, - wantPruneBelow: ptr(90_000), - wantPruned: []bool{true, true}, - }, - { - name: "infinite retention on every store skips pruning", - rollbackWindow: 10_000, - stores: []*mockStore{ - withRetentionWindow(contiguousStore("blockDB", 100_000), InfiniteRetentionWindow), - withRetentionWindow(contiguousStore("receiptDB", 100_000), InfiniteRetentionWindow), - }, - wantPruneBelow: nil, - wantPruned: []bool{false, false}, + wantSnapshotsBelow: ptr(99_999), + wantHistoryBelow: ptr(99_999), }, { - name: "infinite retention on one store leaves others free to prune", - rollbackWindow: 10_000, + name: "both windows 0: nothing is held back but the snapshot", + rollbackWindow: 0, stores: []*mockStore{ - withRetentionWindow(contiguousStore("archiveWAL", 100_000), InfiniteRetentionWindow), - snapshotStore("sc", 100_000, 80_000), + snapshotStore("sc", 100_000, 80_000, 90_000), contiguousStore("stateWAL", 100_000), }, - // archiveWAL skipped; sc 80_000; stateWAL 90_000 → min 80_000. - wantPruneBelow: ptr(80_000), - wantPruned: []bool{false, true, true}, + // sc 90_000; WAL 100_000 → min 90_000. + wantSnapshotsBelow: ptr(90_000), + wantHistoryBelow: ptr(90_000), }, { - name: "snapshot exactly at the cut line is kept", + name: "snapshot exactly at the window depth is kept", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 100_000, 50_000, 90_000), contiguousStore("stateWAL", 100_000), }, - wantPruneBelow: ptr(90_000), - wantPruned: []bool{true, true}, + wantSnapshotsBelow: ptr(90_000), + wantHistoryBelow: ptr(90_000), }, { - name: "all snapshots above cut line: store votes the cut line", + // The shortfall case: SC cannot restore as deep as the window asks, so it answers its + // oldest snapshot to keep that one replayable rather than waiving its claim. + name: "all snapshots above the window: store answers its oldest", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 100_000, 95_000, 97_000), contiguousStore("stateWAL", 100_000), }, - // Both answer 90_000 → shared prune 90_000 (sc's snapshots sit above it, untouched). - wantPruneBelow: ptr(90_000), - wantPruned: []bool{true, true}, + // sc 95_000; WAL 90_000 → min 90_000. + wantSnapshotsBelow: ptr(90_000), + wantHistoryBelow: ptr(90_000), }, { - // Regression: with no contiguous store to bound the min, an answer above cutLine - // would become pruneHeight and delete inside the rollback window. - name: "all snapshots above cut line with no contiguous store", + // The shortfall alone, with no contiguous store to bind the minimum. 95_000 sits above + // head - RollbackWindow, and nothing caps it: the store cannot restore below its oldest + // snapshot, so history below that buys a rollback no store could serve anyway. + name: "a shortfall answer alone sets the cut lines", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 100_000, 95_000, 97_000), }, - wantPruneBelow: ptr(90_000), - wantPruned: []bool{true}, + wantSnapshotsBelow: ptr(95_000), + wantHistoryBelow: ptr(95_000), }, { - name: "lagging store lowers the global head", + // Each store measures the window against its own head, so the WAL answers 90_000 off + // its own 100_000 while the lagging store answers 30_000 off its 50_000. The lagging + // store cannot be pruned past what it holds. + name: "each store answers from its own head", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("lagging", 50_000, 30_000, 50_000), contiguousStore("stateWAL", 100_000), }, - // head 50_000 → cutLine 40_000; lagging answers 30_000; WAL 40_000 → min 30_000. - wantPruneBelow: ptr(30_000), - wantPruned: []bool{true, true}, + wantSnapshotsBelow: ptr(30_000), + wantHistoryBelow: ptr(30_000), }, { - // A store with no snapshot will replay forward once its first one lands, so pruning - // the WAL to its own cut line now would delete the range it replays from. - name: "store with no snapshot blocks the whole cycle", + // A snapshot store with nothing to replay from cannot say which blocks the WAL may + // drop, so it answers 0 and the whole fleet stays where it is. + name: "store with no snapshot holds both cut lines at 0", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 100_000), contiguousStore("stateWAL", 100_000), }, - wantPruneBelow: nil, - wantPruned: []bool{false, false}, }, { - // Same store set, blocker last. This ordering is the one that matters: the WAL votes - // 90_000, so a usable pruneHeight is sitting there when the loop ends and only the - // blocked check stops it from being issued. Ordering the blocker first would leave - // pruneHeight at 0, and the cycle would stall on that alone. - name: "blocker after a store that already voted", + // A store that has ingested nothing is the same case: it answers 0, the minimum carries + // that through, and nothing is deleted anywhere. + name: "empty store holds both cut lines at 0", rollbackWindow: 10_000, stores: []*mockStore{ - contiguousStore("stateWAL", 100_000), - snapshotStore("sc", 100_000), - }, - wantPruneBelow: nil, - wantPruned: []bool{false, false}, - }, - { - // Same store set, but RollbackWindow 0 waives the guarantee, so there is nothing - // left for the snapshot-less store to protect and the others prune. - name: "no snapshot does not block when RollbackWindow is 0", - rollbackWindow: 0, - stores: []*mockStore{ - snapshotStore("sc", 100_000), + contiguousStore("blockDB", 0), contiguousStore("stateWAL", 100_000), }, - wantPruneBelow: ptr(100_000), - wantPruned: []bool{false, true}, }, { - name: "zero head ignored for global head; store still votes", + // A snapshot on disk does not make a store prunable-around while its head is behind the + // window: it answers from the head, so a stalled store holds everything back. + name: "stalled store with a snapshot still answers 0", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("stalled", 0, 50_000), contiguousStore("stateWAL", 100_000), }, - // head from WAL 100_000; stalled still answers 50_000 → min 50_000. - wantPruneBelow: ptr(50_000), - wantPruned: []bool{true, true}, }, { - name: "head inside retain window: no prune", + // Every store owes a rollback deeper than its whole history, so every one answers 0. + name: "head inside the rollback window: nothing deleted", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 5_000, 1_000, 2_000), contiguousStore("stateWAL", 5_000), }, - wantPruneBelow: nil, - wantPruned: []bool{false, false}, }, { - name: "head inside one store's window skips only that store", - rollbackWindow: 60_000, + // The one case where the cycle reaches one cut line and not the other: the rollback + // promise is satisfiable, so snapshots below 10_000 go, but the lookback window reaches + // below genesis from there and history is left alone. + name: "lookback window reaches below genesis: snapshots only", + rollbackWindow: 1_000, + lookbackWindow: 90_000, stores: []*mockStore{ - snapshotStore("sc", 100_000, 500, 1_000), - withRetentionWindow(contiguousStore("stateWAL", 100_000), 40_000), + snapshotStore("sc", 50_000, 10_000), + contiguousStore("stateWAL", 50_000), }, - // WAL cutLine 0 (skipped); sc cutLine 40_000 → 1_000. - wantPruneBelow: ptr(1_000), - wantPruned: []bool{true, false}, + wantSnapshotsBelow: ptr(10_000), }, { name: "no store has a latest block", @@ -349,78 +340,119 @@ func TestPruneDecisions(t *testing.T) { snapshotStore("sc", 0), contiguousStore("stateWAL", 0), }, - wantPruneBelow: nil, - wantPruned: []bool{false, false}, }, { name: "no stores at all", rollbackWindow: 10_000, stores: nil, - wantPruneBelow: nil, - wantPruned: nil, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - require.Len(t, tc.wantPruned, len(tc.stores), "wantPruned must have one entry per store") - - require.NoError(t, prune(testConfig(t, tc.rollbackWindow), prunableStores(tc.stores...))) + config := testConfig(t, tc.rollbackWindow, tc.lookbackWindow) + require.NoError(t, prune(config, prunableStores(tc.stores...))) + + for _, store := range tc.stores { + if tc.wantSnapshotsBelow == nil { + require.Falsef(t, store.snapshotsPruned.Load(), "%s snapshots should not be pruned", store.name) + } else { + require.Truef(t, store.snapshotsPruned.Load(), "%s snapshots should be pruned", store.name) + require.Equalf(t, *tc.wantSnapshotsBelow, store.prunedSnapshotsBelow.Load(), + "%s snapshot cut line", store.name) + } - for i, store := range tc.stores { - if !tc.wantPruned[i] { - require.Falsef(t, store.pruneBelowCalled.Load(), "%s should not be pruned", store.name) + if tc.wantHistoryBelow == nil { + require.Falsef(t, store.historyPruned.Load(), "%s history should not be pruned", store.name) continue } - require.NotNil(t, tc.wantPruneBelow, "wantPruned expects a prune height") - require.Truef(t, store.pruneBelowCalled.Load(), "%s should be pruned", store.name) - require.Equalf(t, *tc.wantPruneBelow, store.prunedBelow.Load(), "%s prune height", store.name) + require.Truef(t, store.historyPruned.Load(), "%s history should be pruned", store.name) + require.Equalf(t, *tc.wantHistoryBelow, store.prunedHistoryBelow.Load(), + "%s history cut line", store.name) } }) } } -func TestPruneOnYoungChainWithDeepContiguousRetention(t *testing.T) { - // head 100 << WAL retain window (rollback 1_000 + retention 100_000) → both cutLines 0. +// The whole reason PruneSnapshots and PruneHistory are separate calls: they are handed cut lines one +// lookback window apart. Collapsing them onto one height would either strand the retained snapshot +// (snapshots down at the history cut line are the restore points nothing can ask for) or throw away +// the lookback window (history stopping at the snapshot cut line). +func TestPruneUsesDifferentDepthsForSnapshotsAndHistory(t *testing.T) { + sc := snapshotStore("sc", 100_000, 60_000) + wal := contiguousStore("stateWAL", 100_000) + + require.NoError(t, prune(testConfig(t, 10_000, 5_000), prunableStores(sc, wal))) + + require.Equal(t, uint64(60_000), sc.prunedSnapshotsBelow.Load(), + "sc's own snapshot binds the minimum, and it is never itself a candidate") + require.Equal(t, uint64(55_000), wal.prunedHistoryBelow.Load(), + "history goes a lookback window below that snapshot, keeping it replayable") + require.Equal(t, uint64(60_000), wal.prunedSnapshotsBelow.Load(), + "a contiguous store is still called; it holds no snapshots and no-ops") +} + +// One cut line reaching genesis must not silence the other. Here the rollback promise is satisfiable +// so snapshots below the floor are dead weight and go, while the lookback window from that floor +// lands below genesis and history is left untouched. +func TestPruneStillPrunesSnapshotsWhenHistoryIsHeldAtZero(t *testing.T) { + sc := snapshotStore("sc", 50_000, 10_000, 40_000) + wal := contiguousStore("stateWAL", 50_000) + + require.NoError(t, prune(testConfig(t, 1_000, 90_000), prunableStores(sc, wal))) + + require.False(t, sc.historyPruned.Load(), "the lookback window reaches below genesis") + require.True(t, sc.snapshotsPruned.Load(), "snapshots are still pruned") + require.Equal(t, uint64(40_000), sc.prunedSnapshotsBelow.Load(), + "sc's floor snapshot binds, so the 10_000 one below it goes") +} + +// A chain younger than its own rollback window must lose nothing: every store owes a rollback +// deeper than its whole history, so every one answers 0 and the prune height stays there. The +// deletions are still issued — they are no-ops at 0 — which is what keeps the collector free of a +// special case for the young chain. +func TestPruneOnYoungChain(t *testing.T) { sc := snapshotStore("sc", 100, 50, 100) - wal := withRetentionWindow(contiguousStore("stateWAL", 100), 100_000) + wal := contiguousStore("stateWAL", 100) - require.NoError(t, prune(testConfig(t, 1_000), prunableStores(sc, wal))) + require.NoError(t, prune(testConfig(t, 1_000, 100_000), prunableStores(sc, wal))) - require.False(t, sc.pruneBelowCalled.Load()) - require.False(t, wal.pruneBelowCalled.Load()) + require.Equal(t, uint64(0), sc.prunedHistoryBelow.Load()) + require.Equal(t, uint64(0), wal.prunedHistoryBelow.Load()) } -// Answers are held positionally rather than keyed by Name(), so a store sharing a name with a -// participating one must not inherit its prune. The stake is data loss: an infinite-retention -// store exists precisely to keep what the others are dropping. -func TestPruneKeepsInfiniteRetentionStoreWithDuplicateName(t *testing.T) { - archive := withRetentionWindow(contiguousStore("ss", 100_000), InfiniteRetentionWindow) +// Answers are held positionally rather than keyed by Name(), so a store sharing a name with another +// must not inherit its decision. The stake is data loss: only the blocked store's own answer says +// whether the range it needs still exists. +func TestPruneKeepsAnswersPositionalWithDuplicateNames(t *testing.T) { + selfPruned := withSelfPruning(contiguousStore("ss", 100_000)) participating := snapshotStore("ss", 100_000, 80_000) wal := contiguousStore("stateWAL", 100_000) - require.NoError(t, prune(testConfig(t, 10_000), prunableStores(archive, participating, wal))) + require.NoError(t, prune(testConfig(t, 10_000, 0), prunableStores(selfPruned, participating, wal))) - require.False(t, archive.pruneBelowCalled.Load(), "an infinite-retention store must never be pruned") - require.True(t, participating.pruneBelowCalled.Load()) - require.Equal(t, uint64(80_000), participating.prunedBelow.Load()) - require.Equal(t, uint64(80_000), wal.prunedBelow.Load()) + require.False(t, selfPruned.historyPruned.Load(), "a self-pruning store must never be pruned") + require.True(t, participating.historyPruned.Load()) + require.Equal(t, uint64(80_000), participating.prunedHistoryBelow.Load()) + require.Equal(t, uint64(80_000), wal.prunedHistoryBelow.Load()) } -func TestPruneGetLatestBlockError(t *testing.T) { - sentinel := errors.New("boom") +// A store that cannot read its own head has no separate error path to the collector: it answers a +// floor of 0, which the minimum carries into both cut lines, and the cycle deletes nothing anywhere. +// Refusing to delete is the whole requirement, and it falls out of the ordinary reduction. +func TestPruneUnreadableStoreStopsTheCycle(t *testing.T) { sc := snapshotStore("sc", 100_000, 80_000) - broken := contiguousStore("brokenStore", 100_000) - broken.getErr = sentinel + unreadable := contiguousStore("brokenStore", 100_000) + unreadable.floor = func(uint64) uint64 { return 0 } - err := prune(testConfig(t, 10_000), prunableStores(sc, broken)) - require.ErrorIs(t, err, sentinel) - require.ErrorContains(t, err, "brokenStore") - require.False(t, sc.pruneBelowCalled.Load()) - require.False(t, broken.pruneBelowCalled.Load()) + require.NoError(t, prune(testConfig(t, 10_000, 0), prunableStores(sc, unreadable))) + + require.False(t, sc.historyPruned.Load()) + require.False(t, sc.snapshotsPruned.Load()) + require.False(t, unreadable.historyPruned.Load()) } -func TestPruneBelowErrorContinuesRemainingStores(t *testing.T) { +func TestPruneErrorContinuesRemainingStores(t *testing.T) { firstErr := errors.New("boom1") secondErr := errors.New("boom2") first := snapshotStore("first", 100_000, 80_000) @@ -429,209 +461,145 @@ func TestPruneBelowErrorContinuesRemainingStores(t *testing.T) { second.pruneErr = secondErr wal := contiguousStore("stateWAL", 100_000) - err := prune(testConfig(t, 10_000), prunableStores(first, second, wal)) + err := prune(testConfig(t, 10_000, 0), prunableStores(first, second, wal)) require.ErrorIs(t, err, firstErr) require.ErrorIs(t, err, secondErr) require.ErrorContains(t, err, "first") require.ErrorContains(t, err, "second") require.ErrorContains(t, err, "80000") - require.True(t, first.pruneBelowCalled.Load()) - require.True(t, second.pruneBelowCalled.Load()) - require.True(t, wal.pruneBelowCalled.Load()) - require.Equal(t, uint64(80_000), wal.prunedBelow.Load()) + require.True(t, first.historyPruned.Load()) + require.True(t, second.historyPruned.Load()) + require.True(t, wal.historyPruned.Load()) + require.Equal(t, uint64(80_000), wal.prunedHistoryBelow.Load()) } -func TestGetCutLine(t *testing.T) { - cases := []struct { - name string - head uint64 - rollbackWindow uint64 - retention int64 - want uint64 - }{ - {name: "rollback window only", head: 100_000, rollbackWindow: 10_000, want: 90_000}, - {name: "zero rollback window", head: 100_000, rollbackWindow: 0, want: 100_000}, - {name: "zero rollback with retention", head: 100_000, rollbackWindow: 0, retention: 10_000, want: 90_000}, - {name: "retention adds to rollback window", head: 100_000, rollbackWindow: 1, retention: 10_000, want: 89_999}, - {name: "the two windows add", head: 100_000, rollbackWindow: 10_000, retention: 5_000, want: 85_000}, - {name: "zero retention", head: 100_000, rollbackWindow: 10_000, want: 90_000}, - {name: "infinite retention", head: 100_000, rollbackWindow: 10_000, retention: InfiniteRetentionWindow, want: 0}, - {name: "any negative retention is infinite", head: 100_000, rollbackWindow: 10_000, retention: -99, want: 0}, - {name: "head one above the window", head: 10_001, rollbackWindow: 10_000, want: 1}, - {name: "head exactly at the window", head: 10_000, rollbackWindow: 10_000, want: 0}, - {name: "head one below the window", head: 9_999, rollbackWindow: 10_000, want: 0}, - {name: "head far below the window", head: 100, rollbackWindow: 1_000, retention: 100_000, want: 0}, - {name: "head at genesis", head: 0, rollbackWindow: 10_000, want: 0}, - {name: "rollback plus retention overflows uint64", head: math.MaxUint64, rollbackWindow: math.MaxUint64, retention: 1, want: 0}, - {name: "max rollback alone does not overflow", head: math.MaxUint64, rollbackWindow: math.MaxUint64, want: 0}, - } +// A failure pruning snapshots must not skip the history prune on the same store: the two are +// independent deletions, and permission-to-drop is not transactional. +func TestPruneHistoryStillRunsAfterSnapshotError(t *testing.T) { + sentinel := errors.New("boom") + sc := snapshotStore("sc", 100_000, 80_000) + sc.pruneErr = sentinel - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - require.Equal(t, tc.want, getCutLine(tc.head, tc.rollbackWindow, tc.retention)) - }) - } + err := prune(testConfig(t, 10_000, 0), prunableStores(sc)) + require.ErrorIs(t, err, sentinel) + require.ErrorContains(t, err, "snapshots below 80000") + require.ErrorContains(t, err, "history below 80000") + require.True(t, sc.historyPruned.Load()) } -func TestGetGlobalLatestBlock(t *testing.T) { - storesWithHeads := func(heads ...uint64) []PrunableStore { - list := make([]*mockStore, len(heads)) - for i, head := range heads { - list[i] = contiguousStore("store", head) - } - return prunableStores(list...) - } - +func TestGetHistoryCutLine(t *testing.T) { cases := []struct { - name string - heads []uint64 - want uint64 + name string + snapshotCutLine uint64 + lookbackWindow uint64 + want uint64 }{ - {name: "single store", heads: []uint64{100}, want: 100}, - {name: "all agree", heads: []uint64{100, 100, 100}, want: 100}, - {name: "smallest first", heads: []uint64{80, 100}, want: 80}, - {name: "smallest last", heads: []uint64{100, 80}, want: 80}, - {name: "smallest in the middle", heads: []uint64{100, 50, 90}, want: 50}, - {name: "leading zero ignored", heads: []uint64{0, 100}, want: 100}, - {name: "trailing zero ignored", heads: []uint64{100, 0}, want: 100}, - {name: "zero among many ignored", heads: []uint64{100, 0, 80}, want: 80}, - {name: "every store reports zero", heads: []uint64{0, 0}, want: 0}, - {name: "no stores", heads: nil, want: 0}, + {name: "no lookback leaves the cut line alone", snapshotCutLine: 90_000, want: 90_000}, + {name: "lookback deepens the cut line", snapshotCutLine: 90_000, lookbackWindow: 5_000, want: 85_000}, + {name: "one block of lookback", snapshotCutLine: 90_000, lookbackWindow: 1, want: 89_999}, + {name: "lookback one below the cut line", snapshotCutLine: 5_001, lookbackWindow: 5_000, want: 1}, + {name: "lookback exactly at the cut line", snapshotCutLine: 5_000, lookbackWindow: 5_000, want: 0}, + {name: "lookback past genesis", snapshotCutLine: 100, lookbackWindow: 100_000, want: 0}, + {name: "a cut line of 0 stays 0", snapshotCutLine: 0, lookbackWindow: 10, want: 0}, + {name: "no cut line and no lookback", want: 0}, + {name: "max cut line", snapshotCutLine: math.MaxUint64, lookbackWindow: 1, want: math.MaxUint64 - 1}, + {name: "max lookback", snapshotCutLine: 100, lookbackWindow: math.MaxUint64, want: 0}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - head, err := getGlobalLatestBlock(storesWithHeads(tc.heads...)) - require.NoError(t, err) - require.Equal(t, tc.want, head) + require.Equal(t, tc.want, getHistoryCutLine(tc.snapshotCutLine, tc.lookbackWindow)) }) } } -// The prune log is the stated mechanism for reconstructing a deletion after the fact, so the three -// outcomes must not collapse onto the same rendering: never asked (cutLine 0), unable to serve a -// rollback, and asked with a boundary. The first two both carry boundary 0, so cutLine is the only -// thing separating them. -func TestDescribeDecisionsRendersEachOutcomeDistinctly(t *testing.T) { +// The prune log is the stated mechanism for reconstructing a deletion after the fact, so every store +// has to appear whatever it answered — including the one answering 0, which is the entry that explains +// a cycle that deleted nothing — and a store the collector does not prune has to be distinguishable +// from one it pruned to no effect. +func TestDescribeDecisionsRendersEveryStore(t *testing.T) { stores := prunableStores( - withRetentionWindow(contiguousStore("archiveWAL", 100_000), InfiniteRetentionWindow), - snapshotStore("sc", 100_000), + contiguousStore("blockDB", 0), contiguousStore("stateWAL", 100_000), withSelfPruning(snapshotStore("selfSC", 100_000, 50_000)), ) decisions := []storeDecision{ - {cutLine: 0, boundary: CannotServeRollback}, // skipped before being asked - {cutLine: 90_000, boundary: CannotServeRollback}, // asked, cannot serve a rollback - {cutLine: 90_000, boundary: 90_000, externalPruning: true}, // asked, reported a boundary - {cutLine: 90_000, boundary: 50_000}, // asked, but prunes itself + {floor: 0, externalPruning: true}, // asked, needs everything kept + {floor: 90_000, externalPruning: true}, // asked, named a floor + {floor: 50_000}, // asked, but prunes itself } require.Equal(t, - "archiveWAL=notAsked sc=cannotServeRollback(cutLine=90000) "+ - "stateWAL=90000(cutLine=90000) selfSC=50000(cutLine=90000,selfPruned)", + "blockDB=0 stateWAL=90000 selfSC=50000(selfPruned)", describeDecisions(stores, decisions), ) } // The whole point of ExternalPruning: a self-pruning store is still a full participant in the -// decision — its boundary drags the shared minimum down and protects the range it replays from — -// while never being pruned by the collector. Withdrawing it from the vote instead would prune the -// WAL to 99_000 and strand its snapshot at 50_000 with nothing to replay forward. -func TestPruneSkipsSelfPruningStoreButKeepsItsVote(t *testing.T) { +// decision — its floor drags the shared minimum down and protects the range it replays from — while +// never being pruned by the collector. Withdrawing its answer instead would prune the WAL to 99_000 +// and strand its snapshot at 50_000 with nothing to replay forward. +func TestPruneSkipsSelfPruningStoreButKeepsItsAnswer(t *testing.T) { selfPruned := withSelfPruning(snapshotStore("sc", 100_000, 50_000)) wal := contiguousStore("stateWAL", 100_000) - require.NoError(t, prune(testConfig(t, 1_000), prunableStores(selfPruned, wal))) + require.NoError(t, prune(testConfig(t, 1_000, 0), prunableStores(selfPruned, wal))) - require.Equal(t, uint64(1), selfPruned.boundaryCalls.Load(), "a self-pruning store is still asked") - require.False(t, selfPruned.pruneBelowCalled.Load(), "its own pruner enforces its retention") - require.True(t, wal.pruneBelowCalled.Load()) - require.Equal(t, uint64(50_000), wal.prunedBelow.Load(), - "the self-pruning store's boundary must still hold the WAL back to its snapshot") + require.Equal(t, uint64(1), selfPruned.floorCalls.Load(), "a self-pruning store is still asked") + require.False(t, selfPruned.historyPruned.Load(), "its own pruner enforces its retention") + require.False(t, selfPruned.snapshotsPruned.Load(), "including its snapshots") + require.True(t, wal.historyPruned.Load()) + require.Equal(t, uint64(50_000), wal.prunedHistoryBelow.Load(), + "the self-pruning store's floor must still hold the WAL back to its snapshot") } -// A self-pruning store that cannot serve a rollback still abandons the cycle. Who deletes its data -// is a separate question from whether the range it needs exists yet, and only the latter is what -// CannotServeRollback reports. -func TestPruneSelfPruningStoreStillBlocksCycle(t *testing.T) { - blocker := withSelfPruning(snapshotStore("sc", 100_000)) // no snapshots +// A self-pruning store that has ingested nothing still holds history where it is. Who deletes its +// data is a separate question from whether the range it will need exists yet, and only the second is +// what its floor reports. +func TestPruneSelfPruningEmptyStoreStillHoldsHistory(t *testing.T) { + empty := withSelfPruning(contiguousStore("ss", 0)) wal := contiguousStore("stateWAL", 100_000) - require.NoError(t, prune(testConfig(t, 1_000), prunableStores(blocker, wal))) - - require.False(t, wal.pruneBelowCalled.Load(), "the cycle must be abandoned") -} - -// One blocker abandons the cycle, but every store is still asked so the blocked cycle can log a -// complete decision set. Under RollbackWindow 0 the guarantee is waived and the blocker is ignored -// outright, which is what lets the later stores prune. -func TestPruneAsksEveryStoreEvenWhenBlocked(t *testing.T) { - blocker := snapshotStore("sc", 100_000) // no snapshot -> CannotServeRollback - afterBlocker := contiguousStore("stateWAL", 100_000) - require.NoError(t, prune(testConfig(t, 1_000), prunableStores(blocker, afterBlocker))) - - require.Equal(t, uint64(1), blocker.boundaryCalls.Load()) - require.False(t, afterBlocker.pruneBelowCalled.Load(), "the cycle must be abandoned") - require.Equal(t, uint64(1), afterBlocker.boundaryCalls.Load(), - "a blocked cycle still asks every store, so its log names every blocker at once") - - waived := snapshotStore("sc", 100_000) - afterWaived := contiguousStore("stateWAL", 100_000) - require.NoError(t, prune(testConfig(t, 0), prunableStores(waived, afterWaived))) + require.NoError(t, prune(testConfig(t, 1_000, 0), prunableStores(empty, wal))) - require.Equal(t, uint64(1), afterWaived.boundaryCalls.Load(), - "RollbackWindow 0 ignores the blocker, so later stores must still be asked") - require.True(t, afterWaived.pruneBelowCalled.Load()) - require.Equal(t, uint64(100_000), afterWaived.prunedBelow.Load()) + require.Equal(t, uint64(0), wal.prunedHistoryBelow.Load(), "history must not move past an empty store") + require.False(t, empty.historyPruned.Load(), "its own pruner enforces its retention") } -// InfiniteRetentionWindow on a snapshot store is a footgun, and this pins which way it cuts. A -// store with no cut line is never asked, so it leaves the minimum entirely and the WAL holding its -// replay range is pruned to the WAL's own cut line — the old snapshot survives but can no longer be -// replayed forward. Retention 0 protects it strictly better, because then it answers its oldest -// needed snapshot and drags the WAL down to it. -// -// The other infinite-retention cases in TestPruneDecisions only cover the safe direction, where the -// infinite-retention store is the contiguous archive (a replay source, not a consumer). -func TestPruneInfiniteRetentionSnapshotStore(t *testing.T) { - archive := withRetentionWindow(snapshotStore("archiveSC", 100_000, 20_000), InfiniteRetentionWindow) +// Every store is asked exactly once per cycle, whatever the others answer, so the log carries a +// complete decision set and a store cannot be skipped on the strength of an earlier one's answer. +func TestPruneAsksEveryStoreOncePerCycle(t *testing.T) { + empty := contiguousStore("blockDB", 0) + sc := snapshotStore("sc", 100_000, 80_000) wal := contiguousStore("stateWAL", 100_000) - require.NoError(t, prune(testConfig(t, 1_000), prunableStores(archive, wal))) - - require.Zero(t, archive.boundaryCalls.Load(), "no cut line means it is never asked") - require.False(t, archive.pruneBelowCalled.Load(), "infinite retention is never pruned") - require.True(t, wal.pruneBelowCalled.Load()) - require.Equal(t, uint64(99_000), wal.prunedBelow.Load(), - "the archive contributes nothing, so the WAL only honors its own cut line") - // Same store, contract-sanctioned retention of 0: it participates and protects its snapshot. - compliant := snapshotStore("archiveSC", 100_000, 20_000) - wal2 := contiguousStore("stateWAL", 100_000) - require.NoError(t, prune(testConfig(t, 1_000), prunableStores(compliant, wal2))) + require.NoError(t, prune(testConfig(t, 1_000, 0), prunableStores(empty, sc, wal))) - require.Equal(t, uint64(20_000), wal2.prunedBelow.Load(), - "retention 0 pulls the WAL down to the snapshot, keeping it replayable") + for _, store := range []*mockStore{empty, sc, wal} { + require.Equalf(t, uint64(1), store.floorCalls.Load(), "%s asked exactly once", store.name) + } + require.Equal(t, uint64(0), wal.prunedHistoryBelow.Load(), "the empty store's answer binds the cycle") } func TestDefaultStorageGarbageCollectorConfig(t *testing.T) { cfg := DefaultStorageGarbageCollectorConfig() require.Equal(t, uint64(1_000), cfg.RollbackWindow) + require.Equal(t, uint64(0), cfg.LookbackWindow) require.Equal(t, 5*time.Minute, cfg.PruneInterval) require.NoError(t, cfg.Validate()) } +// Both windows are independent, so every combination validates. Only the interval is constrained. func TestValidate(t *testing.T) { require.ErrorContains(t, (*StorageGarbageCollectorConfig)(nil).Validate(), "config is required") - require.NoError(t, (&StorageGarbageCollectorConfig{ - RollbackWindow: 0, - PruneInterval: time.Minute, - }).Validate()) - - require.NoError(t, (&StorageGarbageCollectorConfig{ - RollbackWindow: 1, - PruneInterval: time.Minute, - }).Validate()) + for _, windows := range [][2]uint64{{0, 0}, {1, 0}, {0, 1}, {1_000, 50_000}, {50_000, 1_000}} { + require.NoError(t, (&StorageGarbageCollectorConfig{ + RollbackWindow: windows[0], + LookbackWindow: windows[1], + PruneInterval: time.Minute, + }).Validate(), "windows %v", windows) + } require.ErrorContains(t, (&StorageGarbageCollectorConfig{ RollbackWindow: 1, @@ -710,13 +678,13 @@ func TestRunTickerDrivesPruneCycles(t *testing.T) { startCollector(t, 10*time.Millisecond, sc, wal) require.Eventually(t, func() bool { - return sc.pruneBelowCalled.Load() && wal.pruneBelowCalled.Load() + return sc.historyPruned.Load() && wal.historyPruned.Load() }, 2*time.Second, 5*time.Millisecond, "ticker should drive a prune cycle") - require.Equal(t, uint64(80_000), sc.prunedBelow.Load()) - require.Equal(t, uint64(80_000), wal.prunedBelow.Load()) + require.Equal(t, uint64(80_000), sc.prunedHistoryBelow.Load()) + require.Equal(t, uint64(80_000), wal.prunedHistoryBelow.Load()) } -// TestRunSurvivesPruneError covers run()'s logger.Error branch: a PruneBelow failure is logged and the loop keeps +// TestRunSurvivesPruneError covers run()'s logger.Error branch: a prune failure is logged and the loop keeps // ticking rather than exiting after the first error. func TestRunSurvivesPruneError(t *testing.T) { broken := snapshotStore("broken", 100_000, 80_000) @@ -725,7 +693,7 @@ func TestRunSurvivesPruneError(t *testing.T) { startCollector(t, 10*time.Millisecond, broken) require.Eventually(t, func() bool { - return broken.pruneBelowCalls.Load() > 1 + return broken.historyPruneCalls.Load() > 1 }, 2*time.Second, 5*time.Millisecond, "a failed prune must not kill the run loop") } diff --git a/sei-db/state_db/sc/flatkv/snapshot.go b/sei-db/state_db/sc/flatkv/snapshot.go index 12f6059ede..1d6b95676b 100644 --- a/sei-db/state_db/sc/flatkv/snapshot.go +++ b/sei-db/state_db/sc/flatkv/snapshot.go @@ -511,7 +511,7 @@ func (s *CommitStore) WriteSnapshot(_ string) (err error) { logger.Error("failed to update SNAPSHOT_BASE", "err", err) } - pruned = s.pruneSnapshots(dir, version) + pruned = s.pruneSnapshotsByCount(dir, version) success = true s.lastSnapshotTime = time.Now() @@ -523,15 +523,17 @@ func (s *CommitStore) WriteSnapshot(_ string) (err error) { return nil } -// pruneSnapshots removes old snapshots beyond SnapshotKeepRecent, keeping +// pruneSnapshotsByCount removes old snapshots beyond SnapshotKeepRecent, keeping // the latest snapshot (currentVersion) plus the N most recent older ones. // Best-effort: errors are logged but do not fail the snapshot operation. // // Disabled by config.ExternalPruning, which hands retention to the -// StorageGarbageCollector. Both must never run: this one counts snapshots and -// knows nothing of the shared rollback window, so it would delete the snapshot -// the collector is holding to serve it. -func (s *CommitStore) pruneSnapshots(dir string, currentVersion int64) int { +// StorageGarbageCollector and its PruneSnapshots. Both must never run: this one +// counts snapshots and knows nothing of the shared rollback window, so it would +// delete the snapshot the collector is holding to serve it. Counting is what the +// name records — it is the distinction from the collector's by-block-height +// PruneSnapshots, not a second implementation of the same policy. +func (s *CommitStore) pruneSnapshotsByCount(dir string, currentVersion int64) int { if s.config.ExternalPruning { return 0 } diff --git a/sei-db/state_db/sc/flatkv/store_gc.go b/sei-db/state_db/sc/flatkv/store_gc.go index 3ab691f0e7..78b1804ffa 100644 --- a/sei-db/state_db/sc/flatkv/store_gc.go +++ b/sei-db/state_db/sc/flatkv/store_gc.go @@ -11,16 +11,16 @@ import ( // CommitStore joins the shared prune cycle as a snapshot store: it can restore only at a snapshot // boundary, replaying the state WAL forward from there to reach any higher height. So the history it -// must hold to serve a rollback is not a block range but the newest snapshot at or below the target, -// which is what GetPruningBoundary reports and what holds the WAL back for it. +// must hold to serve a rollback is not a block range but a single snapshot, which is what +// GetRollbackFloor reports and what holds the WAL back for it. // // Participation is conditional on config.ExternalPruning, which is what stands the store's own two -// pruners down — see ExternalPruning below. With it unset the store is still asked for its boundary, -// and still protects the WAL it replays from, but enforces its own retention by snapshot count. +// pruners down — see ExternalPruning below. With it unset the store is still asked for its floor, and +// still protects the WAL it replays from, but enforces its own retention by snapshot count. // // Precondition beyond the collector's own: the state WAL must be managed alongside this store. The // snapshot alone only restores the exact height it was taken at; every height above it needs the WAL -// blocks that follow. Managed without the WAL, this store answers a boundary nothing acts on while the +// blocks that follow. Managed without the WAL, this store answers a floor nothing acts on while the // WAL is pruned on its own schedule — and under ExternalPruning tryTruncateWAL has stood down too, so // nothing bounds that WAL at all. // @@ -34,10 +34,11 @@ func (s *CommitStore) Name() string { return "FlatKV" } -// ExternalPruning reports config.ExternalPruning, the same field pruneSnapshots and tryTruncateWAL -// consult to stand down. Reading one field from all three is what makes "the collector prunes this -// store" and "this store does not prune itself" a single fact: there is no combination of settings -// that turns both on, so the count-based pruner can never delete a snapshot the collector is holding. +// ExternalPruning reports config.ExternalPruning, the same field pruneSnapshotsByCount and +// tryTruncateWAL consult to stand down. Reading one field from all three is what makes "the collector +// prunes this store" and "this store does not prune itself" a single fact: there is no combination of +// settings that turns both on, so the count-based pruner can never delete a snapshot the collector is +// holding. // // It is a construction-time value, so the answer is stable for the life of the store and safe to read // from the collector's goroutine. @@ -45,46 +46,51 @@ func (s *CommitStore) ExternalPruning() bool { return s.config.ExternalPruning } -// PruneBelow deletes every snapshot below blockNumber, leaving the WAL alone — the collector prunes -// that directly as its own store. +// PruneHistory does nothing: what this store replays over is the state WAL, and the collector manages +// that as a store in its own right, pruning it to the minimum across every store rather than to what +// this one alone still needs. Truncating it from here would apply a narrower view to a WAL that SS +// replays from too. +func (s *CommitStore) PruneHistory(uint64) error { + return nil +} + +// PruneSnapshots deletes every snapshot strictly below blockNumber. Restoring to any height at or +// above it starts from a snapshot at or above it and replays the WAL forward, which is what makes +// the ones underneath restore points nothing can ask for. // -// The boundary snapshot this store reported survives, because blockNumber is a minimum across stores -// and so never exceeds that boundary (see GetPruningBoundary). It is therefore always left with at -// least one snapshot to restore from. +// The snapshot this store reported from GetRollbackFloor is never a candidate: blockNumber is the +// minimum across every store, so it sits at or below that answer. Keeping the reported restore point +// therefore follows from the collector's reduction and needs no guard here. The active snapshot is a +// separate question, and pruneCutLine is where that one is answered. // // Deletion is not all-or-nothing: a snapshot that fails to delete is reported and the rest are still // attempted, since each snapshot directory is independent and a single undeletable one must not strand // the disk space of the others. Already-gone snapshots are not an error — the SnapshotKeepRecent // pruner in WriteSnapshot deletes from the same set, and losing that race means the work is done. -func (s *CommitStore) PruneBelow(blockNumber uint64) error { +func (s *CommitStore) PruneSnapshots(blockNumber uint64) error { if blockNumber == 0 { - return nil + return nil // nothing is eligible; the collector does not call with this } - - // The active snapshot is what the next open resolves to, so it is never a candidate however deep the - // request. Nothing in the contract should produce such a request — a boundary is never above the - // snapshot it names — but the cost of the check is one readlink against making a wrong answer - // elsewhere unbootable here. - _, activeVersion, err := currentSnapshotDir(s.flatkvDir()) + heights, err := s.snapshotHeights() if err != nil { - return fmt.Errorf("resolve active snapshot before pruning: %w", err) + return fmt.Errorf("scan snapshots: %w", err) } - - blocks, err := s.snapshotBlocks() + if len(heights) == 0 { + return nil + } + active, err := s.activeSnapshotHeight() if err != nil { - return fmt.Errorf("scan snapshots: %w", err) + return fmt.Errorf("prune snapshots below %d: %w", blockNumber, err) } + cutLine := min(blockNumber, active) var errs error pruned := 0 - for _, block := range blocks { - if block >= blockNumber { + for _, height := range heights { + if height >= cutLine { break // ascending, so nothing further is a candidate } - if block == uint64(activeVersion) { //nolint:gosec // snapshot versions are non-negative - continue - } - removed, err := s.deleteSnapshot(block) + removed, err := s.deleteSnapshot(height) if err != nil { errs = errors.Join(errs, err) continue @@ -95,18 +101,105 @@ func (s *CommitStore) PruneBelow(blockNumber uint64) error { } // One line per cycle rather than one per snapshot: the count is the whole story here, since the - // set pruned is always "everything below the floor". A cycle that prunes nothing is the common + // set pruned is always "everything below the cut line". A cycle that prunes nothing is the common // case and says nothing, so it stays silent. if pruned > 0 { - logger.Info("pruned snapshots below retention floor", "count", pruned, "floor", blockNumber) + logger.Info("pruned snapshots below the rollback cut line", "count", pruned, "cutLine", cutLine) } return errs } -// snapshotBlocks returns the block number of every snapshot on disk, ascending. A missing snapshot +// activeSnapshotHeight returns the height of the snapshot "current" points at: the one the next open +// clones into the working directory and replays the state WAL forward from. +// +// Both halves of this store's GC surface are bounded by it, and for the same reason — this is where +// the store will actually resume from, whatever else is on disk: +// +// the floor it reports must not sit above it, or the collector prunes the WAL past the blocks +// that open needs to replay +// the cut line it prunes to must not either, or the deletion takes the very snapshot open +// resolves to +// +// Normally it is the newest snapshot, so both bounds are slack and neither changes an answer. +// Normally, not always — two ordinary failures leave "current" below the newest directory on disk: +// +// a crash between WriteSnapshot's rename and its symlink update, which leaves the new snapshot +// on disk with "current" still on the previous one; the next open takes the symlink, so the +// newer directory stays an orphan rather than being adopted +// a rollback that fails partway, which repoints "current" at the rollback base and only clears +// the snapshots above it at the very end +// +// Both bounds fail quietly if they are missing, which is why they are worth their cost: os.Readlink +// resolves a dangling symlink, so a store that lost its active snapshot opens against a directory +// that is not there rather than failing where the mistake was made. +func (s *CommitStore) activeSnapshotHeight() (uint64, error) { + _, version, err := currentSnapshotDir(s.flatkvDir()) + if err != nil { + return 0, fmt.Errorf("resolve active snapshot: %w", err) + } + return uint64(max(version, 0)), nil //nolint:gosec // clamped non-negative above +} + +// snapshotFloor picks the oldest snapshot that has to survive a rollback of rollbackWindow blocks +// behind head. 0 means nothing here is eligible for pruning. +// +// a snapshot at or below head - rollbackWindow → the newest such snapshot, since restoring +// anywhere inside the window starts from it and replays the WAL forward, and the ones below +// it are restore points nothing can ask for +// every snapshot above that height → the oldest snapshot, which is then the deepest this store +// can restore to at all +// no snapshot at all, or a window deeper than the head → 0 +// +// The second case is a snapshot-retention shortfall: the oldest snapshot on disk is newer than +// head - rollbackWindow, so the store cannot in fact restore as deep as it is being asked to. This +// is the normal state early on, before snapshots reach that far back, and can also arise from a +// large SnapshotInterval relative to the window. Reporting its oldest snapshot is what keeps that +// shortfall from compounding — the collector holds every store's history back to it, so the one +// snapshot this store does have stays replayable. It resolves on its own as the chain advances and +// snapshots accumulate below the window. +// +// Version 0 is never the answer. It restores to no committed height, so it is not a restore point at +// all, and it is itself a candidate for deletion once a real snapshot sits above it. A store holding +// only that snapshot therefore reports 0, the same as one holding none. +// +// blocks may arrive in any order: both answers are chosen by value, not by position, so the result +// does not rest on a sort the caller happens to do. Answering too high is the one damaging direction +// here — it would let the WAL be pruned past blocks a restore replays — so the floor is not left to a +// distant invariant. +func snapshotFloor(blocks []uint64, head uint64, rollbackWindow uint64) uint64 { + if head <= rollbackWindow { + return 0 // the rollback owed reaches past genesis, so no snapshot can be given up + } + target := head - rollbackWindow + + var oldest, newestPastWindow uint64 + foundReal := false + for _, block := range blocks { + if block == 0 { + continue // version 0 restores to no committed height, so it is never a floor + } + if !foundReal || block < oldest { + oldest = block + } + foundReal = true + if block <= target && block > newestPastWindow { + newestPastWindow = block + } + } + + if !foundReal { + return 0 // no real snapshot at all + } + if newestPastWindow > 0 { + return newestPastWindow + } + return oldest +} + +// snapshotHeights returns the block number of every snapshot on disk, ascending. A missing snapshot // directory yields no blocks rather than an error, matching traverseSnapshots: a store that has not // snapshotted yet has nothing to prune, which is not a failure. -func (s *CommitStore) snapshotBlocks() ([]uint64, error) { +func (s *CommitStore) snapshotHeights() ([]uint64, error) { var blocks []uint64 err := traverseSnapshots(s.flatkvDir(), true, func(version int64) (bool, error) { if version >= 0 { @@ -137,76 +230,58 @@ func (s *CommitStore) deleteSnapshot(block uint64) (bool, error) { return true, nil } -// GetRetentionWindow reports 0: this store asks for no history of its own beyond the collector's shared -// rollback window, which is the contract for a snapshot store (see gc.PrunableStore). +// GetRollbackFloor returns the oldest snapshot this store must keep to serve a rollback of +// rollbackWindow blocks behind its own head — normally the newest snapshot at or below that depth, +// since restoring anywhere above a snapshot replays the WAL forward from it. See snapshotFloor for +// every outcome. // -// 0 is what keeps it in the shared minimum, and being in that minimum is how its snapshots are -// protected — it answers its oldest needed snapshot as a boundary and the WAL is held there with it. -// InfiniteRetentionWindow would do the opposite of what it reads like: a store with no cut line is -// never asked for a boundary, so the WAL would be pruned to its own cut line and the snapshots left -// with nothing to replay from. +// The snapshot named here survives the prune that follows, because the collector's cut line is the +// minimum across stores and so is at or below this answer. // -// How deep this store can actually restore is a function of SnapshotInterval and SnapshotKeepRecent, -// not of anything declared here. Those two decide which snapshots exist; if they retain less than -// RollbackWindow of history, the window is not servable no matter what this returns. -func (s *CommitStore) GetRetentionWindow() int64 { - return 0 -} - -// GetPruningBoundary returns the newest snapshot at or below cutLine — the oldest point this store must -// keep to restore to cutLine, since restoring to anything above that snapshot replays the WAL forward -// from it. -// -// When every snapshot sits above cutLine it returns cutLine, per the contract: none of them can be -// dropped, so holding the other stores back to the oldest of them would buy nothing. Note what this -// means operationally — the store cannot in fact restore to cutLine in that case, only to its oldest -// snapshot. That is a snapshot-retention shortfall (SnapshotInterval × SnapshotKeepRecent shallower -// than RollbackWindow), and reporting it here as a boundary would not fix it, it would only stall -// every other store's pruning while the gap persisted. -// -// The initial empty snapshot at version 0 is not a boundary. It restores to no committed height, and -// reaching any height from it requires replaying the WAL from the very first block — which is exactly -// CannotServeRollback, both in meaning and in the value 0 would collide with. -// -// A failed directory scan is also reported as CannotServeRollback, abandoning the cycle. Not knowing -// which snapshots exist means not knowing which blocks are needed to replay from them, and the WAL is -// what would be pruned on the strength of a guess. -func (s *CommitStore) GetPruningBoundary(cutLine uint64) uint64 { - var newestAtOrBelow, newest int64 - err := traverseSnapshots(s.flatkvDir(), false, func(version int64) (bool, error) { - if version < 1 { - return false, nil - } - if newest == 0 { - newest = version // descending, so the first is the newest - } - if uint64(version) <= cutLine { //nolint:gosec // guarded >= 1 above - newestAtOrBelow = version - return true, nil - } - return false, nil - }) +// It measures against its own committed head rather than a height handed down by the collector, so a +// store running ahead of the fleet reports from where it actually is. The answer is bounded by the +// active snapshot, because a floor above where this store will resume from would let the WAL be pruned +// past the blocks that resume needs (see activeSnapshotHeight). +// +// 0 where there is no snapshot to name: nothing here is eligible for pruning, because a store that +// cannot restore anywhere cannot say which blocks the WAL may drop. Both "no snapshot on disk" and a +// window deeper than the head land there, as does a store that has committed nothing. The initial +// empty snapshot at version 0 does not count as one, because it restores to no committed height. +// +// A failed scan reports 0 for the same reason: not knowing which snapshots exist, or which one is +// active, means not knowing which blocks are needed to replay from them, and the WAL is what would be +// pruned on the strength of the guess. +func (s *CommitStore) GetRollbackFloor(rollbackWindow uint64) uint64 { + heights, err := s.snapshotHeights() if err != nil { - logger.Error("failed to scan snapshots for pruning boundary; blocking the prune cycle", - "cutLine", cutLine, "err", err) - return gc.CannotServeRollback + logger.Error("failed to scan snapshots for the rollback floor; holding it at 0", + "rollbackWindow", rollbackWindow, "err", err) + return 0 } - - switch { - case newestAtOrBelow > 0: - return uint64(newestAtOrBelow) //nolint:gosec // guarded > 0 - case newest > 0: - return cutLine - default: - return gc.CannotServeRollback + head, err := s.GetLatestBlock() + if err != nil { + logger.Error("failed to read the committed version for the rollback floor; holding it at 0", + "rollbackWindow", rollbackWindow, "err", err) + return 0 + } + floor := snapshotFloor(heights, head, rollbackWindow) + if floor == 0 { + return 0 // nothing is eligible, so there is no bound to apply + } + active, err := s.activeSnapshotHeight() + if err != nil { + logger.Error("failed to resolve the active snapshot for the rollback floor; holding it at 0", + "rollbackWindow", rollbackWindow, "err", err) + return 0 } + return min(floor, active) } // GetLatestBlock returns the highest committed version, or 0 when nothing has been committed. // // This is the committed version rather than the newest snapshot: it is the store's ingest position, -// which is what the collector takes a minimum over to find the fleet's head. The snapshot layout only -// enters through GetPruningBoundary. +// which is the head this store measures the rollback window against. The snapshot layout enters +// through snapshotFloor. // // Takes the read lock because Commit advances this field under the write lock, and the collector reads // it from its own goroutine. diff --git a/sei-db/state_db/sc/flatkv/store_gc_test.go b/sei-db/state_db/sc/flatkv/store_gc_test.go index b233d4eccb..902f931691 100644 --- a/sei-db/state_db/sc/flatkv/store_gc_test.go +++ b/sei-db/state_db/sc/flatkv/store_gc_test.go @@ -20,11 +20,30 @@ func gcStore(t *testing.T, dir string) (*CommitStore, gc.PrunableStore) { return s, s } +// gcStoreAtHead is gcStore with a committed head. The head is what the rollback window is resolved +// against, so a test that wants a particular depth sets one rather than passing a height in. +func gcStoreAtHead(t *testing.T, dir string, head int64) (*CommitStore, gc.PrunableStore) { + t.Helper() + s, store := gcStore(t, dir) + s.committedVersion = head + return s, store +} + +// mkSnapshots creates snapshot directories and points "current" at the newest one on disk, which is +// the shape production leaves behind: WriteSnapshot repoints the symlink at each snapshot it writes. +// The GC surface is bounded by the active snapshot, so a test that left "current" unset would be +// measuring a state a live store never reaches. The tests that want a stale or missing symlink say so +// after calling this. func mkSnapshots(t *testing.T, dir string, versions ...int64) { t.Helper() for _, v := range versions { require.NoError(t, os.MkdirAll(filepath.Join(dir, snapshotName(v)), 0750)) } + onDisk := snapshotVersions(t, dir) + if len(onDisk) == 0 { + return + } + require.NoError(t, updateCurrentSymlink(dir, snapshotName(onDisk[len(onDisk)-1]))) } func snapshotVersions(t *testing.T, dir string) []int64 { @@ -37,16 +56,20 @@ func snapshotVersions(t *testing.T, dir string) []int64 { return found } -// A snapshot store asks for no history of its own: 0 is what keeps it inside the collector's shared -// minimum, which is what protects its snapshots. See GetRetentionWindow for why the sentinel that -// reads like "keep more" would do the opposite here. -func TestGCRetentionWindowIsZero(t *testing.T) { - _, store := gcStore(t, t.TempDir()) - require.Equal(t, int64(0), store.GetRetentionWindow()) +// This store's history is the state WAL, which the collector manages as a store in its own right and +// prunes to the shared minimum. So PruneHistory here is a no-op, and must not reach the snapshots +// PruneSnapshots owns. +func TestGCPruneHistoryIsANoOp(t *testing.T) { + s, store := gcStore(t, t.TempDir()) + dir := s.flatkvDir() + mkSnapshots(t, dir, 5, 10, 20) + + require.NoError(t, store.PruneHistory(20)) + require.Equal(t, []int64{5, 10, 20}, snapshotVersions(t, dir)) } // The head is the committed version, not the newest snapshot: it is this store's ingest position, and -// the collector takes a minimum over those to find the fleet's head. +// what GetRollbackFloor measures the rollback window against. func TestGCLatestBlockIsCommittedVersion(t *testing.T) { s, store := gcStore(t, t.TempDir()) @@ -66,138 +89,259 @@ func TestGCLatestBlockIsCommittedVersion(t *testing.T) { require.Equal(t, uint64(42), latest) } -// Restoring to a height means starting at the newest snapshot at or below it and replaying the WAL -// forward, so that snapshot is the oldest thing this store must keep. -func TestGCPruningBoundaryIsNewestSnapshotAtOrBelowCutLine(t *testing.T) { - s, store := gcStore(t, t.TempDir()) +// snapshotFloor is fed a directory listing that happens to be sorted today, but it must not depend on +// that: choosing the floor by iteration position rather than by value would answer too high on an +// unsorted slice, and too high is the one direction that lets the WAL be pruned past blocks a restore +// replays. Every case runs its blocks in a scrambled order and expects the same answer as the sorted +// one would give. +func TestSnapshotFloorIsOrderIndependent(t *testing.T) { + const head = 100 + for _, tc := range []struct { + name string + blocks []uint64 + rollbackWindow uint64 + want uint64 + }{ + {name: "newest at or below the window", blocks: []uint64{20, 5, 10}, rollbackWindow: 85, want: 10}, + {name: "shortfall answers the oldest", blocks: []uint64{97, 90, 95}, rollbackWindow: 10, want: 90}, + {name: "version 0 is never the floor", blocks: []uint64{30, 0, 10}, rollbackWindow: 95, want: 10}, + {name: "only version 0 reads as none", blocks: []uint64{0}, rollbackWindow: 10, want: 0}, + {name: "window past genesis", blocks: []uint64{40, 10}, rollbackWindow: 100, want: 0}, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, snapshotFloor(tc.blocks, head, tc.rollbackWindow)) + }) + } +} + +// Restoring to a height inside the rollback window means starting at the newest snapshot at or below +// that depth and replaying the WAL forward, so that snapshot is the oldest thing this store must keep. +// The depth comes from its own head, which is why every case here fixes a head of 100. +func TestGCRollbackFloorIsNewestSnapshotPastTheWindow(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 100) mkSnapshots(t, s.flatkvDir(), 5, 10, 20) for _, tc := range []struct { - cutLine uint64 - want uint64 - why string + rollbackWindow uint64 + want uint64 + why string }{ - {cutLine: 15, want: 10, why: "newest at or below"}, - {cutLine: 20, want: 20, why: "a snapshot exactly on the cut line serves it"}, - {cutLine: 100, want: 20, why: "far above every snapshot"}, - {cutLine: 5, want: 5, why: "the oldest snapshot exactly on the cut line"}, + {rollbackWindow: 85, want: 10, why: "newest at or below head - window"}, + {rollbackWindow: 80, want: 20, why: "a snapshot exactly at that depth serves it"}, + {rollbackWindow: 0, want: 20, why: "a window of 0 reaches the newest snapshot"}, + {rollbackWindow: 95, want: 5, why: "the oldest snapshot exactly at that depth"}, } { - require.Equal(t, tc.want, store.GetPruningBoundary(tc.cutLine), tc.why) + require.Equal(t, tc.want, store.GetRollbackFloor(tc.rollbackWindow), tc.why) } } -// The answer must never exceed the cut line: the collector takes a minimum across stores without -// clamping, so a higher answer here would raise pruneHeight above head-RollbackWindow and prune away -// the rollback window every other store is holding. -func TestGCPruningBoundaryNeverExceedsCutLine(t *testing.T) { - s, store := gcStore(t, t.TempDir()) +// Whatever the window, the answer is either a snapshot that exists or 0. That is what lets the +// collector hold history at a floor this store can restore from, and it is the property a store must +// never break — a floor naming a height with no snapshot on it would be a restore point the store +// cannot serve. +func TestGCRollbackFloorIsAlwaysAnExistingSnapshot(t *testing.T) { + const head = 40 + s, store := gcStoreAtHead(t, t.TempDir(), head) mkSnapshots(t, s.flatkvDir(), 7, 13, 29) - for cutLine := uint64(1); cutLine <= 40; cutLine++ { - require.LessOrEqual(t, store.GetPruningBoundary(cutLine), cutLine, - "boundary above cut line at cutLine=%d", cutLine) + for rollbackWindow := uint64(0); rollbackWindow <= 60; rollbackWindow++ { + if rollbackWindow >= head { + require.Equal(t, uint64(0), store.GetRollbackFloor(rollbackWindow), + "a window deeper than the head leaves nothing eligible at rollbackWindow=%d", rollbackWindow) + continue + } + require.Contains(t, []uint64{7, 13, 29}, store.GetRollbackFloor(rollbackWindow), + "floor is not a snapshot at rollbackWindow=%d", rollbackWindow) } } -// Every snapshot above the cut line means none can be dropped, so the store answers the cut line -// rather than its oldest snapshot — holding the fleet back to that snapshot would buy nothing. The -// store genuinely cannot restore to the cut line here; that is a snapshot-retention shortfall, not -// something a lower answer would repair. -func TestGCPruningBoundaryAllSnapshotsAboveCutLine(t *testing.T) { - s, store := gcStore(t, t.TempDir()) +// Every snapshot above the window means none can be dropped, and the store reports its oldest — the +// deepest it can actually restore to. That is a snapshot-retention shortfall (the snapshot depth is +// shallower than RollbackWindow), and naming that snapshot is what keeps it replayable rather than +// letting history be pruned out from under it. +func TestGCRollbackFloorAllSnapshotsInsideTheWindow(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 1_000) mkSnapshots(t, s.flatkvDir(), 500, 900) - require.Equal(t, uint64(100), store.GetPruningBoundary(100)) + require.Equal(t, uint64(500), store.GetRollbackFloor(900), "head - window is 100, below both snapshots") + require.Equal(t, uint64(0), store.GetRollbackFloor(2_000), + "a window deeper than the head leaves nothing eligible for pruning") } -// With no snapshot to restore from, the store stops the cycle rather than dropping out of it: it will -// replay forward once its first snapshot lands, and the WAL holds the range it will replay from. -func TestGCPruningBoundaryWithoutSnapshots(t *testing.T) { - s, store := gcStore(t, t.TempDir()) - require.Equal(t, gc.CannotServeRollback, store.GetPruningBoundary(100)) +// With no snapshot there is none to name, so nothing here is eligible for pruning: a store that +// cannot restore anywhere cannot say which blocks the WAL may drop. +func TestGCRollbackFloorWithoutSnapshots(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 100) + require.Equal(t, uint64(0), store.GetRollbackFloor(10)) // A directory that does not exist yet reads the same way. require.NoError(t, os.RemoveAll(s.flatkvDir())) - require.Equal(t, gc.CannotServeRollback, store.GetPruningBoundary(100)) + require.Equal(t, uint64(0), store.GetRollbackFloor(10)) + + // So does a store that has committed nothing. + _, fresh := gcStore(t, t.TempDir()) + require.Equal(t, uint64(0), fresh.GetRollbackFloor(10)) } -// The initial empty snapshot restores to no committed height, so reaching any height from it needs the -// WAL from its very first block — which is what CannotServeRollback already means, and is the value 0 -// would collide with anyway. -func TestGCPruningBoundaryIgnoresInitialSnapshot(t *testing.T) { - s, store := gcStore(t, t.TempDir()) +// The initial empty snapshot restores to no committed height, so it is not a restore point and cannot +// be a floor. A store holding only that one is read as holding none. +func TestGCRollbackFloorIgnoresInitialSnapshot(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 100) mkSnapshots(t, s.flatkvDir(), 0) - require.Equal(t, gc.CannotServeRollback, store.GetPruningBoundary(100)) + require.Equal(t, uint64(0), store.GetRollbackFloor(10), "version 0 does not count as a snapshot") - // With a real snapshot present, version 0 still contributes nothing. + // With a real snapshot present, version 0 still contributes nothing — not even as the oldest. mkSnapshots(t, s.flatkvDir(), 30) - require.Equal(t, uint64(30), store.GetPruningBoundary(100)) - require.Equal(t, uint64(10), store.GetPruningBoundary(10), "not 0, and not the initial snapshot") + require.Equal(t, uint64(30), store.GetRollbackFloor(0)) + require.Equal(t, uint64(30), store.GetRollbackFloor(90), + "head - window is 10; the oldest real snapshot answers, not version 0") } // Not knowing which snapshots exist means not knowing which blocks are needed to replay from them, and -// the WAL would be pruned on the strength of that answer. So a failed scan blocks the cycle. -func TestGCPruningBoundaryScanFailureBlocksTheCycle(t *testing.T) { +// the WAL would be pruned on the strength of that answer. So a failed scan reports 0, holding the +// fleet's history where it is. +func TestGCRollbackFloorScanFailureHoldsHistory(t *testing.T) { dir := filepath.Join(t.TempDir(), "flatkv") require.NoError(t, os.WriteFile(dir, []byte("not a directory"), 0600)) - _, store := gcStore(t, dir) - require.Equal(t, gc.CannotServeRollback, store.GetPruningBoundary(100)) + _, store := gcStoreAtHead(t, dir, 100) + require.Equal(t, uint64(0), store.GetRollbackFloor(10)) } -// PruneBelow drops every snapshot under the floor. The floor is a minimum across stores and so never -// exceeds the boundary this store reported, which is why the snapshot it named always survives. -func TestGCPruneBelowDeletesSnapshotsBelowFloor(t *testing.T) { - s, store := gcStore(t, t.TempDir()) +// PruneSnapshots drops everything strictly below the height it is given, and the height need not +// have a snapshot on it. +func TestGCPruneSnapshotsDeletesBelowTheCutLine(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 30) dir := s.flatkvDir() mkSnapshots(t, dir, 0, 5, 10, 20, 30) require.NoError(t, updateCurrentSymlink(dir, snapshotName(30))) - require.NoError(t, store.PruneBelow(20)) - require.Equal(t, []int64{20, 30}, snapshotVersions(t, dir)) + // A cut line of 25 has no snapshot on it; 20 is the newest below it and goes with the rest. + require.NoError(t, store.PruneSnapshots(25)) + require.Equal(t, []int64{30}, snapshotVersions(t, dir)) - // Idempotent: the same floor twice deletes nothing more. - require.NoError(t, store.PruneBelow(20)) - require.Equal(t, []int64{20, 30}, snapshotVersions(t, dir)) + // Idempotent: the same cut line twice deletes nothing more. + require.NoError(t, store.PruneSnapshots(25)) + require.Equal(t, []int64{30}, snapshotVersions(t, dir)) } -// A floor of 0 is the collector's "nothing to do" and must not be read as "delete everything below -// any height". -func TestGCPruneBelowZeroIsNoOp(t *testing.T) { - s, store := gcStore(t, t.TempDir()) +// The snapshot the store reported must survive the cycle that follows, which is what makes the +// collector's minimum meaningful: history is held at a floor the store still has the snapshot for. +// The collector's cut line is the minimum across stores and so is at or below this store's own +// answer, which the loop walks the tightest case of. +func TestGCPruneSnapshotsKeepsTheSnapshotItReported(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 40) + dir := s.flatkvDir() + mkSnapshots(t, dir, 5, 10, 20, 30) + require.NoError(t, updateCurrentSymlink(dir, snapshotName(30))) + + for rollbackWindow := uint64(0); rollbackWindow <= 40; rollbackWindow++ { + floor := store.GetRollbackFloor(rollbackWindow) + before := snapshotVersions(t, dir) + require.NoError(t, store.PruneSnapshots(floor)) + remaining := snapshotVersions(t, dir) + + if floor == 0 { + require.Equal(t, before, remaining, + "nothing is eligible at rollbackWindow=%d, so nothing may be deleted", rollbackWindow) + continue + } + require.NotEmpty(t, remaining, "a store that had a snapshot must still have one") + require.Contains(t, remaining, int64(floor), //nolint:gosec // bounded by the loop + "the reported floor must survive at rollbackWindow=%d", rollbackWindow) + } +} + +// A cut line at or below the oldest snapshot leaves the lot. It must not be read as "delete +// everything below the newest". +func TestGCPruneSnapshotsBelowTheOldestSnapshotIsNoOp(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 1_000) + dir := s.flatkvDir() + mkSnapshots(t, dir, 500, 900) + require.NoError(t, updateCurrentSymlink(dir, snapshotName(900))) + + require.NoError(t, store.PruneSnapshots(500)) + require.Equal(t, []int64{500, 900}, snapshotVersions(t, dir)) +} + +// The collector never passes 0 — it means nothing is eligible — but the store absorbs it as a no-op +// rather than reading it as an empty range to delete against. +func TestGCPruneSnapshotsAtZeroIsNoOp(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 10) dir := s.flatkvDir() mkSnapshots(t, dir, 5, 10) require.NoError(t, updateCurrentSymlink(dir, snapshotName(10))) - require.NoError(t, store.PruneBelow(0)) + require.NoError(t, store.PruneSnapshots(0)) require.Equal(t, []int64{5, 10}, snapshotVersions(t, dir)) } -// The active snapshot is what the next open resolves to, so it survives however deep the request. The -// contract should never produce such a request; this pins that a wrong answer elsewhere cannot leave -// the store unbootable here. -func TestGCPruneBelowKeepsActiveSnapshot(t *testing.T) { - s, store := gcStore(t, t.TempDir()) +// A store with no snapshots has nothing to prune, which is not a failure — and notably not an error +// about the missing active symlink, since there is no deletion to protect. +func TestGCPruneSnapshotsWithoutSnapshots(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 100) + require.NoError(t, store.PruneSnapshots(10)) + + require.NoError(t, os.RemoveAll(s.flatkvDir())) + require.NoError(t, store.PruneSnapshots(10)) +} + +// The active snapshot is what the next open resolves to, so the cut line stops there even when it is +// asked to go deeper. This is the shape a crash between WriteSnapshot's rename and its symlink update +// leaves behind: snapshot 10 is on disk while "current" is still 5. Deleting 5 would leave a dangling +// symlink, which os.Readlink resolves happily, so the store would open against a directory that is +// not there. +func TestGCPruneSnapshotsKeepsActiveSnapshotBelowTheCutLine(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 10) dir := s.flatkvDir() - mkSnapshots(t, dir, 5, 10) + mkSnapshots(t, dir, 3, 5, 10) require.NoError(t, updateCurrentSymlink(dir, snapshotName(5))) - require.NoError(t, store.PruneBelow(1_000)) - require.Equal(t, []int64{5}, snapshotVersions(t, dir)) + require.NoError(t, store.PruneSnapshots(10)) + require.Equal(t, []int64{5, 10}, snapshotVersions(t, dir), + "the cut line stops at the active snapshot, so 3 goes and 5 stays") } -// Without a resolvable active snapshot there is nothing to protect the deletion against, so the prune -// is refused rather than run blind. -func TestGCPruneBelowRefusesWithoutActiveSnapshot(t *testing.T) { - s, store := gcStore(t, t.TempDir()) +// Without a resolvable active snapshot there is nothing to bound the deletion by, so the prune is +// refused rather than run blind. +func TestGCPruneSnapshotsRefusesWithoutActiveSnapshot(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 10) dir := s.flatkvDir() mkSnapshots(t, dir, 5, 10) + require.NoError(t, os.Remove(currentPath(dir))) - require.Error(t, store.PruneBelow(10)) + require.Error(t, store.PruneSnapshots(10)) require.Equal(t, []int64{5, 10}, snapshotVersions(t, dir)) } +// The same refusal on the reporting side: a floor above the snapshot this store would resume from +// would let the WAL be pruned past the blocks that resume replays, so an unresolvable "current" holds +// the fleet's history where it is. +func TestGCRollbackFloorHoldsHistoryWithoutActiveSnapshot(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 100) + dir := s.flatkvDir() + mkSnapshots(t, dir, 10, 20) + require.Equal(t, uint64(20), store.GetRollbackFloor(80), "the newest snapshot past the window") + + require.NoError(t, os.Remove(currentPath(dir))) + require.Equal(t, uint64(0), store.GetRollbackFloor(80)) +} + +// A snapshot newer than "current" is what a crash between WriteSnapshot's rename and its symlink +// update leaves behind, and the next open takes the symlink rather than adopting the orphan. So the +// floor stops at the active snapshot: reporting the orphan would hold the WAL only from there, and +// the replay that starts at the active snapshot needs the blocks below it. +func TestGCRollbackFloorStopsAtTheActiveSnapshot(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 100) + dir := s.flatkvDir() + mkSnapshots(t, dir, 10, 50) + require.NoError(t, updateCurrentSymlink(dir, snapshotName(10))) + + require.Equal(t, uint64(10), store.GetRollbackFloor(20), + "head - window is 80, so the orphan at 50 would answer if the active snapshot did not bound it") +} + // ExternalPruning is off unless asked for: a store built without a collector must keep pruning // itself, since standing down with nothing to replace it grows snapshots without bound. func TestGCExternalPruningDefaultsOff(t *testing.T) { @@ -224,7 +368,7 @@ func TestGCExternalPruningStandsDownSnapshotPruner(t *testing.T) { }, } mkSnapshots(t, dir, 5, 10, 15) - s.pruneSnapshots(dir, 15) + s.pruneSnapshotsByCount(dir, 15) return snapshotVersions(t, dir) } @@ -263,10 +407,10 @@ func TestGCPrunesRealSnapshotsAndStoreStillOpens(t *testing.T) { require.NoError(t, err) require.Equal(t, uint64(6), latest) - boundary := store.GetPruningBoundary(5) - require.Equal(t, uint64(4), boundary, "newest snapshot at or below the cut line") + floor := store.GetRollbackFloor(1) + require.Equal(t, uint64(4), floor, "newest snapshot at or below head - 1") - require.NoError(t, store.PruneBelow(boundary)) + require.NoError(t, store.PruneSnapshots(floor)) require.Equal(t, []int64{4, 6}, snapshotVersions(t, cfg.DataDir)) require.NoError(t, s.Close()) @@ -314,14 +458,12 @@ func TestGCConcurrentWithCommitter(t *testing.T) { head, err := store.GetLatestBlock() require.NoError(t, err) require.LessOrEqual(t, head, uint64(blocks)) - if head <= 10 { - continue - } - boundary := store.GetPruningBoundary(head - 10) - require.LessOrEqual(t, boundary, head-10, "boundary must never exceed the cut line") - if boundary != gc.CannotServeRollback { - require.NoError(t, store.PruneBelow(boundary)) - } + // No assertion on the floor itself: the committer advances the head between these calls, + // so any relation to the head read above is racy by construction. What is under test is + // that reading and pruning concurrently with a committer is safe at all. + floor := store.GetRollbackFloor(10) + require.NoError(t, store.PruneSnapshots(floor)) + require.NoError(t, store.PruneHistory(floor)) } <-done } diff --git a/sei-db/state_db/statewal/state_wal_gc.go b/sei-db/state_db/statewal/state_wal_gc.go index d3097dae7b..9ebcd1a45c 100644 --- a/sei-db/state_db/statewal/state_wal_gc.go +++ b/sei-db/state_db/statewal/state_wal_gc.go @@ -14,9 +14,9 @@ import ( // concurrent PruneBefore by contract. None of them touch the plain fields the writer owns. // // Precondition beyond the collector's own: SC and SS must be managed alongside this WAL. The WAL is -// what they replay from, and it is their boundaries — the oldest snapshot each still needs — that -// hold it back, since the collector prunes every store to the shared minimum. Managed without them, -// the WAL is pruned to its own cut line and the replay range they depend on goes with it. +// what they replay from, and it is their floors — the oldest snapshot each still needs — that hold it +// back, since the collector prunes every store to the shared minimum. Managed without them, the WAL +// is pruned to its own floor and the replay range they depend on goes with it. var _ gc.PrunableStore = (*stateWALImpl)(nil) func (w *stateWALImpl) Name() string { @@ -34,58 +34,66 @@ func (w *stateWALImpl) Name() string { // stopped, the collector declining, and the WAL growing without bound. // // Where FlatKV has not handed over, both prune, and the shared minimum makes that safe rather than -// merely redundant: a store is asked for its boundary whatever it answers here, so FlatKV holds the +// merely redundant: a store is asked for its floor whatever it answers here, so FlatKV holds the // collector back to the oldest snapshot it still replays from. That relies on FlatKV being // registered at all, which is the precondition on the type above. func (w *stateWALImpl) ExternalPruning() bool { return true } -// PruneBelow schedules removal of the blocks below blockNumber, going straight to the underlying WAL -// from the collector's goroutine. Prune is the equivalent for the WAL's own owner; this exists +// PruneHistory schedules removal of the blocks below blockNumber, going straight to the underlying +// WAL from the collector's goroutine. Prune is the equivalent for the WAL's own owner; this exists // separately only because it must not read the closed/fatalErr bookkeeping that Prune does. // +// blockNumber is the shared minimum rather than this WAL's own boundary, which for this store is the +// whole point: SC and SS answer their oldest live snapshot, and that minimum is what holds the WAL +// back far enough for them to replay forward from it. +// // Deliberately does not brick the WAL on failure, unlike every writer-goroutine path here. Bricking // means writing fatalErr, which the writer reads unsynchronized, so doing it from this goroutine would // be a data race. Nothing is lost by declining: a prune fails only when the WAL is already closed or // dead underneath, and the writer's next operation discovers that on its own. -func (w *stateWALImpl) PruneBelow(blockNumber uint64) error { +func (w *stateWALImpl) PruneHistory(blockNumber uint64) error { if err := w.wal.PruneBefore(blockNumber); err != nil { return fmt.Errorf("failed to prune state WAL below block %d: %w", blockNumber, err) } return nil } -// GetRetentionWindow reports 0, and this is a property of what the WAL is rather than a default -// something might want to raise. How deep this WAL must go is not its own to declare: it is a replay -// source, and the depth it needs is whatever its consumers need it to be. SC and SS express that by -// answering their oldest live snapshot as a pruning boundary, and the collector prunes every store -// to the shared minimum, so the WAL is already held exactly as far back as they can replay from. -// -// A window here would be additive on top of that, and — because the minimum is shared — it would -// retain every managed store that much further back rather than this WAL alone. That is a fleet-wide -// retention decision wearing a per-store name, which is what RollbackWindow already is. -func (w *stateWALImpl) GetRetentionWindow() int64 { - return 0 +// PruneSnapshots does nothing: the WAL keeps no snapshots. It is what snapshots are replayed +// forward from, which reaches it as the shared minimum PruneHistory receives. +func (w *stateWALImpl) PruneSnapshots(uint64) error { + return nil } -// GetPruningBoundary returns cutLine, the contract's answer for a contiguous store: the WAL holds -// every block from its floor to its head, so cutLine itself is replayable and nothing below it has -// to be held back. +// GetRollbackFloor returns head - rollbackWindow, the contract's answer for a contiguous store: +// the WAL holds every block from its floor to its head, so that height is replayable directly and +// nothing below it has to be held back. It keeps no snapshots — it is what snapshots replay from — so +// there is nothing to resolve the window against beyond its own head. // -// Unconditional on purpose. A WAL whose floor already sits above cutLine — pruned there by an -// earlier cycle, or freshly created above it — also answers cutLine, because the PruneBelow that -// follows is a no-op on it while a lower answer would hold every other store back to this WAL's -// floor. CannotServeRollback is never right here: it is the replay source, not a replay consumer. -func (w *stateWALImpl) GetPruningBoundary(cutLine uint64) uint64 { - return cutLine +// It reports against its own head even when that runs ahead of the fleet's, because the collector +// takes a minimum across stores. This WAL's own answer is rarely the binding one: SC and SS report +// their oldest live snapshot, which sits below this and is what actually holds the WAL back far +// enough for them to replay forward. +// +// 0 when the window is deeper than the whole WAL — including a freshly created one, whose head is 0. +// Nothing here is eligible for pruning until the head clears the window. +func (w *stateWALImpl) GetRollbackFloor(rollbackWindow uint64) uint64 { + head, err := w.GetLatestBlock() + if err != nil { + return 0 // cannot say what it holds, so nothing may be dropped anywhere + } + if head <= rollbackWindow { + return 0 + } + return head - rollbackWindow } // GetLatestBlock returns the highest block ended by SignalEndOfBlock, or 0 when none has been. // // A block that has been written but not yet ended is deliberately excluded: it is still buffered, -// not a record, and reporting it would put the collector's head one block above what the WAL can -// actually replay. See stateWALImpl.lastCompletedBlock for the 0 case. +// not a record, and reporting it would put the floor this store reports one block above what the WAL +// can actually replay. See stateWALImpl.lastCompletedBlock for the 0 case. func (w *stateWALImpl) GetLatestBlock() (uint64, error) { return w.lastCompletedBlock.Load(), nil } diff --git a/sei-db/state_db/statewal/state_wal_gc_test.go b/sei-db/state_db/statewal/state_wal_gc_test.go index bccedc034d..4e4acad450 100644 --- a/sei-db/state_db/statewal/state_wal_gc_test.go +++ b/sei-db/state_db/statewal/state_wal_gc_test.go @@ -20,8 +20,8 @@ func openWALForGC(t *testing.T, cfg *Config) (StateWAL, gc.PrunableStore) { } // The head is the last block ended by SignalEndOfBlock. A block that has been written but not ended -// is still buffered rather than a record, so counting it would put the collector's head one block -// above what the WAL can replay. +// is still buffered rather than a record, so counting it would put this store's head — and with it +// the floor it reports — one block above what the WAL can replay. func TestGCLatestBlockCountsOnlyCompletedBlocks(t *testing.T) { w, store := openWALForGC(t, testConfig(t.TempDir())) @@ -47,8 +47,8 @@ func TestGCLatestBlockCountsOnlyCompletedBlocks(t *testing.T) { require.Equal(t, uint64(3), latest) } -// The head survives a reopen: a WAL that reported 0 after a restart would drop out of the -// collector's head minimum and let the other stores prune past blocks it still holds. +// The head survives a reopen: a WAL that reported 0 after a restart would report a floor of 0 with +// it, holding the whole fleet's history where it is until the head is rebuilt. func TestGCLatestBlockRecoveredOnOpen(t *testing.T) { cfg := testConfig(t.TempDir()) @@ -65,33 +65,51 @@ func TestGCLatestBlockRecoveredOnOpen(t *testing.T) { require.Equal(t, uint64(5), latest) } -// The WAL asks for no history of its own, and there is no configuration that changes that. What -// holds it back is SC/SS answering their oldest live snapshot as a boundary, which the collector -// applies to every store as the shared minimum — so its depth tracks its consumers automatically -// (see GetRetentionWindow). -func TestGCRetentionWindowIsZero(t *testing.T) { - _, store := openWALForGC(t, testConfig(t.TempDir())) - require.Equal(t, int64(0), store.GetRetentionWindow()) +// The WAL keeps no snapshots — it is what snapshots replay forward from — so its half of the split +// contract is a no-op that must not touch the stored range. Its depth is instead set by SC/SS +// answering their oldest live snapshot as a rollback floor, which reaches the WAL as the shared +// minimum PruneHistory receives. +func TestGCPruneSnapshotsIsANoOp(t *testing.T) { + cfg := testConfig(t.TempDir()) + cfg.TargetFileSize = 1 + w, store := openWALForGC(t, cfg) + + for block := uint64(1); block <= 10; block++ { + writeBlock(t, w, block) + } + require.NoError(t, w.Flush()) + + require.NoError(t, store.PruneSnapshots(8)) + require.NoError(t, w.Flush()) + + ok, first, last, err := w.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), first, "a snapshot prune must not move the WAL floor") + require.Equal(t, uint64(10), last) } -// A contiguous store answers the cut line it was given whatever it holds, including on an empty WAL -// and above its own head. -func TestGCPruningBoundaryIsAlwaysCutLine(t *testing.T) { +// A contiguous store resolves the window against its own head, and reports 0 where the window is +// deeper than that head — including on an empty WAL, where the answer holds the fleet's history +// where it is. +func TestGCRollbackFloorComesFromOwnHead(t *testing.T) { w, store := openWALForGC(t, testConfig(t.TempDir())) - require.Equal(t, uint64(42), store.GetPruningBoundary(42)) + require.Equal(t, uint64(0), store.GetRollbackFloor(42), "an empty WAL needs every block kept") for block := uint64(1); block <= 5; block++ { writeBlock(t, w, block) } - require.Equal(t, uint64(3), store.GetPruningBoundary(3)) - require.Equal(t, uint64(1_000), store.GetPruningBoundary(1_000)) + require.Equal(t, uint64(5), store.GetRollbackFloor(0)) + require.Equal(t, uint64(2), store.GetRollbackFloor(3)) + require.Equal(t, uint64(0), store.GetRollbackFloor(1_000), + "a window deeper than the head leaves nothing eligible for pruning") } -// PruneBelow goes straight to the WAL, so reclamation does not wait on write traffic. A WAL that has +// PruneHistory goes straight to the WAL, so reclamation does not wait on write traffic. A WAL that has // stopped receiving blocks is exactly where a deferred prune would strand history indefinitely, so no // further block is written here before the result is checked. -func TestGCPruneBelowDoesNotWaitForTheNextBlock(t *testing.T) { +func TestGCPruneHistoryDoesNotWaitForTheNextBlock(t *testing.T) { cfg := testConfig(t.TempDir()) cfg.TargetFileSize = 1 // seal after every block, so whole-file pruning can act per block w, store := openWALForGC(t, cfg) @@ -101,7 +119,7 @@ func TestGCPruneBelowDoesNotWaitForTheNextBlock(t *testing.T) { } require.NoError(t, w.Flush()) - require.NoError(t, store.PruneBelow(6)) + require.NoError(t, store.PruneHistory(6)) require.NoError(t, w.Flush()) // the prune is async; order behind it ok, first, last, err := w.GetStoredRange() @@ -117,7 +135,7 @@ func TestGCPruneBelowDoesNotWaitForTheNextBlock(t *testing.T) { // performed. Nothing here enforces that — a prune only ever deletes — but the collector is free to // issue a lower floor after a rollback, and this pins that doing so is harmless rather than a way to // resurrect blocks or corrupt the range. -func TestGCPruneBelowIgnoresALowerFloor(t *testing.T) { +func TestGCPruneHistoryIgnoresALowerFloor(t *testing.T) { cfg := testConfig(t.TempDir()) cfg.TargetFileSize = 1 w, store := openWALForGC(t, cfg) @@ -127,8 +145,8 @@ func TestGCPruneBelowIgnoresALowerFloor(t *testing.T) { } require.NoError(t, w.Flush()) - require.NoError(t, store.PruneBelow(8)) - require.NoError(t, store.PruneBelow(3)) + require.NoError(t, store.PruneHistory(8)) + require.NoError(t, store.PruneHistory(3)) require.NoError(t, w.Flush()) ok, first, _, err := w.GetStoredRange() @@ -168,7 +186,7 @@ func TestGCConcurrentWithWriter(t *testing.T) { require.NoError(t, err) require.LessOrEqual(t, head, uint64(blocks)) if head > 20 { - require.NoError(t, store.PruneBelow(store.GetPruningBoundary(head-20))) + require.NoError(t, store.PruneHistory(store.GetRollbackFloor(20))) } } <-done @@ -182,7 +200,7 @@ func TestGCConcurrentWithWriter(t *testing.T) { // A prune issued before a close is ordered ahead of it rather than dropped, and survives into the next // session. What must not happen is the close failing, or the WAL bricking, because of it. -func TestGCPruneBelowBeforeClose(t *testing.T) { +func TestGCPruneHistoryBeforeClose(t *testing.T) { cfg := testConfig(t.TempDir()) cfg.TargetFileSize = 1 @@ -190,7 +208,7 @@ func TestGCPruneBelowBeforeClose(t *testing.T) { for block := uint64(1); block <= 5; block++ { writeBlock(t, w, block) } - require.NoError(t, w.(gc.PrunableStore).PruneBelow(4)) + require.NoError(t, w.(gc.PrunableStore).PruneHistory(4)) require.NoError(t, w.Close()) w2, store := openWALForGC(t, cfg) @@ -206,9 +224,9 @@ func TestGCPruneBelowBeforeClose(t *testing.T) { } // The collector can still be mid-cycle when the node shuts down, so a prune arriving after the close -// must be reported rather than panic, and must not brick the WAL on the way out — PruneBelow declines +// must be reported rather than panic, and must not brick the WAL on the way out — PruneHistory declines // to write fatalErr precisely because the writer reads it unsynchronized. -func TestGCPruneBelowAfterClose(t *testing.T) { +func TestGCPruneHistoryAfterClose(t *testing.T) { cfg := testConfig(t.TempDir()) cfg.TargetFileSize = 1 @@ -220,7 +238,7 @@ func TestGCPruneBelowAfterClose(t *testing.T) { require.True(t, ok) require.NoError(t, w.Close()) - require.Error(t, store.PruneBelow(4)) + require.Error(t, store.PruneHistory(4)) // The head is still readable, and the close committed what was written. latest, err := store.GetLatestBlock() diff --git a/sei-db/state_db/statewal/state_wal_impl.go b/sei-db/state_db/statewal/state_wal_impl.go index d2aedecd12..e2b3a3c868 100644 --- a/sei-db/state_db/statewal/state_wal_impl.go +++ b/sei-db/state_db/statewal/state_wal_impl.go @@ -51,7 +51,8 @@ type stateWALImpl struct { // // 0 doubles as "no block completed yet", which is what GetLatestBlock reports. A WAL whose only // completed block is block 0 is indistinguishable from an empty one, and that is the safe direction: - // it drops out of the collector's head rather than pulling every store's cut line down to 0. + // GetRollbackFloor then answers 0, which holds the whole fleet's history where it is rather than + // letting anything be dropped against a head this WAL cannot vouch for. lastCompletedBlock atomic.Uint64 } From 9c0e746a0072417a4d4680d9db9dd9b4d35795a1 Mon Sep 17 00:00:00 2001 From: YimingZang Date: Mon, 10 Aug 2026 21:18:55 -0700 Subject: [PATCH 10/11] Add unit test and infinite retention --- .../gc/storage_garbage_collector.go | 19 +++-- .../gc/storage_garbage_collector_config.go | 15 +++- .../gc/storage_garbage_collector_test.go | 83 +++++++++++++++++-- 3 files changed, 102 insertions(+), 15 deletions(-) diff --git a/sei-db/management/gc/storage_garbage_collector.go b/sei-db/management/gc/storage_garbage_collector.go index 290ff54d5e..34866d27ed 100644 --- a/sei-db/management/gc/storage_garbage_collector.go +++ b/sei-db/management/gc/storage_garbage_collector.go @@ -22,7 +22,8 @@ var logger = seilog.NewLogger("db", "gc") // asked to roll back to, which it resolves against its own head // 2. snapshotCutLine = min of those answers, the deepest rollback the fleet still owes // 3. historyCutLine = snapshotCutLine - LookbackWindow, so the lookback window sits entirely below -// the deepest promised rollback point rather than overlapping it +// the deepest promised rollback point rather than overlapping it; a LookbackWindow of -1 makes +// this 0, which is infinite history retention (snapshots are still reclaimed to the floor) // 4. on every store reporting ExternalPruning, PruneSnapshots(snapshotCutLine) then // PruneHistory(historyCutLine) // @@ -193,11 +194,17 @@ func describeDecisions(stores []PrunableStore, decisions []storeDecision) string // Subtracting from the minimum of the stores' answers is what makes the result safe without clamping // any of them: it can only sit at or below every store's own floor. // -// 0 when the window reaches below genesis, and it needs no special handling — it is the literal -// height "keep everything from block 0 up", which every store's PruneHistory absorbs as a no-op. -func getHistoryCutLine(snapshotCutLine uint64, lookbackWindow uint64) uint64 { - if snapshotCutLine <= lookbackWindow { +// A lookbackWindow of -1 is infinite retention: it returns 0, the literal height "keep everything +// from block 0 up", so history is never pruned however far the snapshot floor advances. The same 0 +// falls out when a finite window reaches below genesis, and it needs no special handling on the +// other end either — every store's PruneHistory absorbs a height of 0 as a no-op. +func getHistoryCutLine(snapshotCutLine uint64, lookbackWindow int64) uint64 { + if lookbackWindow < 0 { + return 0 // -1: infinite retention, nothing below the snapshot floor is ever given up + } + window := uint64(lookbackWindow) + if snapshotCutLine <= window { return 0 } - return snapshotCutLine - lookbackWindow + return snapshotCutLine - window } diff --git a/sei-db/management/gc/storage_garbage_collector_config.go b/sei-db/management/gc/storage_garbage_collector_config.go index f57eebb6ea..def285657e 100644 --- a/sei-db/management/gc/storage_garbage_collector_config.go +++ b/sei-db/management/gc/storage_garbage_collector_config.go @@ -32,7 +32,13 @@ type StorageGarbageCollectorConfig struct { // since they are restore points and this window buys history to read rather than history to // restore to. History goes LookbackWindow below that same floor, because a retained snapshot // is only restorable if the blocks above it survive. See PrunableStore. - LookbackWindow uint64 + // + // -1 means infinite: history is never pruned, so every block ever ingested stays readable. + // Snapshots below the rollback floor are still reclaimed — they are restore points nothing can + // ask for once the floor has passed them, and holding history forever does not make them one. + // The type is signed only to carry this sentinel; every other value is a block count and must + // be >= 0. + LookbackWindow int64 // PruneInterval is how often the collector runs a prune cycle. Must be > 0. PruneInterval time.Duration @@ -54,6 +60,10 @@ func DefaultStorageGarbageCollectorConfig() *StorageGarbageCollectorConfig { // // The windows are additive, so every combination of them is meaningful and neither is constrained // against the other. A sum that overflows uint64 is handled in getHistoryCutLine. +// +// LookbackWindow is a block count and so is bounded below by 0, except for -1, the infinite-retention +// sentinel. Any other negative is a typo — most likely -1 miswritten — and is rejected rather than +// silently read as a huge count once converted. func (c *StorageGarbageCollectorConfig) Validate() error { if c == nil { return fmt.Errorf("config is required") @@ -61,5 +71,8 @@ func (c *StorageGarbageCollectorConfig) Validate() error { if c.PruneInterval <= 0 { return fmt.Errorf("prune interval must be greater than 0") } + if c.LookbackWindow < -1 { + return fmt.Errorf("lookback window must be >= 0, or -1 for infinite retention (got %d)", c.LookbackWindow) + } return nil } diff --git a/sei-db/management/gc/storage_garbage_collector_test.go b/sei-db/management/gc/storage_garbage_collector_test.go index 9f5af6e3b8..6c3aea26ea 100644 --- a/sei-db/management/gc/storage_garbage_collector_test.go +++ b/sei-db/management/gc/storage_garbage_collector_test.go @@ -119,7 +119,7 @@ func prunableStores(list ...*mockStore) []PrunableStore { return result } -func testConfig(t *testing.T, rollbackWindow, lookbackWindow uint64) *StorageGarbageCollectorConfig { +func testConfig(t *testing.T, rollbackWindow uint64, lookbackWindow int64) *StorageGarbageCollectorConfig { t.Helper() config := &StorageGarbageCollectorConfig{ RollbackWindow: rollbackWindow, @@ -142,7 +142,7 @@ func TestPruneDecisions(t *testing.T) { cases := []struct { name string rollbackWindow uint64 - lookbackWindow uint64 + lookbackWindow int64 stores []*mockStore wantSnapshotsBelow *uint64 wantHistoryBelow *uint64 @@ -333,6 +333,64 @@ func TestPruneDecisions(t *testing.T) { }, wantSnapshotsBelow: ptr(10_000), }, + { + // -1 is infinite history retention: snapshots below the rollback floor are still + // reclaimed, but history is never pruned however far that floor advances. + name: "infinite lookback: snapshots only, history untouched", + rollbackWindow: 1_000, + lookbackWindow: -1, + stores: []*mockStore{ + snapshotStore("sc", 100_000, 10_000), + contiguousStore("stateWAL", 100_000), + }, + wantSnapshotsBelow: ptr(10_000), + }, + { + // Infinite lookback must not force a deletion on a chain younger than its rollback + // window. Every store owes a rollback deeper than its whole history, answers 0, and the + // minimum holds both cut lines there — the -1 that zeroes the history cut line finds it + // already 0, so nothing is pruned anywhere. + name: "infinite lookback on a young chain: nothing deleted", + rollbackWindow: 1_000, + lookbackWindow: -1, + stores: []*mockStore{ + snapshotStore("sc", 100, 50), + contiguousStore("blockDB", 100), + contiguousStore("stateWAL", 100), + }, + }, + { + // A fleet that keeps no snapshots — blockDB, receiptDB, WAL — still has a snapshot cut + // line: the collector issues PruneSnapshots and each store no-ops it. Infinite lookback + // holds every block of history above genesis. + name: "infinite lookback on a contiguous fleet: history untouched", + rollbackWindow: 10_000, + lookbackWindow: -1, + stores: []*mockStore{ + contiguousStore("blockDB", 100_000), + contiguousStore("receiptDB", 100_000), + contiguousStore("stateWAL", 100_000), + }, + // All three answer 90_000; the snapshot cut line fires as a no-op on stores that hold + // none, and history is never pruned. + wantSnapshotsBelow: ptr(90_000), + }, + { + // The mixed fleet the collector actually runs: SC holds the snapshots and binds the + // snapshot cut line, while blockDB, receiptDB and the WAL answer from their own heads. + // Infinite lookback keeps all history across every one of them. + name: "infinite lookback across SC, blockDB, receiptDB and WAL", + rollbackWindow: 10_000, + lookbackWindow: -1, + stores: []*mockStore{ + snapshotStore("sc", 100_000, 50_000), + contiguousStore("blockDB", 100_000), + contiguousStore("receiptDB", 100_000), + contiguousStore("stateWAL", 100_000), + }, + // sc 50_000; the contiguous three 90_000 → min 50_000. History is never pruned. + wantSnapshotsBelow: ptr(50_000), + }, { name: "no store has a latest block", rollbackWindow: 10_000, @@ -491,7 +549,7 @@ func TestGetHistoryCutLine(t *testing.T) { cases := []struct { name string snapshotCutLine uint64 - lookbackWindow uint64 + lookbackWindow int64 want uint64 }{ {name: "no lookback leaves the cut line alone", snapshotCutLine: 90_000, want: 90_000}, @@ -503,7 +561,8 @@ func TestGetHistoryCutLine(t *testing.T) { {name: "a cut line of 0 stays 0", snapshotCutLine: 0, lookbackWindow: 10, want: 0}, {name: "no cut line and no lookback", want: 0}, {name: "max cut line", snapshotCutLine: math.MaxUint64, lookbackWindow: 1, want: math.MaxUint64 - 1}, - {name: "max lookback", snapshotCutLine: 100, lookbackWindow: math.MaxUint64, want: 0}, + {name: "infinite lookback holds history at 0", snapshotCutLine: math.MaxUint64, lookbackWindow: -1, want: 0}, + {name: "infinite lookback with no cut line", snapshotCutLine: 0, lookbackWindow: -1, want: 0}, } for _, tc := range cases { @@ -584,23 +643,31 @@ func TestPruneAsksEveryStoreOncePerCycle(t *testing.T) { func TestDefaultStorageGarbageCollectorConfig(t *testing.T) { cfg := DefaultStorageGarbageCollectorConfig() require.Equal(t, uint64(1_000), cfg.RollbackWindow) - require.Equal(t, uint64(0), cfg.LookbackWindow) + require.Equal(t, int64(0), cfg.LookbackWindow) require.Equal(t, 5*time.Minute, cfg.PruneInterval) require.NoError(t, cfg.Validate()) } -// Both windows are independent, so every combination validates. Only the interval is constrained. +// The windows are independent, so every combination of non-negative counts validates, as does the +// -1 infinite-retention sentinel on the lookback window. The interval and a lookback below -1 are +// the only rejections. func TestValidate(t *testing.T) { require.ErrorContains(t, (*StorageGarbageCollectorConfig)(nil).Validate(), "config is required") - for _, windows := range [][2]uint64{{0, 0}, {1, 0}, {0, 1}, {1_000, 50_000}, {50_000, 1_000}} { + for _, windows := range [][2]int64{{0, 0}, {1, 0}, {0, 1}, {1_000, 50_000}, {50_000, 1_000}, {1_000, -1}} { require.NoError(t, (&StorageGarbageCollectorConfig{ - RollbackWindow: windows[0], + RollbackWindow: uint64(windows[0]), LookbackWindow: windows[1], PruneInterval: time.Minute, }).Validate(), "windows %v", windows) } + require.ErrorContains(t, (&StorageGarbageCollectorConfig{ + RollbackWindow: 1, + LookbackWindow: -2, + PruneInterval: time.Minute, + }).Validate(), "lookback window") + require.ErrorContains(t, (&StorageGarbageCollectorConfig{ RollbackWindow: 1, PruneInterval: 0, From 5104f64550799f22bcf4c9ab53231d22ed8e481c Mon Sep 17 00:00:00 2001 From: YimingZang Date: Mon, 10 Aug 2026 22:21:22 -0700 Subject: [PATCH 11/11] Address comments --- sei-db/management/gc/storage_garbage_collector.go | 10 ++++++---- .../management/gc/storage_garbage_collector_config.go | 11 +++++++---- .../management/gc/storage_garbage_collector_test.go | 4 ++-- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/sei-db/management/gc/storage_garbage_collector.go b/sei-db/management/gc/storage_garbage_collector.go index 34866d27ed..5b54d666d6 100644 --- a/sei-db/management/gc/storage_garbage_collector.go +++ b/sei-db/management/gc/storage_garbage_collector.go @@ -21,9 +21,10 @@ var logger = seilog.NewLogger("db", "gc") // 1. ask every store GetRollbackFloor(RollbackWindow) — the earliest height it could still be // asked to roll back to, which it resolves against its own head // 2. snapshotCutLine = min of those answers, the deepest rollback the fleet still owes -// 3. historyCutLine = snapshotCutLine - LookbackWindow, so the lookback window sits entirely below -// the deepest promised rollback point rather than overlapping it; a LookbackWindow of -1 makes -// this 0, which is infinite history retention (snapshots are still reclaimed to the floor) +// 3. historyCutLine sits LookbackWindow below snapshotCutLine, so the lookback window falls +// entirely beneath the deepest promised rollback point rather than overlapping it; a +// LookbackWindow of -1 pins it to 0, which is infinite history retention (snapshots are still +// reclaimed to the floor) // 4. on every store reporting ExternalPruning, PruneSnapshots(snapshotCutLine) then // PruneHistory(historyCutLine) // @@ -120,7 +121,8 @@ func prune(config *StorageGarbageCollectorConfig, stores []PrunableStore) error } // pruneStores issues the deletions the cycle decided on: snapshots below snapshotHeight, history -// below the deeper historyHeight. See StorageGarbageCollector for why the two depths differ. +// below historyHeight — normally the deeper of the two, and 0 (never prune) under an infinite +// lookback window. See StorageGarbageCollector for why the two depths differ. // // A cut line of 0 is skipped rather than passed down. It means nothing is eligible, which every // store would absorb as a no-op anyway; not making the call keeps a cycle that decided to delete diff --git a/sei-db/management/gc/storage_garbage_collector_config.go b/sei-db/management/gc/storage_garbage_collector_config.go index def285657e..7e3bf9f6ac 100644 --- a/sei-db/management/gc/storage_garbage_collector_config.go +++ b/sei-db/management/gc/storage_garbage_collector_config.go @@ -20,8 +20,9 @@ type StorageGarbageCollectorConfig struct { // makes the promise independent of rollback depth: however far the node rewinds inside the // rollback window, at least LookbackWindow blocks below the new head stay readable. // - // One window covers every managed store. With F = LatestBlock - RollbackWindow - - // LookbackWindow, collection guarantees: + // One window covers every managed store. With a finite LookbackWindow and + // F = LatestBlock - RollbackWindow - LookbackWindow, collection guarantees (a LookbackWindow of + // -1 keeps everything, so F is genesis): // // 1. Nothing needed to roll back to any block in // [LatestBlock - RollbackWindow, LatestBlock] is deleted. @@ -58,8 +59,10 @@ func DefaultStorageGarbageCollectorConfig() *StorageGarbageCollectorConfig { // Validate checks that required fields are set to usable values. // -// The windows are additive, so every combination of them is meaningful and neither is constrained -// against the other. A sum that overflows uint64 is handled in getHistoryCutLine. +// The two windows are independent — RollbackWindow bounds each store's floor, LookbackWindow is +// subtracted from the resulting minimum — so every combination is meaningful and neither is +// constrained against the other. A lookback large enough to reach below genesis is not an error; +// getHistoryCutLine saturates it to 0. // // LookbackWindow is a block count and so is bounded below by 0, except for -1, the infinite-retention // sentinel. Any other negative is a typo — most likely -1 miswritten — and is rejected rather than diff --git a/sei-db/management/gc/storage_garbage_collector_test.go b/sei-db/management/gc/storage_garbage_collector_test.go index 6c3aea26ea..9a8d802589 100644 --- a/sei-db/management/gc/storage_garbage_collector_test.go +++ b/sei-db/management/gc/storage_garbage_collector_test.go @@ -60,7 +60,7 @@ func snapshotStore(name string, latestHeight uint64, snapshots ...uint64) *mockS // contiguousStore models blockDB / receiptDB / WAL: it can restore to any height it holds, so it // answers its own head less the window — and 0 where the window is deeper than that head, which -// includes the empty store and holds the prune height at 0. +// includes the empty store and holds both cut lines at 0. func contiguousStore(name string, latestHeight uint64) *mockStore { return &mockStore{ name: name, @@ -466,7 +466,7 @@ func TestPruneStillPrunesSnapshotsWhenHistoryIsHeldAtZero(t *testing.T) { } // A chain younger than its own rollback window must lose nothing: every store owes a rollback -// deeper than its whole history, so every one answers 0 and the prune height stays there. The +// deeper than its whole history, so every one answers 0 and both cut lines stay there. The // deletions are still issued — they are no-ops at 0 — which is what keeps the collector free of a // special case for the young chain. func TestPruneOnYoungChain(t *testing.T) {