diff --git a/AGENTS.md b/AGENTS.md index fa3426f844..0d3b65b1ee 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,6 +51,38 @@ gofmt -s -l . goimports -l . ``` +### Godoc + +Godocs say **what** a thing is, not why it came to be or how it works inside. + +1. **Explain WHAT, not WHY or HOW.** Rationale, trade-offs, and mechanism belong in + an inline comment at the line that needs them, or nowhere. +2. **Never record design history.** No "this was previously X", "used to mean Y", + "renamed from Z". The diff and the git log hold that. +3. **Multi-paragraph godocs are rare.** Most functions do not earn a second + paragraph. One or two sentences is the norm. +4. **Rewrite, don't patch.** When a godoc needs to change, write it again from + scratch; incrementally editing one reliably produces a rambling comment. +5. **Document the subject, not the system.** A godoc is not the place to explain + the surrounding architecture. Describe this function, type, or field. + +```go +// ❌ BAD — history, mechanism, and a system tour +// GetRollbackFloor returns the earliest height a rollback may target. The window is +// measured against the store's own head rather than a height handed down, because the +// collector takes a minimum across stores, so a lagging store sets the depth. 0 means +// nothing is eligible; it is a height rather than a sentinel, since CannotServeRollback +// used to serve that role and was removed. Answering high is the damaging direction, +// as nothing above clamps it: the collector derives its cut lines from these answers. +func (s *blockDB) GetRollbackFloor(rollbackWindow uint64) uint64 + +// ✅ GOOD — what it returns, and what the caller must know +// GetRollbackFloor returns the earliest height a rollback may target, measured against +// this store's own head. It returns 0 when the window is deeper than the store's +// history, meaning no data here is eligible for pruning. +func (s *blockDB) GetRollbackFloor(rollbackWindow uint64) uint64 +``` + ## Structural corrections When a defect is found, the change that closes it has to leave the code readable as a diff --git a/app/testdata/state-commit.golden b/app/testdata/state-commit.golden index 58522b9721..bd9be14299 100644 --- a/app/testdata/state-commit.golden +++ b/app/testdata/state-commit.golden @@ -15,6 +15,7 @@ FlatKVConfig.Fsync = bool(false) FlatKVConfig.AsyncWriteBuffer = int(0) FlatKVConfig.SnapshotInterval = uint32(10000) FlatKVConfig.SnapshotKeepRecent = uint32(1) +FlatKVConfig.ExternalPruning = bool(false) FlatKVConfig.EnablePebbleMetrics = bool(true) FlatKVConfig.EnableReadWriteMetrics = bool(false) FlatKVConfig.AccountDBConfig.DataDir = string("") diff --git a/sei-cosmos/server/config/testdata/server_config.golden b/sei-cosmos/server/config/testdata/server_config.golden index a2ce09015b..a8b4777fa6 100644 --- a/sei-cosmos/server/config/testdata/server_config.golden +++ b/sei-cosmos/server/config/testdata/server_config.golden @@ -67,6 +67,7 @@ StateCommit.FlatKVConfig.Fsync = bool(false) StateCommit.FlatKVConfig.AsyncWriteBuffer = int(0) StateCommit.FlatKVConfig.SnapshotInterval = uint32(10000) StateCommit.FlatKVConfig.SnapshotKeepRecent = uint32(1) +StateCommit.FlatKVConfig.ExternalPruning = bool(false) StateCommit.FlatKVConfig.EnablePebbleMetrics = bool(true) StateCommit.FlatKVConfig.EnableReadWriteMetrics = bool(false) StateCommit.FlatKVConfig.AccountDBConfig.DataDir = string("") diff --git a/sei-db/config/giga_config.go b/sei-db/config/giga_config.go index 6aad843839..e7451d363f 100644 --- a/sei-db/config/giga_config.go +++ b/sei-db/config/giga_config.go @@ -14,15 +14,14 @@ import ( // callers build it via DefaultGigaStorageConfig. Nested knobs live on the store and // collector configs they already own (StateStoreConfig, ReceiptStoreConfig, // gc.StorageGarbageCollectorConfig) rather than being redeclared here — in particular -// RollbackWindow has a single source of truth in -// gc.DefaultStorageGarbageCollectorConfig; per-store extras live on each -// PrunableStore via GetRetentionWindow. +// RollbackWindow and LookbackWindow have a single source of truth in +// gc.DefaultStorageGarbageCollectorConfig, and cover every managed store at once. type GigaStorageConfig struct { HomePath string FlatKVConfig *flatkvConfig.Config SSConfig StateStoreConfig ReceiptDBConfig ReceiptStoreConfig - BlockDBConfig *littblock.LittBlockConfig + BlockDBConfig *littblock.BlockDBConfig PruningConfig *gc.StorageGarbageCollectorConfig } diff --git a/sei-db/config/receipt_config.go b/sei-db/config/receipt_config.go index 2a18f17d52..fd40748531 100644 --- a/sei-db/config/receipt_config.go +++ b/sei-db/config/receipt_config.go @@ -55,6 +55,16 @@ type ReceiptStoreConfig struct { // default to every 600 seconds PruneIntervalSeconds int `mapstructure:"prune-interval-seconds"` + // ExternalPruning hands retention to the StorageGarbageCollector: the littidx backend stops + // running its own KeepRecent pruner, and the collector prunes it instead. + // + // Like KeepRecent this is not read from the receipt-store config. It is set by whatever + // constructs the collector, since it is only correct when this store is registered with a + // running one. + // + // Only the littidx backend supports it; newReceiptBackend rejects it on pebbledb. + ExternalPruning bool `mapstructure:"-"` + // EnableReadWriteMetrics emits simple estimated read/write counters for Pebble-backed receipt storage. // defaults to false EnableReadWriteMetrics bool `mapstructure:"enable-read-write-metrics"` diff --git a/sei-db/config/receipt_config_fuzz_test.go b/sei-db/config/receipt_config_fuzz_test.go index 37f1e48bc0..62b245225f 100644 --- a/sei-db/config/receipt_config_fuzz_test.go +++ b/sei-db/config/receipt_config_fuzz_test.go @@ -206,6 +206,12 @@ func TestManifestNamesEveryField(t *testing.T) { // sitting in a config struct that configuration cannot address is exactly the kind of // thing a replacement manager would otherwise try to map a key onto. "KeepRecent", + // ExternalPruning is tagged mapstructure:"-" for a sharper reason than KeepRecent: it is + // only correct when this store is registered with a running StorageGarbageCollector, which + // is a property of how the process was wired and not something an operator can assert from + // app.toml. Exposing a key for it would let a node stand its pruner down with nothing to + // replace it, and the resulting unbounded growth is silent. + "ExternalPruning", ) } diff --git a/sei-db/config/testdata/receipt-store.golden b/sei-db/config/testdata/receipt-store.golden index fb4f0c5ebb..e6891d7a05 100644 --- a/sei-db/config/testdata/receipt-store.golden +++ b/sei-db/config/testdata/receipt-store.golden @@ -3,5 +3,6 @@ Backend = string("pebbledb") AsyncWriteBuffer = int(100) KeepRecent = int(0) PruneIntervalSeconds = int(600) +ExternalPruning = bool(false) EnableReadWriteMetrics = bool(false) LogFilterParallelism = int(16) diff --git a/sei-db/config/toml_test.go b/sei-db/config/toml_test.go index deee57a2ea..5ffb274af6 100644 --- a/sei-db/config/toml_test.go +++ b/sei-db/config/toml_test.go @@ -58,6 +58,7 @@ func TestStateCommitConfigTemplate(t *testing.T) { require.NotContains(t, flatKVSection, "snapshot-interval", "FlatKV snapshot-interval should not be exposed in app.toml") require.NotContains(t, flatKVSection, "snapshot-keep-recent", "FlatKV snapshot-keep-recent should not be exposed in app.toml") require.NotContains(t, flatKVSection, "enable-read-write-metrics", "FlatKV read/write metrics flag should not be exposed in app.toml") + require.NotContains(t, flatKVSection, "external-pruning", "FlatKV external-pruning should not be exposed in app.toml") // sc-snapshot-writer-limit is intentionally removed from template (hardcoded to 4) // but old configs with this field still parse fine via mapstructure diff --git a/sei-db/ledger_db/block/block_db_test.go b/sei-db/ledger_db/block/block_db_test.go index 7b8bde798b..1c07f8e703 100644 --- a/sei-db/ledger_db/block/block_db_test.go +++ b/sei-db/ledger_db/block/block_db_test.go @@ -1402,10 +1402,10 @@ func TestMemblockPruneIntoCohortRoundsDown(t *testing.T) { // littConfig builds a littblock config rooted at dir with a tiny retention so // the prune watermark is the sole observable reclamation gate in tests. -func littConfig(t *testing.T, dir string) *littblock.LittBlockConfig { +func littConfig(t *testing.T, dir string) *littblock.BlockDBConfig { cfg, err := littblock.DefaultConfig(dir) require.NoError(t, err) - cfg.Retention = time.Nanosecond + cfg.RetentionTime = time.Nanosecond return cfg } diff --git a/sei-db/ledger_db/block/blocksim/blocksim.go b/sei-db/ledger_db/block/blocksim/blocksim.go index e70e8d4a87..2e74e261bf 100644 --- a/sei-db/ledger_db/block/blocksim/blocksim.go +++ b/sei-db/ledger_db/block/blocksim/blocksim.go @@ -506,7 +506,7 @@ func openBlockDB(config *BlocksimConfig) (types.BlockDB, error) { if err != nil { return nil, fmt.Errorf("failed to build litt block db config: %w", err) } - littConfig.Retention = time.Duration(config.LittRetentionSeconds) * time.Second + littConfig.RetentionTime = time.Duration(config.LittRetentionSeconds) * time.Second // Record litt_* metrics into blocksim's already-configured global OTel MeterProvider (set up in // main before the DB is opened). MetricsServeEndpoint stays false so LittDB does not stand up its // own registry/server; the metrics surface on blocksim's single /metrics endpoint. diff --git a/sei-db/ledger_db/block/littblock/litt_block_config.go b/sei-db/ledger_db/block/littblock/litt_block_config.go index 1efa7e8d08..bae72539bd 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_config.go +++ b/sei-db/ledger_db/block/littblock/litt_block_config.go @@ -7,44 +7,49 @@ import ( littdb "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt" ) -// LittBlockConfig configures a LittDB-backed types.BlockDB. -type LittBlockConfig struct { +// BlockDBConfig configures a LittDB-backed types.BlockDB. +type BlockDBConfig struct { // Litt is the underlying LittDB configuration, including the data directory - // paths. The block store builds its two tables (blocks, qcs) on top of this - // DB. Required; use DefaultConfig to obtain one with sane defaults, then - // override fields as needed (e.g. Litt.Fsync, Litt.GCPeriod). + // paths. The block store builds its single table (see tableName, which holds + // blocks and QCs both) on top of this DB. Required; use DefaultConfig to obtain + // one with sane defaults, then override fields as needed (e.g. Litt.Fsync, + // Litt.GCPeriod). Litt *littdb.Config - // Retention is the failsafe minimum age before any pruned record may be + // RetentionTime is the failsafe minimum age before any pruned record may be // reclaimed. Reclamation requires BOTH this age to elapse AND the prune - // watermark to advance past the record, so even an over-eager watermark - // cannot delete data younger than Retention. Must be positive. - Retention time.Duration + // watermark to advance past the record. Must be positive. + // + // It is an age floor, not a retention policy: how much history this store keeps + // is the RollbackWindow and LookbackWindow on gc.StorageGarbageCollectorConfig. + // + // Default: 1h. + RetentionTime time.Duration } -// DefaultConfig returns a LittBlockConfig preloaded with all defaults, rooted at +// DefaultConfig returns a BlockDBConfig preloaded with all defaults, rooted at // dir. Override fields as needed, then pass it to NewBlockDB (which validates). -func DefaultConfig(dir string) (*LittBlockConfig, error) { +func DefaultConfig(dir string) (*BlockDBConfig, error) { littConfig, err := littdb.DefaultConfig(dir) if err != nil { return nil, fmt.Errorf("failed to build litt config: %w", err) } - return &LittBlockConfig{ - Litt: littConfig, - Retention: 24 * time.Hour, + return &BlockDBConfig{ + Litt: littConfig, + RetentionTime: time.Hour, }, nil } // Validate performs a sanity check on the configuration. -func (c *LittBlockConfig) Validate() error { +func (c *BlockDBConfig) Validate() error { if c == nil { return fmt.Errorf("config is required") } if c.Litt == nil { return fmt.Errorf("config.Litt is required") } - if c.Retention <= 0 { - return fmt.Errorf("config.Retention must be positive (got %s)", c.Retention) + if c.RetentionTime <= 0 { + return fmt.Errorf("config.RetentionTime must be positive (got %s)", c.RetentionTime) } return nil } diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index f29576b62d..68cf7af46a 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -2,6 +2,8 @@ package littblock import ( "fmt" + "os" + "path/filepath" "sync" "sync/atomic" @@ -12,11 +14,16 @@ import ( "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" ) -// ledgerTableName is the single table holding both blocks and QCs. They share -// one table so a crash leaves a contiguous write-order prefix spanning both -// record kinds (see NewBlockDB), which is what guarantees a persisted block is -// always covered by a persisted QC. -const ledgerTableName = "ledger" +// tableName is the single table holding blocks and QCs both, despite the name. +// +// It is persisted layout rather than just an identifier: littdb stores a table's data at +// //segments, so changing it makes NewBlockDB open a fresh empty table and leaves +// data under the old name unreachable. +const tableName = "blocks" + +// legacyTableName is the name tableName had in earlier versions. Nothing opens it; refuseLegacyTable +// only uses it to recognize a directory left behind by one of those versions. +const legacyTableName = "ledger" var _ types.BlockDB = (*blockDB)(nil) @@ -74,13 +81,18 @@ type blockDB struct { } // NewBlockDB opens (or creates) a LittDB-backed types.BlockDB from config. The -// underlying LittDB is built from config.Litt, and the two tables apply -// config.Retention as a TTL failsafe (pruning never reclaims data younger than -// that even once the watermark has advanced past it). -func NewBlockDB(config *LittBlockConfig) (types.BlockDB, error) { +// underlying LittDB is built from config.Litt, and the table applies +// config.RetentionTime as a TTL failsafe (pruning never reclaims data younger +// than that even once the watermark has advanced past it). +func NewBlockDB(config *BlockDBConfig) (types.BlockDB, error) { if err := config.Validate(); err != nil { return nil, fmt.Errorf("invalid block db config: %w", err) } + // Before littbuilder.NewDB, so a refused open leaves the directory exactly as it found it + // rather than adding an empty table beside the one it is complaining about. + if err := refuseLegacyTable(config.Litt.Paths); err != nil { + return nil, err + } db, err := littbuilder.NewDB(config.Litt) if err != nil { return nil, fmt.Errorf("failed to open litt db: %w", err) @@ -96,8 +108,8 @@ func NewBlockDB(config *LittBlockConfig) (types.BlockDB, error) { // guarantees a persisted block is always covered by a persisted QC. It also // backs the write-order cursors and contiguous-QC recovery. ShardingFactor // > 1, or splitting blocks and QCs across two tables, would void this. - tableConfig := littdb.DefaultTableConfig(ledgerTableName) - tableConfig.TTL = config.Retention + tableConfig := littdb.DefaultTableConfig(tableName) + tableConfig.TTL = config.RetentionTime tableConfig.GCFilter = s.gcFilter tableConfig.ShardingFactor = 1 // DO NOT CHANGE!! table, err := db.BuildTable(tableConfig) @@ -119,6 +131,29 @@ func NewBlockDB(config *LittBlockConfig) (types.BlockDB, error) { return s, nil } +// refuseLegacyTable fails the open when any root path holds a table directory under legacyTableName, +// which this process cannot reach and would otherwise leave the store looking healthy and empty. The +// operator action is to delete the directory or move it aside. +// +// A root that cannot be stat'd is also refused, since it cannot rule out such a directory. +func refuseLegacyTable(paths []string) error { + for _, root := range paths { + legacy := filepath.Join(root, legacyTableName) + switch _, err := os.Stat(legacy); { + case err == nil: + return fmt.Errorf( + "block db: found a pre-rename %q table at %s; the table is now named %q, so those "+ + "blocks and QCs would be neither served nor reclaimed. Delete or move the "+ + "directory aside to start from an empty store", + legacyTableName, legacy, tableName) + case !os.IsNotExist(err): + return fmt.Errorf("block db: check for a pre-rename %q table at %s: %w", + legacyTableName, legacy, err) + } + } + return nil +} + // recoverCursors reloads the write-order cursors (lastBlockNumber, lastQCNext, // and their presence flags) from on-disk state. Without this, a reopened DB // would treat itself as empty and let WriteBlock/WriteQC silently accept diff --git a/sei-db/ledger_db/block/littblock/litt_block_gc.go b/sei-db/ledger_db/block/littblock/litt_block_gc.go new file mode 100644 index 0000000000..ff48ba9c32 --- /dev/null +++ b/sei-db/ledger_db/block/littblock/litt_block_gc.go @@ -0,0 +1,56 @@ +package littblock + +import ( + "github.com/sei-protocol/sei-chain/sei-db/management/gc" + "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" +) + +// blockDB participates in the shared prune cycle as a contiguous store: it holds every block from its +// retention watermark to its head, so any height in that range is restorable directly. +var _ gc.PrunableStore = (*blockDB)(nil) + +func (s *blockDB) Name() string { + return "BlockDB" +} + +// ExternalPruning is unconditionally true: this store has no pruner of its own. +func (s *blockDB) ExternalPruning() bool { + return true +} + +// PruneHistory advances the retention watermark to blockNumber, capping it at the newest retained +// (block, QC) pair. Reclaiming the data happens later, on LittDB's GC schedule and no earlier than +// BlockDBConfig.RetentionTime. +func (s *blockDB) PruneHistory(blockNumber uint64) error { + return s.PruneBefore(types.GlobalBlockNumber(blockNumber)) +} + +// PruneSnapshots does nothing: blockDB keeps no snapshots. +func (s *blockDB) PruneSnapshots(uint64) error { + return nil +} + +// GetRollbackFloor returns head - rollbackWindow, measured against this store's own head. It returns +// 0 — nothing here is eligible for pruning — when the window is deeper than the whole history, which +// includes the empty store, or when the head cannot be read. +func (s *blockDB) GetRollbackFloor(rollbackWindow uint64) uint64 { + head, err := s.GetLatestBlock() + if err != nil || head <= rollbackWindow { + return 0 + } + return head - rollbackWindow +} + +// GetLatestBlock returns the newest block number written, or 0 when none has been. It reports the +// written cursor rather than the flushed one, so a block a crash would lose still counts as ingested. +// +// Global block numbers start at genesis block 0, so a store holding only that block is +// indistinguishable from an empty one. +func (s *blockDB) GetLatestBlock() (uint64, error) { + s.mu.Lock() + defer s.mu.Unlock() + if !s.hasBlocks { + return 0, nil + } + return uint64(s.lastBlockNumber), nil +} diff --git a/sei-db/ledger_db/block/littblock/litt_block_gc_test.go b/sei-db/ledger_db/block/littblock/litt_block_gc_test.go new file mode 100644 index 0000000000..73f8382247 --- /dev/null +++ b/sei-db/ledger_db/block/littblock/litt_block_gc_test.go @@ -0,0 +1,156 @@ +package littblock + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/management/gc" + "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" +) + +// gcConfig builds a config for the collector-facing tests: reclamation is gated solely by the +// prune watermark (tiny TTL) and only ForceGC reclaims, so a watermark assertion is never +// confused by a background pass. Segment sizing is left at its defaults — these tests are about +// the decisions the collector drives, not about which segment a record lands in. +func gcConfig(t *testing.T, dir string) *BlockDBConfig { + cfg, err := DefaultConfig(dir) + require.NoError(t, err) + cfg.RetentionTime = time.Nanosecond + cfg.Litt.GCPeriod = time.Hour + cfg.Litt.Fsync = false + return cfg +} + +// openForGC opens a store at dir and returns it as the collector sees it, closing it on cleanup. +// Opening a store is the expensive part of these tests, so each one opens as few as it can. +func openForGC(t *testing.T, dir string) (types.BlockDB, gc.PrunableStore) { + t.Helper() + db, err := NewBlockDB(gcConfig(t, dir)) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + store, ok := db.(gc.PrunableStore) + require.True(t, ok, "a littblock store must satisfy gc.PrunableStore") + return db, store +} + +// The head the collector reads is the newest block written — not the newest QC's coverage, since +// a QC is written before the blocks it covers — and 0 while nothing has been ingested, so an +// empty store drops out of the head minimum instead of dragging the lookback floor to 0. The reopen +// at the end covers recovery: a store that reported 0 after a restart would let the other stores +// prune past a height this one still holds. +func TestGCLatestBlock(t *testing.T) { + dir := t.TempDir() + rng := utils.TestRngFromSeed(1) + db, store := openForGC(t, dir) + + latest, err := store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(0), latest, "a store that has ingested nothing has no head") + + writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19, QCs [0,5)..[15,20) + latest, err = store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(19), latest) + + // A QC covering 20..24 is written but none of its blocks are, so the head must not move. + require.NoError(t, db.WriteQC(types.GenFullCommitQCRange(rng, 20, 25))) + latest, err = store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(19), latest, "a QC ahead of its blocks must not advance the head") + + require.NoError(t, db.WriteBlock(20, types.GenBlock(rng))) + require.NoError(t, db.Flush()) + require.NoError(t, db.Close()) + + _, reopened := openForGC(t, dir) + latest, err = reopened.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(20), latest, "the head must be re-derived on open") +} + +// blockDB keeps no snapshots, so its half of the split contract is a no-op. It is still called +// every cycle, and answering with an error would fail a cycle that has nothing to do with it. +func TestGCPruneSnapshotsIsANoOp(t *testing.T) { + db, store := openForGC(t, t.TempDir()) + rng := utils.TestRngFromSeed(4) + writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19 + + require.NoError(t, store.PruneSnapshots(10)) + require.Equal(t, uint64(0), db.(*blockDB).watermark.Load(), + "pruning snapshots must not move the history watermark") + + blk, err := db.ReadBlockByNumber(0) + require.NoError(t, err) + require.True(t, blk.IsPresent(), "no block may be dropped by a snapshot prune") +} + +// A contiguous store resolves the rollback window against its own head: head - rollbackWindow, or 0 +// where that head has not cleared the window. The states below are the ones where a snapshot store +// would answer differently. +// +// The prune assertions ride along on the same store because both methods are answers about the +// same three states, and opening a second store to re-reach them is the slow part of this file. +func TestGCRollbackFloorAndPruneHistory(t *testing.T) { + db, store := openForGC(t, t.TempDir()) + rng := utils.TestRngFromSeed(2) + impl := db.(*blockDB) + + // Empty, so the head is 0 and every window reports 0 — "keep everything", which holds the + // fleet's history where it is until this store fills. The prune that follows is a no-op rather + // than an error. + require.Equal(t, uint64(0), store.GetRollbackFloor(0)) + require.Equal(t, uint64(0), store.GetRollbackFloor(42)) + require.NoError(t, store.PruneHistory(1_000)) + require.Equal(t, uint64(0), impl.watermark.Load()) + + writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19, QCs [0,5),[5,10),[10,15),[15,20) + require.Equal(t, uint64(19), store.GetRollbackFloor(0), "the whole store is inside a window of 0") + require.Equal(t, uint64(7), store.GetRollbackFloor(12)) + // A window deeper than the store's own head is a rollback promise reaching past genesis, so + // nothing here is eligible for pruning yet. + require.Equal(t, uint64(0), store.GetRollbackFloor(1_000)) + + require.NoError(t, store.PruneHistory(7)) + require.Equal(t, uint64(5), impl.watermark.Load(), "a prune inside QC[5,10) rounds down to its start") + require.NoError(t, store.PruneHistory(3)) + require.Equal(t, uint64(5), impl.watermark.Load(), "the watermark must not move backwards") + + // Already pruned above the answer: the floor is a function of the head, not of the retention + // floor, so a store pruned higher by an earlier cycle does not hold the others back to where it + // started. + require.Equal(t, uint64(15), store.GetRollbackFloor(4)) +} + +// The collector prunes every store to a shared minimum, so a store lagging the head is asked to +// prune past everything it holds. The never-empty cap turns that into a prune to the newest +// cohort rather than a store that can serve nothing. +func TestGCPruneHistoryAboveHeadIsCapped(t *testing.T) { + db, store := openForGC(t, t.TempDir()) + rng := utils.TestRngFromSeed(3) + + writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19, newest cohort QC[15,20) + require.NoError(t, store.PruneHistory(1_000)) + require.Equal(t, uint64(15), db.(*blockDB).watermark.Load(), "a prune past the head is capped to the newest cohort") + + for n := types.GlobalBlockNumber(15); n < 20; n++ { + blk, err := db.ReadBlockByNumber(n) + require.NoError(t, err) + require.True(t, blk.IsPresent(), "block %d in the newest cohort must survive", n) + } +} + +func TestConfigValidateRetentionTime(t *testing.T) { + cfg, err := DefaultConfig(t.TempDir()) + require.NoError(t, err) + require.NoError(t, cfg.Validate()) + + // RetentionTime gates reclamation underneath the watermark, so a non-positive value would + // release records the moment the watermark passed them, removing the failsafe entirely. + for _, retentionTime := range []time.Duration{0, -time.Second} { + cfg.RetentionTime = retentionTime + require.ErrorContains(t, cfg.Validate(), "RetentionTime") + } +} diff --git a/sei-db/ledger_db/block/littblock/litt_block_legacy_table_test.go b/sei-db/ledger_db/block/littblock/litt_block_legacy_table_test.go new file mode 100644 index 0000000000..2559f7327d --- /dev/null +++ b/sei-db/ledger_db/block/littblock/litt_block_legacy_table_test.go @@ -0,0 +1,63 @@ +package littblock + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// A home written before the table rename must not open as an empty store. The data under the old +// name is unreachable either way; the only question is whether the operator is told, and a healthy +// empty store tells them nothing until the history they wanted is already gone. +func TestOpenRefusesAPreRenameTable(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, legacyTableName, "segments"), 0o700)) + + _, err := NewBlockDB(gcConfig(t, dir)) + require.ErrorContains(t, err, legacyTableName) + require.ErrorContains(t, err, "Delete or move the directory aside") +} + +// The refusal happens before littdb builds anything, so a refused open leaves the directory as it +// found it. Otherwise the operator inspecting the failure would find a new empty table sitting +// next to the old one, and could not tell which of the two the error was about. +func TestRefusedOpenLeavesTheDirectoryUntouched(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, legacyTableName, "segments"), 0o700)) + + _, err := NewBlockDB(gcConfig(t, dir)) + require.Error(t, err) + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Len(t, entries, 1, "nothing may be created beside the legacy table") + require.Equal(t, legacyTableName, entries[0].Name()) +} + +// The guard keys on the old name specifically: a normal home has no such directory and must open, +// and the check must not be satisfied by the store's own table. +func TestOpenAcceptsAHomeWithoutAPreRenameTable(t *testing.T) { + dir := t.TempDir() + + db, err := NewBlockDB(gcConfig(t, dir)) + require.NoError(t, err) + require.NoError(t, db.Close()) + + require.DirExists(t, filepath.Join(dir, tableName), "the store builds its table under the current name") + + db2, err := NewBlockDB(gcConfig(t, dir)) + require.NoError(t, err, "reopening a store it just wrote must not trip the guard") + require.NoError(t, db2.Close()) +} + +// Every root is checked, not just the first: littdb spreads a table across all of them, so data +// under the old name in any one of them is data this store cannot reach. +func TestOpenRefusesAPreRenameTableInANonFirstRoot(t *testing.T) { + first, second := t.TempDir(), t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(second, legacyTableName), 0o700)) + + require.NoError(t, refuseLegacyTable([]string{first})) + require.ErrorContains(t, refuseLegacyTable([]string{first, second}), second) +} diff --git a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go index dad0065f8f..002433a083 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go +++ b/sei-db/ledger_db/block/littblock/litt_block_stranding_test.go @@ -15,10 +15,10 @@ import ( // MaxSegmentKeyCount, so a test can place a segment boundary at a precise key. // Retention is tiny (the prune watermark is the sole reclamation gate) and GC is // effectively background-disabled so ForceGC is the only thing that reclaims. -func strandingConfig(t *testing.T, dir string, maxSegmentKeyCount uint32) *LittBlockConfig { +func strandingConfig(t *testing.T, dir string, maxSegmentKeyCount uint32) *BlockDBConfig { cfg, err := DefaultConfig(dir) require.NoError(t, err) - cfg.Retention = time.Nanosecond + cfg.RetentionTime = time.Nanosecond cfg.Litt.TargetSegmentFileSize = math.MaxUint32 cfg.Litt.MaxSegmentKeyCount = maxSegmentKeyCount cfg.Litt.GCPeriod = time.Hour diff --git a/sei-db/ledger_db/receipt/litt_receipt_gc.go b/sei-db/ledger_db/receipt/litt_receipt_gc.go new file mode 100644 index 0000000000..9ba6dee74a --- /dev/null +++ b/sei-db/ledger_db/receipt/litt_receipt_gc.go @@ -0,0 +1,53 @@ +package receipt + +import "github.com/sei-protocol/sei-chain/sei-db/management/gc" + +// littReceiptStore joins the shared prune cycle as a contiguous store: it holds every block from its +// retention floor up to its head, so any height in between can serve a rollback. +// +// Only the littidx backend participates. The pebble backend delegates to the state store, which +// prunes on its own KeepRecent schedule. +var _ gc.PrunableStore = (*littReceiptStore)(nil) + +func (s *littReceiptStore) Name() string { + return "ReceiptDB" +} + +// ExternalPruning reports config.ExternalPruning, the same value runsLocalPruner consults before +// starting the KeepRecent pruner. It is fixed at construction, and false for a store running without +// a collector. +func (s *littReceiptStore) ExternalPruning() bool { + return s.externalPruning +} + +// PruneHistory advances the retention floor to blockNumber and drops the tag-index entries below it. +// Receipt bodies are released to litt's GC rather than deleted here, and are reclaimed once they are +// also past the TTL (see gcFilter). Reads below the floor return not-found in the meantime. +func (s *littReceiptStore) PruneHistory(blockNumber uint64) error { + return s.pruneBlocksBelow(blockNumber) +} + +// PruneSnapshots does nothing: this store keeps no snapshots. +func (s *littReceiptStore) PruneSnapshots(uint64) error { + return nil +} + +// GetRollbackFloor returns head - rollbackWindow, measured against this store's own head. It returns +// 0 — nothing here is eligible for pruning — when the window is deeper than the whole history, which +// includes a store that has written nothing, or when the head cannot be read. +func (s *littReceiptStore) GetRollbackFloor(rollbackWindow uint64) uint64 { + head, err := s.GetLatestBlock() + if err != nil || head <= rollbackWindow { + return 0 + } + return head - rollbackWindow +} + +// GetLatestBlock returns the newest block whose receipts have been written, or 0 when none have. +func (s *littReceiptStore) GetLatestBlock() (uint64, error) { + latest := s.latestVersion.Load() + if latest <= 0 { + return 0, nil + } + return uint64(latest), nil //nolint:gosec // guarded non-negative above +} diff --git a/sei-db/ledger_db/receipt/litt_receipt_gc_test.go b/sei-db/ledger_db/receipt/litt_receipt_gc_test.go new file mode 100644 index 0000000000..0cb19d07a4 --- /dev/null +++ b/sei-db/ledger_db/receipt/litt_receipt_gc_test.go @@ -0,0 +1,230 @@ +package receipt_test + +import ( + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + storetypes "github.com/sei-protocol/sei-chain/sei-cosmos/store/types" + "github.com/sei-protocol/sei-chain/sei-cosmos/testutil" + sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" + dbconfig "github.com/sei-protocol/sei-chain/sei-db/config" + "github.com/sei-protocol/sei-chain/sei-db/ledger_db/receipt" + "github.com/sei-protocol/sei-chain/sei-db/management/gc" + "github.com/stretchr/testify/require" +) + +// setupLittIdxForGC opens a littidx store configured the way a collector-managed one actually is, +// so the only thing that moves the retention floor is the collector call under test. Note that this +// is the real mechanism rather than a test-only dodge: zeroing PruneIntervalSeconds would silence +// the local pruner just as well, and would not exercise the flag the collector depends on. +func setupLittIdxForGC(t *testing.T, keepRecent int) (receipt.ReceiptStore, gc.PrunableStore, sdk.Context) { + t.Helper() + storeKey := storetypes.NewKVStoreKey("evm") + tkey := storetypes.NewTransientStoreKey("evm_transient") + ctx := testutil.DefaultContext(storeKey, tkey).WithBlockHeight(1) + + cfg := dbconfig.DefaultReceiptStoreConfig() + cfg.Backend = "littidx" + cfg.DBDirectory = t.TempDir() + cfg.KeepRecent = keepRecent + cfg.ExternalPruning = true + + store, err := receipt.NewReceiptStore(cfg, storeKey) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + prunable, ok := store.(gc.PrunableStore) + require.True(t, ok, "a littidx receipt store must satisfy gc.PrunableStore") + return store, prunable, ctx +} + +// ExternalPruning is off unless asked for: a store built without a collector must keep pruning +// itself, since standing down with nothing to replace it grows the tag index without bound. +func TestReceiptGCExternalPruningDefaultsOff(t *testing.T) { + storeKey := storetypes.NewKVStoreKey("evm") + cfg := dbconfig.DefaultReceiptStoreConfig() + cfg.Backend = "littidx" + cfg.DBDirectory = t.TempDir() + require.False(t, cfg.ExternalPruning, "the default must leave retention with the store") + + store, err := receipt.NewReceiptStore(cfg, storeKey) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + prunable, ok := store.(gc.PrunableStore) + require.True(t, ok) + require.False(t, prunable.ExternalPruning()) + + _, managed, _ := setupLittIdxForGC(t, 0) + require.True(t, managed.ExternalPruning()) +} + +// The pebble backend is not a gc.PrunableStore, so the collector would never prune it and honoring +// ExternalPruning would leave it with no pruner at all. Refused at startup rather than discovered +// later from a full disk. +func TestReceiptExternalPruningRejectedOnPebbleBackend(t *testing.T) { + cfg := dbconfig.DefaultReceiptStoreConfig() + cfg.DBDirectory = t.TempDir() + cfg.ExternalPruning = true + + _, err := receipt.NewReceiptStore(cfg, storetypes.NewKVStoreKey("evm")) + require.ErrorContains(t, err, "does not support external pruning") +} + +// The standalone shape, and the one a collector-shaped change is most likely to break: with +// ExternalPruning unset there is no collector, and the store's own pruner has to advance the +// retention floor or the tag index grows without bound. Nodes running rs-backend = "littidx" with +// KeepRecent from min-retain-blocks depend on exactly this. +// +// Pays a real wait because the pruner is on a jittered timer and there is no way to observe it +// otherwise; TestRunsLocalPruner covers the decision itself without waiting. +func TestReceiptLocalPrunerAdvancesFloorWithoutCollector(t *testing.T) { + storeKey := storetypes.NewKVStoreKey("evm") + tkey := storetypes.NewTransientStoreKey("evm_transient") + ctx := testutil.DefaultContext(storeKey, tkey).WithBlockHeight(1) + + cfg := dbconfig.DefaultReceiptStoreConfig() + cfg.Backend = "littidx" + cfg.DBDirectory = t.TempDir() + cfg.KeepRecent = 2 + cfg.PruneIntervalSeconds = 1 // the shortest cadence there is; the pruner jitters to 1-2s + + store, err := receipt.NewReceiptStore(cfg, storeKey) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + addr := common.HexToAddress("0xabcd") + topic := common.HexToHash("0x1111") + for block := uint64(1); block <= 5; block++ { + writeLitBlock(t, store, ctx, block, litReceipt(block, 0, addr, topic)) + } + + // KeepRecent 2 at head 5 keeps [4, 5], so the floor lands on 4. + require.Eventually(t, func() bool { + return store.EarliestVersion() == 4 + }, 6*time.Second, 25*time.Millisecond, "the local pruner must advance the floor with no collector present") +} + +// KeepRecent belongs to the local pruner and must not reach the collector's answers: how deep the +// collector prunes is its own fleet-wide window. Real receipts are written first so the floor is a +// nonzero head - rollbackWindow: with an empty store GetRollbackFloor short-circuits on +// head <= rollbackWindow and returns 0 before KeepRecent could matter, so the empty case would pass +// even if the floor were computed from KeepRecent — exactly the bug this pins against (0 used to +// mean "keep everything" here and would once have suppressed pruning entirely). +func TestReceiptGCAnswersDoNotDependOnKeepRecent(t *testing.T) { + addr := common.HexToAddress("0xabcd") + topic := common.HexToHash("0x1111") + for _, keepRecent := range []int{0, 100_000} { + store, prunable, ctx := setupLittIdxForGC(t, keepRecent) + for block := uint64(1); block <= 10; block++ { + writeLitBlock(t, store, ctx, block, litReceipt(block, 0, addr, topic)) + } + require.Equal(t, uint64(7), prunable.GetRollbackFloor(3), + "keepRecent %d must not change the floor (head 10 - window 3)", keepRecent) + } +} + +// This store keeps no snapshots, so its half of the split contract is a no-op that must leave the +// retention floor where it is. +func TestReceiptGCPruneSnapshotsIsANoOp(t *testing.T) { + store, prunable, ctx := setupLittIdxForGC(t, 0) + addr := common.HexToAddress("0xabcd") + topic := common.HexToHash("0x1111") + for block := uint64(1); block <= 3; block++ { + writeLitBlock(t, store, ctx, block, litReceipt(block, 0, addr, topic)) + } + + require.NoError(t, prunable.PruneSnapshots(3)) + require.Equal(t, int64(0), store.EarliestVersion(), "a snapshot prune must not move the floor") + + kept, err := store.GetReceiptFromStore(ctx, litTxHash(1, 0)) + require.NoError(t, err) + require.Equal(t, uint64(1), kept.BlockNumber) +} + +// The head is 0 until receipts land, so a store that is still filling reports a floor of 0 and the +// fleet holds its history where it is rather than pruning against a head this store does not have. +func TestReceiptGCLatestBlock(t *testing.T) { + store, prunable, ctx := setupLittIdxForGC(t, 0) + + latest, err := prunable.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(0), latest, "a store that has ingested nothing has no head") + + addr := common.HexToAddress("0xabcd") + topic := common.HexToHash("0x1111") + writeLitBlock(t, store, ctx, 1, litReceipt(1, 0, addr, topic)) + writeLitBlock(t, store, ctx, 2, litReceipt(2, 0, addr, topic)) + writeLitBlock(t, store, ctx, 3, litReceipt(3, 0, addr, topic)) + + latest, err = prunable.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(3), latest) + require.Equal(t, int64(3), store.LatestVersion(), "the head reported to the collector must agree with the store's own version") +} + +// A contiguous store resolves the window against its own head, and the prune that follows moves the +// retention floor to the height it is given — which is what makes the receipts below it stop being +// served. Reclaiming their bodies lags that, since litt also waits for the TTL, but it can no +// longer lead it (see TestGCFilterMakesLittReclamationFollowTheBlockFloor). +func TestReceiptGCRollbackFloorAndPruneHistory(t *testing.T) { + store, prunable, ctx := setupLittIdxForGC(t, 0) + addr := common.HexToAddress("0xabcd") + topic := common.HexToHash("0x1111") + + // Holding nothing: the head is 0, so every window reports 0 — keep everything — which holds the + // fleet's history where it is until this store fills. + require.Equal(t, uint64(0), prunable.GetRollbackFloor(42)) + + for block := uint64(1); block <= 3; block++ { + writeLitBlock(t, store, ctx, block, litReceipt(block, 0, addr, topic)) + } + require.Equal(t, uint64(1), prunable.GetRollbackFloor(2)) + // A window deeper than its own head is a rollback promise reaching past genesis, so nothing here + // is eligible for pruning yet. + require.Equal(t, uint64(0), prunable.GetRollbackFloor(1_000)) + + require.NoError(t, prunable.PruneHistory(3)) + require.Equal(t, int64(3), store.EarliestVersion(), "PruneHistory must advance the retention floor") + + _, err := store.GetReceiptFromStore(ctx, litTxHash(1, 0)) + require.ErrorIs(t, err, receipt.ErrNotFound, "a receipt below the floor must not be served") + kept, err := store.GetReceiptFromStore(ctx, litTxHash(3, 0)) + require.NoError(t, err) + require.Equal(t, uint64(3), kept.BlockNumber) + + // The floor only advances: a later, lower cycle must not re-expose what was pruned. + require.NoError(t, prunable.PruneHistory(1)) + require.Equal(t, int64(3), store.EarliestVersion()) +} + +// PruneHistory carries a minimum taken across every managed store, so it can arrive above this +// store's head whenever this one lags or has ingested nothing. Both cases are the store's own to +// survive: this store's own floor of 0 makes them unlikely, but that is a property of the caller. +func TestReceiptGCPruneHistoryAboveHead(t *testing.T) { + addr := common.HexToAddress("0xabcd") + topic := common.HexToHash("0x1111") + + t.Run("empty store keeps its floor at zero", func(t *testing.T) { + store, prunable, _ := setupLittIdxForGC(t, 0) + + require.NoError(t, prunable.PruneHistory(1_000)) + require.Equal(t, int64(0), store.EarliestVersion(), + "a floor above an empty store would have to be walked back once blocks arrive") + }) + + t.Run("lagging store is capped at its head", func(t *testing.T) { + store, prunable, ctx := setupLittIdxForGC(t, 0) + for block := uint64(1); block <= 3; block++ { + writeLitBlock(t, store, ctx, block, litReceipt(block, 0, addr, topic)) + } + + require.NoError(t, prunable.PruneHistory(1_000)) + require.Equal(t, int64(3), store.EarliestVersion(), "the floor stops at the head, not the request") + + kept, err := store.GetReceiptFromStore(ctx, litTxHash(3, 0)) + require.NoError(t, err, "honoring the request literally would drop every block the store holds") + require.Equal(t, uint64(3), kept.BlockNumber) + }) +} diff --git a/sei-db/ledger_db/receipt/litt_receipt_gcfilter_internal_test.go b/sei-db/ledger_db/receipt/litt_receipt_gcfilter_internal_test.go new file mode 100644 index 0000000000..2b98ce7d16 --- /dev/null +++ b/sei-db/ledger_db/receipt/litt_receipt_gcfilter_internal_test.go @@ -0,0 +1,189 @@ +package receipt + +import ( + "testing" + "time" + + "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt" + "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/littbuilder" + "github.com/stretchr/testify/require" +) + +// floorStore returns a bare store whose retention floor is set, which is all gcFilter reads. +// Reclamation decisions are a pure function of that floor and the key, so nothing else is opened. +func floorStore(floor int64) *littReceiptStore { + s := &littReceiptStore{} + s.earliestVersion.Store(floor) + return s +} + +func requireFilter(t *testing.T, s *littReceiptStore, key []byte, isPrimary, want bool) { + t.Helper() + got, err := s.gcFilter(key, isPrimary) + require.NoError(t, err) + require.Equal(t, want, got) +} + +// The point of the filter: a body at or above the floor is not reclaimable however old it is, so a +// TTL sized for the wrong window cannot delete what reads still serve. +func TestGCFilterHoldsBodiesAtOrAboveTheFloor(t *testing.T) { + s := floorStore(100) + + requireFilter(t, s, littPartKey(99, 0), true, true) + requireFilter(t, s, littPartKey(100, 0), true, false) + requireFilter(t, s, littPartKey(101, 0), true, false) +} + +// Every part of a block shares its fate, since the floor is a block boundary and a partially +// reclaimed block would serve some of its receipts and not others. +func TestGCFilterTreatsEveryPartOfABlockAlike(t *testing.T) { + s := floorStore(100) + + for _, part := range []uint32{0, 1, 7} { + requireFilter(t, s, littPartKey(99, part), true, true) + requireFilter(t, s, littPartKey(100, part), true, false) + } +} + +// Tx-hash secondaries alias a body in their own segment, so the body's part key is what gates the +// segment. A secondary that blocked on its own would pin segments by tx hash, which carries no +// block number to compare against the floor. +func TestGCFilterPassesSecondaryKeys(t *testing.T) { + s := floorStore(100) + requireFilter(t, s, make([]byte, 32), false, true) +} + +// An unestablished floor must block rather than permit: 0 means nothing has been pruned yet, or +// the index has not been read, and reclaiming on that reading is not recoverable. +func TestGCFilterBlocksEverythingWithoutAFloor(t *testing.T) { + for _, floor := range []int64{0, -1} { + s := floorStore(floor) + requireFilter(t, s, littPartKey(0, 0), true, false) + requireFilter(t, s, littPartKey(1_000_000, 0), true, false) + } +} + +// litt requires the filter to be monotonic: once true for a key, always true. The floor only +// advances, so this holds — assert it across an advancing floor rather than trusting the field. +func TestGCFilterIsMonotonicAsTheFloorAdvances(t *testing.T) { + s := floorStore(0) + key := littPartKey(50, 0) + + var everTrue bool + for _, floor := range []int64{0, 10, 50, 51, 500} { + s.earliestVersion.Store(floor) + got, err := s.gcFilter(key, true) + require.NoError(t, err) + if everTrue { + require.True(t, got, "the filter went back on a key it already released at floor %d", floor) + } + everTrue = everTrue || got + } + require.True(t, everTrue, "an advancing floor must eventually release the key") +} + +// gcFilterTable builds a real litt table wired to s.gcFilter, sized so reclamation is observable in +// a unit test: segments seal on every write, and the TTL is already expired the moment a segment is +// sealed. That leaves the retention floor as the only thing deciding what may be reclaimed, which +// is the property under test. +// +// The receipt store's own segment thresholds (512MB) never seal in a test, so this drives litt +// directly rather than through newLittReceiptStore. What it cannot prove is that the constructor +// attaches the filter; the tests above and the wiring at the BuildTable call cover that separately. +func gcFilterTable(t *testing.T, s *littReceiptStore) litt.ManagedTable { + t.Helper() + + cfg, err := litt.DefaultConfig(t.TempDir()) + require.NoError(t, err) + cfg.TargetSegmentFileSize = 1 // seal on every write, so each block lands in a collectable segment + cfg.GCPeriod = time.Hour // only the explicit RunGC below collects, so nothing races a timer + cfg.Fsync = false + + db, err := littbuilder.NewDB(cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + + tableConfig := litt.DefaultTableConfig(littReceiptTableName) + tableConfig.ShardingFactor = 1 + tableConfig.TTL = time.Nanosecond // the age gate is satisfied at once; only the floor is left + tableConfig.GCFilter = s.gcFilter + table, err := db.BuildTable(tableConfig) + require.NoError(t, err) + + // RunGC lives on ManagedTable; without it a pass only happens on GCPeriod, which this test + // deliberately pushes out of reach so the assertions are not racing a timer. + managed, ok := table.(litt.ManagedTable) + require.True(t, ok, "forcing a GC pass requires a ManagedTable") + return managed +} + +func requireBlockRetained(t *testing.T, table litt.Table, block uint64) { + t.Helper() + ok, err := table.Exists(littPartKey(block, 0)) + require.NoError(t, err) + require.True(t, ok, "block %d was reclaimed while the retention floor still covered it", block) +} + +// Reclamation is not synchronous with a GC pass: the pass schedules keymap deletes and the control +// loop drops the files once they are durable. Poll rather than assert straight after RunGC. +func requireBlockReclaimed(t *testing.T, table litt.Table, block uint64) { + t.Helper() + require.Eventually(t, func() bool { + ok, err := table.Exists(littPartKey(block, 0)) + require.NoError(t, err) + return !ok + }, 5*time.Second, 10*time.Millisecond, "block %d stayed readable after the floor passed it", block) +} + +// The end of the chain this change exists to close: pruning by block number is what releases receipt +// bodies to litt, and an expired TTL alone does not. Without the filter every block here would be +// reclaimable from the very first pass, since the TTL expires the moment a segment seals. +// +// The two directions are not symmetric, and the test asserts them at different strengths on purpose. +// Retention is exact — a block at or above the floor must never be reclaimed, and that is the +// guarantee the collector depends on. Reclamation is only eventual and segment-granular: litt frees +// whole segments, so a segment holding one block above the floor keeps every block below it in that +// same segment. Blocks just under the floor may therefore survive a pass, which is why the +// reclaimed-side assertion leaves a margin rather than pinning the block after the floor. +func TestGCFilterMakesLittReclamationFollowTheBlockFloor(t *testing.T) { + const ( + head = 12 + floor = 10 + margin = 5 // comfortably below any segment that could straddle the floor + ) + + s := floorStore(0) + table := gcFilterTable(t, s) + + for block := uint64(1); block <= head; block++ { + require.NoError(t, table.Put(littPartKey(block, 0), []byte{byte(block)})) + } + require.NoError(t, table.Flush()) + + // No floor established and the TTL long expired: nothing may go. This is the case the filter + // exists for — every one of these is TTL-reclaimable and only the floor is holding it. + require.NoError(t, table.RunGC()) + for block := uint64(1); block <= head; block++ { + requireBlockRetained(t, table, block) + } + + // Pruning by block number is what releases them. + s.earliestVersion.Store(floor) + require.NoError(t, table.RunGC()) + + for block := uint64(1); block <= margin; block++ { + requireBlockReclaimed(t, table, block) + } + for block := uint64(floor); block <= head; block++ { + requireBlockRetained(t, table, block) + } +} + +// A primary key that is not a part key means the table's key layout changed without this filter +// being updated. Guessing a block number out of it would reclaim by coincidence, so litt's contract +// (an error crashes the DB loudly) is the right response. +func TestGCFilterRejectsAnUnknownPrimaryKey(t *testing.T) { + s := floorStore(100) + _, err := s.gcFilter(make([]byte, 32), true) + require.ErrorContains(t, err, "unexpected primary receipt key length") +} diff --git a/sei-db/ledger_db/receipt/litt_receipt_pruner_internal_test.go b/sei-db/ledger_db/receipt/litt_receipt_pruner_internal_test.go new file mode 100644 index 0000000000..f684e41a7a --- /dev/null +++ b/sei-db/ledger_db/receipt/litt_receipt_pruner_internal_test.go @@ -0,0 +1,70 @@ +package receipt + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// Which driver enforces retention is a four-way decision, and getting it wrong in either direction +// is a production bug rather than a test nicety: two pruners race to different floors, and none +// lets the tag index grow without bound. Enumerated here rather than observed through the jittered +// ticker so every combination is covered without waiting on any of them. +func TestRunsLocalPruner(t *testing.T) { + for _, tc := range []struct { + name string + externalPruning bool + keepRecent int64 + pruneInterval int64 + want bool + }{ + { + name: "standalone node prunes itself", + keepRecent: 100_000, + pruneInterval: 600, + want: true, + }, + { + name: "under the collector it stands down", + externalPruning: true, + keepRecent: 100_000, + pruneInterval: 600, + want: false, + }, + { + // KeepRecent 0 is the default and means keep everything, so there is nothing for a + // local pruner to do. It reaches none of the collector's answers either. + name: "keep everything", + keepRecent: 0, + pruneInterval: 600, + want: false, + }, + { + name: "no cadence configured", + keepRecent: 100_000, + pruneInterval: 0, + want: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + s := &littReceiptStore{ + externalPruning: tc.externalPruning, + keepRecent: tc.keepRecent, + pruneInterval: tc.pruneInterval, + } + require.Equal(t, tc.want, s.runsLocalPruner()) + }) + } +} + +// The invariant behind the table: with retention configured at all, the local pruner is exactly the +// negation of ExternalPruning. Both on means two pruners racing to different floors; both off means +// nothing advances the floor. Asserted against the method the collector calls, so it covers the two +// staying wired to each other and not merely today's values. +func TestRunsLocalPrunerIsTheNegationOfExternalPruning(t *testing.T) { + for _, external := range []bool{false, true} { + s := &littReceiptStore{externalPruning: external, keepRecent: 100_000, pruneInterval: 600} + require.Equal(t, !s.ExternalPruning(), s.runsLocalPruner(), + "exactly one of the collector and the local pruner may enforce retention") + } +} diff --git a/sei-db/ledger_db/receipt/litt_receipt_store.go b/sei-db/ledger_db/receipt/litt_receipt_store.go index 069bc127a5..7ff821ddd7 100644 --- a/sei-db/ledger_db/receipt/litt_receipt_store.go +++ b/sei-db/ledger_db/receipt/litt_receipt_store.go @@ -49,10 +49,17 @@ import ( // the most recent receipt bodies while the index still lists them — tolerable // for auxiliary, non-consensus RPC data, since reads return not-found for a // missing body. The tight interval is only affordable because litt flushes its -// keymap asynchronously off the control loop. Retention: receipt values -// expire via litt's per-table TTL (time based), tag keys are pruned by block -// range, and reads enforce the KeepRecent floor, so visible retention never -// exceeds KeepRecent regardless of GC timing. +// keymap asynchronously off the control loop. +// +// Retention: tag keys are pruned by block range, reads enforce the retention +// floor, and receipt bodies are reclaimed once they are both below that floor and +// older than litt's per-table TTL (see gcFilter). +// +// Exactly one driver enforces that retention, selected by cfg.ExternalPruning: +// +// - unset: the background pruner below keeps the last KeepRecent blocks. +// - set: the StorageGarbageCollector prunes through the gc.PrunableStore +// implementation in litt_receipt_gc.go, and startPruning stands down. type littReceiptStore struct { values litt.DB receipts litt.Table @@ -64,6 +71,7 @@ type littReceiptStore struct { keepRecent int64 pruneInterval int64 + externalPruning bool logFilterParallelism int stopBackground chan struct{} backgroundWg sync.WaitGroup @@ -87,12 +95,11 @@ const ( // control loop; without that, a per-block flush regresses write throughput // badly (≈-48% observed before the async keymap landed). littFlushInterval = 5 * time.Millisecond - // littTTLPerBlock converts the KeepRecent block count into litt's wall-clock - // TTL (KeepRecent * littTTLPerBlock). Set above Giga block times so the TTL - // over-retains; only a sustained block time above this would expire a body - // still inside the height-based KeepRecent window, which reads mask as - // not-found (the earliest-version floor is authoritative). - littTTLPerBlock = 2 * time.Second + // littRetentionTime is litt's per-table TTL: the failsafe minimum age before a + // receipt body may be reclaimed. It is a flat duration rather than a function of + // any block count, since how much history this store keeps is decided by the + // retention floor that gcFilter enforces. + littRetentionTime = time.Hour littPartCountLen = 4 ) @@ -106,6 +113,27 @@ func littPartKey(blockNumber uint64, part uint32) []byte { return key } +// gcFilter reports whether litt may reclaim key, making the retention floor a precondition so the +// per-table TTL can only ever reclaim what the floor has already released. +// +// Only primary part keys gate; tx-hash secondaries alias a body in their own segment, so the body's +// part key is what holds that segment back. A floor of 0 means none has been established yet and +// blocks everything. +func (s *littReceiptStore) gcFilter(key []byte, isPrimaryKey bool) (bool, error) { + if !isPrimaryKey { + return true, nil + } + if len(key) != blockNumLen+littPartCountLen { + return false, fmt.Errorf("unexpected primary receipt key length %d (want %d)", + len(key), blockNumLen+littPartCountLen) + } + floor := s.earliestVersion.Load() + if floor <= 0 { + return false, nil + } + return binary.BigEndian.Uint64(key[:blockNumLen]) < uint64(floor), nil //nolint:gosec // guarded positive above +} + func newLittReceiptStore(cfg dbconfig.ReceiptStoreConfig, storeKey sdk.StoreKey) (ReceiptStore, error) { if err := os.MkdirAll(cfg.DBDirectory, 0o750); err != nil { return nil, fmt.Errorf("failed to create receipt store directory: %w", err) @@ -128,18 +156,36 @@ func newLittReceiptStore(cfg dbconfig.ReceiptStoreConfig, storeKey sdk.StoreKey) if err != nil { return nil, fmt.Errorf("failed to open littdb: %w", err) } + // getLogs per-query block fan-out; non-positive config falls back to the default. + logFilterParallelism := cfg.LogFilterParallelism + if logFilterParallelism <= 0 { + logFilterParallelism = dbconfig.DefaultReceiptLogFilterParallelism + } + + // Built before its table, because the table's GC filter is a method on it. + s := &littReceiptStore{ + values: values, + storeKey: storeKey, + keepRecent: int64(cfg.KeepRecent), + pruneInterval: int64(cfg.PruneIntervalSeconds), + externalPruning: cfg.ExternalPruning, + logFilterParallelism: logFilterParallelism, + stopBackground: make(chan struct{}), + } + tableConfig := litt.DefaultTableConfig(littReceiptTableName) tableConfig.ShardingFactor = 1 // single shard: flushing one file is cheaper; sharding mainly helps across multiple disks + tableConfig.GCFilter = s.gcFilter receipts, err := values.BuildTable(tableConfig) if err != nil { _ = values.Close() return nil, fmt.Errorf("failed to open littdb receipts table: %w", err) } - if cfg.KeepRecent > 0 { - if err := receipts.SetTTL(time.Duration(cfg.KeepRecent) * littTTLPerBlock); err != nil { - _ = values.Close() - return nil, fmt.Errorf("failed to set littdb ttl: %w", err) - } + s.receipts = receipts + // A TTL is necessary for litt to collect at all, and gcFilter makes it insufficient on its own. + if err := receipts.SetTTL(littRetentionTime); err != nil { + _ = values.Close() + return nil, fmt.Errorf("failed to set littdb ttl: %w", err) } indexCfg := pebbledb.DefaultConfig() @@ -149,23 +195,8 @@ func newLittReceiptStore(cfg dbconfig.ReceiptStoreConfig, storeKey sdk.StoreKey) _ = values.Close() return nil, fmt.Errorf("failed to open receipt log index: %w", err) } + s.index = index - // getLogs per-query block fan-out; non-positive config falls back to the default. - logFilterParallelism := cfg.LogFilterParallelism - if logFilterParallelism <= 0 { - logFilterParallelism = dbconfig.DefaultReceiptLogFilterParallelism - } - - s := &littReceiptStore{ - values: values, - receipts: receipts, - index: index, - storeKey: storeKey, - keepRecent: int64(cfg.KeepRecent), - pruneInterval: int64(cfg.PruneIntervalSeconds), - logFilterParallelism: logFilterParallelism, - stopBackground: make(chan struct{}), - } s.latestVersion.Store(s.readMeta(receiptLatestVersionKey)) s.earliestVersion.Store(s.readMeta(receiptEarliestVersionKey)) s.startPruning() @@ -398,8 +429,16 @@ func (s *littReceiptStore) Close() error { return err } +// runsLocalPruner reports whether this store drives its own retention, which requires that the +// collector is not driving it and that both KeepRecent and PruneIntervalSeconds are set. +func (s *littReceiptStore) runsLocalPruner() bool { + return !s.ExternalPruning() && s.keepRecent > 0 && s.pruneInterval > 0 +} + +// startPruning starts the local retention pruner, which keeps the last keepRecent blocks. It does +// nothing unless runsLocalPruner reports true. func (s *littReceiptStore) startPruning() { - if s.keepRecent <= 0 || s.pruneInterval <= 0 { + if !s.runsLocalPruner() { return } s.backgroundWg.Add(1) @@ -425,10 +464,21 @@ func (s *littReceiptStore) startPruning() { }() } -// pruneBlocksBelow deletes the tag entries in [earliest, cutoff) and advances -// the retention floor. Receipt values are reclaimed independently by litt's TTL -// GC; the read-time floor keeps them invisible in the meantime. +// pruneBlocksBelow deletes the tag entries in [earliest, cutoff) and advances the retention floor. +// Receipt bodies are released to litt's GC rather than deleted here (see gcFilter), and the read-time +// floor keeps them invisible in the meantime. +// +// It is shared by both retention drivers, startPruning and the collector's PruneHistory. A cutoff +// above this store's head is capped at the head rather than honored. func (s *littReceiptStore) pruneBlocksBelow(cutoff uint64) error { + head := s.latestVersion.Load() + if head <= 0 { + return nil // nothing ingested, so nothing to drop + } + if cutoff > uint64(head) { //nolint:gosec // guarded positive above + cutoff = uint64(head) + } + floor := uint64(0) if earliest := s.earliestVersion.Load(); earliest > 0 { floor = uint64(earliest) //nolint:gosec // earliest is non-negative diff --git a/sei-db/ledger_db/receipt/receipt_store.go b/sei-db/ledger_db/receipt/receipt_store.go index 25bdeded24..378106a222 100644 --- a/sei-db/ledger_db/receipt/receipt_store.go +++ b/sei-db/ledger_db/receipt/receipt_store.go @@ -149,6 +149,11 @@ func newReceiptBackend(config dbconfig.ReceiptStoreConfig, storeKey sdk.StoreKey case receiptBackendLittIdx: return newLittReceiptStore(config, storeKey) case receiptBackendPebble: + // This backend is not a gc.PrunableStore, so honoring ExternalPruning would stop its own + // pruner and put nothing in its place. + if config.ExternalPruning { + return nil, fmt.Errorf("receipt store backend %q does not support external pruning; use %q", receiptBackendPebble, receiptBackendLittIdx) + } ssConfig := dbconfig.DefaultStateStoreConfig() ssConfig.DBDirectory = config.DBDirectory ssConfig.AsyncWriteBuffer = config.AsyncWriteBuffer diff --git a/sei-db/management/gc/api.go b/sei-db/management/gc/api.go deleted file mode 100644 index 89644074e0..0000000000 --- a/sei-db/management/gc/api.go +++ /dev/null @@ -1,124 +0,0 @@ -package gc - -// InfiniteRetentionWindow is the GetRetentionWindow value meaning "never prune this store": its -// cut line is forced to 0, so it is never asked for a boundary and never passed to PruneBelow. -// -// Note that 0 here means "no extra beyond the shared RollbackWindow", not "keep forever" as -// KeepRecent / MinRetainBlocks do elsewhere in this repo — hence the negative sentinel. -const InfiniteRetentionWindow int64 = -1 - -// CannotServeRollback is the GetPruningBoundary answer meaning "I hold nothing that can serve a -// rollback to any height": in practice, a snapshot store with no completed snapshot. A store that -// answers this is not idle — it will replay forward once its first snapshot lands, so pruning the -// others now can delete the range it will replay from. The collector therefore stops: -// -// RollbackWindow > 0 → the whole cycle is abandoned; nothing is pruned -// RollbackWindow == 0 → this store is ignored and the others are pruned -// -// 0 is a safe value for this because block heights start at 1 and a store is only asked when its -// own cut line is above 0, so a real boundary is always positive. It also puts the conservative -// answer on Go's zero value: a store that returns nothing meaningful stops pruning rather than -// silently dropping out of the decision. -// -// Two limits are worth knowing. RollbackWindow == 0 puts the cut line at the head itself, so -// contiguous stores are pruned to their head and a store that has not snapshotted yet loses that -// replay range outright — do not run it on a node whose snapshot stores are still bootstrapping. -// And this signal only reaches the collector if the store is asked, which happens only when its -// own cut line is above 0: a snapshot store with InfiniteRetentionWindow, or with a retention -// window deep enough to zero its cut line, is never asked and so cannot stop a cycle. The SC/SS -// retention contract below pins retention at 0 to keep them out of that case. -const CannotServeRollback uint64 = 0 - -// PrunableStore is a store whose old data may be dropped by the StorageGarbageCollector. -type PrunableStore interface { - // Name identifies the store for logs and errors. Duplicate names are allowed; the - // collector keys answers by index, not by name. - Name() string - - // PruneBelow may drop data for all blocks below blockNumber. The store may perform the - // deletion asynchronously. - PruneBelow(blockNumber uint64) error - - // GetRetentionWindow is how many extra blocks beyond the shared RollbackWindow this store - // needs to keep servable, where head is the min non-zero GetLatestBlock. Three answers: - // - // > 0 → extra history on top; cutLine = head - RollbackWindow - retention - // 0 → no extra history; cutLine = head - RollbackWindow - // -1 → never prune this store at all (see InfiniteRetentionWindow) - // - // RollbackWindow is shared so every store stays consistent under rollback. Retention is - // optional history on top, so a store can still serve queries after a rollback has consumed - // the whole window. - // - // This is an input to a shared minimum, not an independently applied per-store policy: - // because every participating store is pruned to the lowest boundary, one store's deep - // retention extends retention for all of them (see StorageGarbageCollector). - // - // By store kind: - // - Contiguous (blockDB, receiptDB, state WAL): -1 / 0 / positive as configured. - // - SC: 0. It only needs the shared rollback window for its snapshots. - // - SS: 0, including on an archive node. What the collector manages for SS is snapshot - // pruning, not state pruning, and those snapshots only need to cover RollbackWindow. SS - // keeps its own version history via KeepRecent, so making a node archival belongs there, - // not here. Returning -1 would freeze SS snapshots forever without retaining any extra - // state — the opposite trade from the one intended — and would also drop SS out of the - // shared minimum, per the warning below. - // - // Do not give a snapshot store InfiniteRetentionWindow expecting deeper history — it does the - // opposite. A store with no cut line is never asked for a boundary, so it drops out of the - // minimum and the contiguous store holding its replay range is pruned to its own cut line. - // With retention 0 the same store answers its oldest needed snapshot and pulls that range down - // with it. Concretely, with head 100_000, RollbackWindow 1_000, and a lone snapshot at 20_000: - // retention 0 holds the WAL at 20_000, while InfiniteRetentionWindow lets it go to 99_000 and - // leaves the snapshot unable to replay forward. Only the snapshot itself survives, so this is - // safe solely for a store that never needs another store's history — which is what the - // StorageGarbageCollector precondition requires. TestPruneInfiniteRetentionSnapshotStore pins - // both directions. - GetRetentionWindow() int64 - - // GetPruningBoundary returns the oldest block this store must keep in order to serve cutLine: - // - // > 0 → that boundary, which must never exceed cutLine - // CannotServeRollback → no snapshots available; abandons the cycle - // - // There is no third answer: a store either has a boundary to report or cannot serve a - // rollback at all. - // - // Contiguous stores can restore to any height they hold, so they return cutLine. Snapshot - // stores can restore only at a snapshot, so they return the newest completed snapshot at or - // below cutLine; when every snapshot sits above cutLine none of them can be dropped, and they - // return cutLine rather than their oldest snapshot. With no completed snapshot at all they - // return CannotServeRollback. - // - // A store holding nothing at or below cutLine — empty, already pruned to a higher floor, or - // restored by state sync above it — still returns cutLine. The PruneBelow that follows is a - // no-op, which is the point: it has nothing to delete and nothing to protect, since a - // contiguous store fills from its own ingest path rather than by replaying another store. - // CannotServeRollback would be wrong there, stalling the fleet until the head advanced a full - // RollbackWindow past that floor. - // - // Never answering above cutLine is what makes pruneHeight <= head - RollbackWindow hold by - // construction, since the collector takes a minimum across stores and does not clamp. A higher - // answer would raise that minimum; with no correctly answering retention-0 store to cap it, it - // can exceed the head and drop the store outright. - // - // Nothing in the collector enforces this — honoring it is the implementor's job. The collector - // cannot repair a bad answer, because cutLine is all it knows: substituting it would still prune - // past the snapshot a snapshot store needed, and refusing to prune would let one faulty store - // stall every other store's pruning. So this bound is load-bearing and unchecked. - // - // A snapshot write in flight need not be reserved, as snapshot creation is assumed to finish - // quickly. Never having produced one is the separate CannotServeRollback case. - GetPruningBoundary(cutLine uint64) uint64 - - // GetLatestBlock returns the highest block this store has ingested. - // - // 0 means "nothing ingested yet" and is excluded when computing the head, so a store still - // filling cannot drag the head — and every cut line with it — down to 0. That exclusion says - // nothing about whether the store may be pruned around: one that cannot serve a rollback - // still stops the cycle, via GetPruningBoundary. - // - // 0 does not mean "disabled". A store disabled for this node is not instantiated and never - // reaches the collector. - GetLatestBlock() (uint64, error) -} diff --git a/sei-db/management/gc/prunable_store.go b/sei-db/management/gc/prunable_store.go new file mode 100644 index 0000000000..cbf0c81f30 --- /dev/null +++ b/sei-db/management/gc/prunable_store.go @@ -0,0 +1,56 @@ +package gc + +// PrunableStore is a store whose old data may be dropped by the StorageGarbageCollector. +// +// There are two dimensions of garbage collection: snapshots, and history. Snapshots are a mechanism +// utilized to roll back the chain. History is what lets data stores answer queries about historical +// blocks. Not all stores have a concept of "history" (e.g. SC), and not all stores have "snapshots" +// (e.g. the state WAL). A store implements both methods regardless, returning nil from the dimension +// it does not have. +type PrunableStore interface { + // Name identifies the store in logs and errors. Duplicate names are allowed. + Name() string + + // PruneHistory may drop the per-block history below blockNumber, and may do so asynchronously. + // A store that keeps no history returns nil. + // + // blockNumber may fall outside the range this store holds, including above its own head. Such a + // request must be clamped to a no-op rather than emptying the store. + // + // Only called when ExternalPruning reports true. + PruneHistory(blockNumber uint64) error + + // PruneSnapshots may drop every snapshot strictly below blockNumber. A store that keeps no + // snapshots returns nil. + // + // blockNumber is never 0, and never above the block this store last returned from + // GetRollbackFloor. + // + // Only called when ExternalPruning reports true. + PruneSnapshots(blockNumber uint64) error + + // ExternalPruning reports whether this store's retention is the collector's to enforce: + // + // true → the collector prunes it, and any pruner inside the store stands down + // false → the store prunes itself, and receives neither PruneHistory nor PruneSnapshots + // + // A store with no pruner of its own returns true unconditionally. + ExternalPruning() bool + + // GetRollbackFloor returns the earliest block a rollback of rollbackWindow blocks may target. + // + // A store that keeps no snapshots returns head - rollbackWindow, every block in that range being + // restorable directly. A store that keeps snapshots returns the block number of its highest + // snapshot at or below head - rollbackWindow, or its lowest snapshot when every snapshot is + // above that block. + // + // It returns 0 — keep everything from block 0 up — when rollbackWindow is deeper than the + // store's own history, when a snapshot store holds no snapshot to name, or when the store cannot + // determine what it holds. A store must never return a block above what it can restore to; + // nothing clamps this answer. + GetRollbackFloor(rollbackWindow uint64) uint64 + + // GetLatestBlock returns the highest block this store has ingested, 0 when it has ingested + // nothing. It is the head GetRollbackFloor measures rollbackWindow against. + GetLatestBlock() (uint64, error) +} diff --git a/sei-db/management/gc/storage_garbage_collector.go b/sei-db/management/gc/storage_garbage_collector.go index 7dc50f3333..47dfe59147 100644 --- a/sei-db/management/gc/storage_garbage_collector.go +++ b/sei-db/management/gc/storage_garbage_collector.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "math" "strings" "sync" "time" @@ -13,51 +14,8 @@ import ( var logger = seilog.NewLogger("db", "gc") -// StorageGarbageCollector periodically prunes a set of PrunableStores. -// -// Every store shares config.RollbackWindow and may request extra history via -// GetRetentionWindow. Each cycle: -// -// 1. head = min non-zero GetLatestBlock across stores -// 2. per store: cutLine = head - RollbackWindow - GetRetentionWindow, skipping the store -// when that is 0 (infinite retention, or head still inside its own window) -// 3. ask GetPruningBoundary(cutLine): a positive answer votes, and CannotServeRollback -// abandons the cycle while RollbackWindow > 0 -// 4. pruneHeight = min of the positive answers; PruneBelow(pruneHeight) on every store -// that answered positively -// -// Step 4 prunes to the shared minimum rather than to each store's own boundary so that a -// retained snapshot stays restorable: contiguous stores must still hold the blocks that follow -// it. The side effect is one effective retention across the fleet — receiptDB retention 100_000 -// also keeps SC/SS snapshots that far back, even though those stores report retention 0. That -// uniform prune is an intentional Giga tradeoff, not independent per-store retention. -// -// Two properties do the safety work. -// -// Answers are bounded by the cut line they were given, so pruneHeight <= head - RollbackWindow -// holds by construction and the collector does not clamp. That yields the invariant: a prune cannot -// take away a rollback that was possible before it. Full headroom is not promised after a rollback -// has already consumed part of the window. -// -// The bound is trusted rather than checked. A check could only substitute cutLine for a bad answer, -// which for a snapshot store is itself too aggressive — the correct answer is the newest snapshot at -// or below cutLine, normally well below it — and refusing to prune instead would let one faulty -// store stall the fleet and grow disk without limit. Both remedies are worse than the bug they -// guard, so the bound stays a contract requirement on GetPruningBoundary. -// -// And a store that can serve no rollback at all is not merely skipped, it abandons the cycle. It -// will replay forward once its first snapshot lands, so pruning the others now could delete the -// range it replays from, and a store contributing no boundary is not covered by the minimum in -// step 4. See CannotServeRollback for the limits of that signal. -// -// Precondition: every store passed in is live. A store disabled for this node is not -// instantiated and never reaches the collector, so the set holds no permanently-empty member. -// -// PruneBelow failures are joined and do not skip later stores: permission-to-drop is not -// transactional, and one unhealthy store must not block pruning of the others. -// -// The type is a ticker around prune; the decision logic lives in prune for unit testing. -// Not safe for concurrent use (Close must be called exactly once). +// StorageGarbageCollector periodically prunes a set of PrunableStores which prune +// old snapshots/checkpoints and history data based on RollbackWindow and LookbackWindow setting. type StorageGarbageCollector struct { config *StorageGarbageCollectorConfig stores []PrunableStore @@ -66,9 +24,8 @@ type StorageGarbageCollector struct { wg sync.WaitGroup } -// NewStorageGarbageCollector starts a collector that prunes stores every config.PruneInterval -// until Close is called or ctx is cancelled. ctx and config are both required: run dereferences -// ctx on a background goroutine, where a nil would panic unrecoverably instead of surfacing here. +// NewStorageGarbageCollector starts a collector that prunes stores every config.PruneInterval until +// Close is called or ctx is cancelled. Both ctx and config are required. func NewStorageGarbageCollector( ctx context.Context, config *StorageGarbageCollectorConfig, @@ -119,153 +76,101 @@ func (s *StorageGarbageCollector) run() { } } -// prune runs one prune cycle. See StorageGarbageCollector for the decision rules. +// prune runs one prune cycle. Each cycle: +// +// 1. every store reports GetRollbackFloor(RollbackWindow) +// 2. snapshotCutLine = the minimum of those answers +// 3. historyCutLine = snapshotCutLine - LookbackWindow, or 0 when LookbackWindow is -1 +// 4. every store reporting ExternalPruning gets PruneSnapshots(snapshotCutLine), then +// PruneHistory(historyCutLine) func prune(config *StorageGarbageCollectorConfig, stores []PrunableStore) error { if len(stores) == 0 { return nil } - globalLatestBlock, err := getGlobalLatestBlock(stores) - if err != nil { - return err - } - if globalLatestBlock == 0 { - logger.Info("skipping pruning: no store has a latest block") - return nil - } - // Answers are positional so duplicate Name() values cannot mis-attribute one. decisions := make([]storeDecision, len(stores)) - var pruneHeight uint64 - blocked := false + snapshotCutLine := uint64(math.MaxUint64) for i, store := range stores { - retention := store.GetRetentionWindow() - cutLine := getCutLine(globalLatestBlock, config.RollbackWindow, retention) - if cutLine == 0 { - // Infinite retention, or head still inside this store's retain window: never asked. - continue - } - - decisions[i].cutLine = cutLine - decisions[i].boundary = store.GetPruningBoundary(cutLine) - if decisions[i].boundary == CannotServeRollback { - // One blocker is enough to abandon the cycle, but the rest are still asked rather than - // broken out of, so the blocked cycle logs every store's decision. These are the cycles - // an operator has to explain, and every blocker surfaces in the first one rather than - // one per cycle. RollbackWindow 0 waives the guarantee, so the blocker is ignored. - if config.RollbackWindow > 0 { - blocked = true - } - continue - } - if pruneHeight == 0 || decisions[i].boundary < pruneHeight { - pruneHeight = decisions[i].boundary - } + decisions[i].externalPruning = store.ExternalPruning() + decisions[i].floor = store.GetRollbackFloor(config.RollbackWindow) + snapshotCutLine = min(snapshotCutLine, decisions[i].floor) } - if blocked { - // Ahead of the pruneHeight check on purpose: every non-blocking store has already voted, so - // a usable pruneHeight is sitting here and must not be issued. decisionByStore names the - // blockers, rendering each as cannotServeRollback. - logger.Info("pruning blocked: a store cannot serve the rollback window yet", - "globalLatestBlock", globalLatestBlock, - "decisionByStore", describeDecisions(stores, decisions), - ) - return nil - } - if pruneHeight == 0 { - logger.Info("skipping pruning: no store reported a pruning boundary", - "globalLatestBlock", globalLatestBlock, - "decisionByStore", describeDecisions(stores, decisions), - ) - return nil - } + historyCutLine := getHistoryCutLine(snapshotCutLine, config.LookbackWindow) logger.Info("pruning stores", - "globalLatestBlock", globalLatestBlock, - "pruneHeight", pruneHeight, + "rollbackWindow", config.RollbackWindow, + "lookbackWindow", config.LookbackWindow, + "snapshotCutLine", snapshotCutLine, + "historyCutLine", historyCutLine, "decisionByStore", describeDecisions(stores, decisions), ) - var pruneErrs error + return pruneStores(stores, decisions, snapshotCutLine, historyCutLine) +} + +// pruneStores issues the cycle's deletions: snapshots below snapshotHeight and history below +// historyHeight. A height of 0 is skipped, and a store that prunes itself is left alone. +func pruneStores( + stores []PrunableStore, + decisions []storeDecision, + snapshotHeight uint64, + historyHeight uint64, +) error { + var errs error for i, store := range stores { - // Covers both a never-asked store and one that cannot serve a rollback. - if decisions[i].boundary == 0 { + if !decisions[i].externalPruning { continue } - if err := store.PruneBelow(pruneHeight); err != nil { - pruneErrs = errors.Join(pruneErrs, fmt.Errorf("failed to prune %s below %d: %w", store.Name(), pruneHeight, err)) + if snapshotHeight > 0 { + if err := store.PruneSnapshots(snapshotHeight); err != nil { + errs = errors.Join(errs, fmt.Errorf("failed to prune %s snapshots below %d: %w", + store.Name(), snapshotHeight, err)) + } + } + if historyHeight > 0 { + if err := store.PruneHistory(historyHeight); err != nil { + errs = errors.Join(errs, fmt.Errorf("failed to prune %s history below %d: %w", + store.Name(), historyHeight, err)) + } } } - return pruneErrs + return errs } -// storeDecision records what the collector asked one store and what it answered, for the prune -// log. cutLine == 0 means the store was never asked, which is what separates it from a store that -// was asked and answered 0. +// storeDecision records what one store answered, for the prune log. type storeDecision struct { - cutLine uint64 - boundary uint64 + floor uint64 + externalPruning bool } -// describeDecisions renders one entry per store, in store order, for the prune log. The three -// outcomes render differently on purpose — never asked, cannot serve a rollback, and reported a -// boundary — since collapsing them would cost exactly the distinction wanted when auditing a -// deletion after the fact. Each asked store carries the cut line its answer is a function of. +// describeDecisions renders one "name=floor" entry per store, in store order, for the prune log. A +// store that prunes itself is tagged selfPruned. func describeDecisions(stores []PrunableStore, decisions []storeDecision) string { var sb strings.Builder for i, store := range stores { if i > 0 { sb.WriteString(" ") } - switch { - case decisions[i].cutLine == 0: - fmt.Fprintf(&sb, "%s=notAsked", store.Name()) - case decisions[i].boundary == CannotServeRollback: - fmt.Fprintf(&sb, "%s=cannotServeRollback(cutLine=%d)", store.Name(), decisions[i].cutLine) - default: - fmt.Fprintf(&sb, "%s=%d(cutLine=%d)", store.Name(), decisions[i].boundary, decisions[i].cutLine) + if decisions[i].externalPruning { + fmt.Fprintf(&sb, "%s=%d", store.Name(), decisions[i].floor) + continue } + fmt.Fprintf(&sb, "%s=%d(selfPruned)", store.Name(), decisions[i].floor) } return sb.String() } -// getCutLine returns head - RollbackWindow - retention for retention >= 0. -// Returns 0 when retention < 0, when the combined window overflows uint64, or when -// head is still inside that window (unsigned subtraction must not wrap). -func getCutLine(globalLatestBlock uint64, rollbackWindow uint64, retention int64) uint64 { - if retention < 0 { - return 0 - } - totalRetainWindow := rollbackWindow + uint64(retention) - if totalRetainWindow < rollbackWindow { - // Addition wrapped; treat as an unsatisfiable retain window (skip pruning). - return 0 +// getHistoryCutLine returns the height history may be pruned below: snapshotCutLine less +// lookbackWindow. It returns 0 — keep everything — when lookbackWindow is -1 (infinite retention) +// or when the window reaches below genesis. +func getHistoryCutLine(snapshotCutLine uint64, lookbackWindow int64) uint64 { + if lookbackWindow < 0 { + return 0 // infinite retention } - if globalLatestBlock <= totalRetainWindow { + window := uint64(lookbackWindow) + if snapshotCutLine <= window { return 0 } - return globalLatestBlock - totalRetainWindow -} - -// getGlobalLatestBlock returns the smallest non-zero GetLatestBlock among stores, or 0 when no -// store has data. The minimum keeps a lagging store from being pruned past blocks it still needs. -// -// Heads of 0 are skipped rather than treated as "unknown". That necessarily excludes the store -// holding nothing from the minimum, so the minimum cannot protect it; what protects it is -// CannotServeRollback, plus the store-set precondition on StorageGarbageCollector. -func getGlobalLatestBlock(stores []PrunableStore) (uint64, error) { - var blockNum uint64 - for _, store := range stores { - storeHeight, err := store.GetLatestBlock() - if err != nil { - return 0, fmt.Errorf("failed to read latest block from %s: %w", store.Name(), err) - } - if storeHeight == 0 { - continue - } - if blockNum == 0 || storeHeight < blockNum { - blockNum = storeHeight - } - } - return blockNum, nil + return snapshotCutLine - window } diff --git a/sei-db/management/gc/storage_garbage_collector_config.go b/sei-db/management/gc/storage_garbage_collector_config.go index 0346bf8f3a..b9131da242 100644 --- a/sei-db/management/gc/storage_garbage_collector_config.go +++ b/sei-db/management/gc/storage_garbage_collector_config.go @@ -6,46 +6,48 @@ import ( ) // StorageGarbageCollectorConfig configures a StorageGarbageCollector. +// +// With F = LatestBlock - RollbackWindow - LookbackWindow, storage garbage collection guarantees the +// following invariants: +// +// 1. Garbage collection will not delete any data that is necessary to roll back to any block +// between LatestBlock and (LatestBlock - RollbackWindow), inclusive. +// 2. Garbage collection will not delete data at or after F. This ensures that even if the system +// rolls back to block (LatestBlock - RollbackWindow), it is still possible to read any of the +// LookbackWindow blocks below that point. +// 3. Garbage collection will eventually delete data older than F. +// +// A LookbackWindow of -1 puts F at genesis, so no history is ever deleted. type StorageGarbageCollectorConfig struct { - // RollbackWindow is how many blocks behind head the system must remain able to - // roll back to. Shared by every managed store so they stay mutually consistent; - // per-store extras beyond this window use PrunableStore.GetRetentionWindow. + // RollbackWindow is how many blocks behind head the system must remain able to roll back to. // - // 0 is allowed and waives the guarantee: cutLine then equals head for any store with - // retention 0, so those stores are pruned to their head. It is safe only when every - // participating store leaves enough history on its own — a positive GetRetentionWindow, - // or snapshot answers that hold the contiguous stores above genesis. It also disables - // the CannotServeRollback stop signal, so a store that has not snapshotted yet loses the - // range it would have replayed from. See that constant. - // - // After a rollback that consumes part of the window, full headroom is not promised - // (e.g. window 10_000, roll back 5_000 → only ~5_000 of headroom remain). + // 0 waives the guarantee: each store then reports its own head as the earliest height a + // rollback may target, or its newest snapshot if it keeps snapshots. RollbackWindow uint64 + // LookbackWindow is how much queryable history is kept below the rollback window, in blocks. + // It is extra on top of RollbackWindow rather than a total that includes it, so at least + // LookbackWindow blocks stay readable below wherever a rollback lands. + // + // -1 means infinite: history is never pruned, though snapshots below the rollback floor are + // still reclaimed. Every other value is a block count and must be >= 0. + LookbackWindow int64 + // PruneInterval is how often the collector runs a prune cycle. Must be > 0. PruneInterval time.Duration } // DefaultStorageGarbageCollectorConfig returns the default collector config. -// -// Both values are deliberately lower than the pre-unification defaults (RollbackWindow 10_000, -// interval 60s), to match how these are actually used: -// -// - RollbackWindow 1_000. Real rollbacks are a few blocks deep, so 1_000 is already ample -// headroom, and this is only the correctness floor. The production shape is a short -// rollback window paired with much longer history, and that history now belongs on each -// store via GetRetentionWindow rather than being folded into this shared window — keeping -// it short here avoids charging every store for retention only some of them need. -// - PruneInterval 5m. Pruning reclaims whole files lazily; running it every minute buys -// nothing and costs I/O against a live node. Anything in the 5-10 minute range is fine. func DefaultStorageGarbageCollectorConfig() *StorageGarbageCollectorConfig { return &StorageGarbageCollectorConfig{ RollbackWindow: 1_000, + LookbackWindow: 0, PruneInterval: 5 * time.Minute, } } -// Validate checks that required fields are set to usable values. +// Validate checks that required fields are set to usable values. LookbackWindow must be >= 0, or -1 +// for infinite retention. func (c *StorageGarbageCollectorConfig) Validate() error { if c == nil { return fmt.Errorf("config is required") @@ -53,5 +55,8 @@ func (c *StorageGarbageCollectorConfig) Validate() error { if c.PruneInterval <= 0 { return fmt.Errorf("prune interval must be greater than 0") } + if c.LookbackWindow < -1 { + return fmt.Errorf("lookback window must be >= 0, or -1 for infinite retention (got %d)", c.LookbackWindow) + } return nil } diff --git a/sei-db/management/gc/storage_garbage_collector_test.go b/sei-db/management/gc/storage_garbage_collector_test.go index 89d562179c..9a8d802589 100644 --- a/sei-db/management/gc/storage_garbage_collector_test.go +++ b/sei-db/management/gc/storage_garbage_collector_test.go @@ -15,37 +15,40 @@ import ( type mockStore struct { name string latestHeight uint64 - // retentionWindow is returned by GetRetentionWindow (0 = shared RollbackWindow only). - retentionWindow int64 - pruningBoundary func(cutLine uint64) uint64 - getErr error - pruneErr error - - pruneBelowCalled atomic.Bool - prunedBelow atomic.Uint64 - pruneBelowCalls atomic.Uint64 - boundaryCalls atomic.Uint64 -} - -// snapshotStore models SC/SS (retention 0). GetPruningBoundary returns the newest snapshot -// ≤ cutLine, or cutLine when every snapshot is above it (nothing can be dropped, and the -// contract forbids answering above cutLine). No snapshots at all → CannotServeRollback. -// Snapshots must be ascending. + floor func(rollbackWindow uint64) uint64 + pruneErr error + // selfPruned inverts ExternalPruning: the constructors below build collector-managed stores, + // which is the common case, so opting out is what a test has to say explicitly. + selfPruned bool + + historyPruned atomic.Bool + prunedHistoryBelow atomic.Uint64 + historyPruneCalls atomic.Uint64 + + snapshotsPruned atomic.Bool + prunedSnapshotsBelow atomic.Uint64 + + floorCalls atomic.Uint64 +} + +// snapshotStore models SC/SS, mirroring flatkv's snapshotFloor: the newest snapshot at or below its +// own head less the window, or its oldest snapshot when every one of them is above that height, and 0 +// when it holds no snapshot or the window is deeper than its head. Snapshots must be ascending. func snapshotStore(name string, latestHeight uint64, snapshots ...uint64) *mockStore { return &mockStore{ - name: name, - latestHeight: latestHeight, - retentionWindow: 0, - pruningBoundary: func(cutLine uint64) uint64 { - if len(snapshots) == 0 { - return CannotServeRollback + name: name, + latestHeight: latestHeight, + floor: func(rollbackWindow uint64) uint64 { + if len(snapshots) == 0 || latestHeight <= rollbackWindow { + return 0 } - if snapshots[0] > cutLine { - return cutLine + target := latestHeight - rollbackWindow + if snapshots[0] > target { + return snapshots[0] } newest := snapshots[0] for _, snapshot := range snapshots { - if snapshot > cutLine { + if snapshot > target { break } newest = snapshot @@ -55,45 +58,56 @@ func snapshotStore(name string, latestHeight uint64, snapshots ...uint64) *mockS } } -// contiguousStore models blockDB / receiptDB / WAL (retention 0 by default): it can restore to -// any height it holds, so it always answers cutLine — including when it holds nothing at or below -// cutLine, where the resulting PruneBelow is simply a no-op. Use withRetentionWindow for extra -// retention or InfiniteRetentionWindow. +// contiguousStore models blockDB / receiptDB / WAL: it can restore to any height it holds, so it +// answers its own head less the window — and 0 where the window is deeper than that head, which +// includes the empty store and holds both cut lines at 0. func contiguousStore(name string, latestHeight uint64) *mockStore { return &mockStore{ - name: name, - latestHeight: latestHeight, - retentionWindow: 0, - pruningBoundary: func(cutLine uint64) uint64 { return cutLine }, + name: name, + latestHeight: latestHeight, + floor: func(rollbackWindow uint64) uint64 { + if latestHeight <= rollbackWindow { + return 0 + } + return latestHeight - rollbackWindow + }, } } -func withRetentionWindow(store *mockStore, retention int64) *mockStore { - store.retentionWindow = retention +// withSelfPruning marks a store as enforcing its own retention, so the collector must count its +// answer but never prune it. +func withSelfPruning(store *mockStore) *mockStore { + store.selfPruned = true return store } +func (m *mockStore) ExternalPruning() bool { + return !m.selfPruned +} + func (m *mockStore) Name() string { return m.name } -func (m *mockStore) GetRetentionWindow() int64 { - return m.retentionWindow +func (m *mockStore) GetLatestBlock() (uint64, error) { + return m.latestHeight, nil } -func (m *mockStore) GetLatestBlock() (uint64, error) { - return m.latestHeight, m.getErr +func (m *mockStore) GetRollbackFloor(rollbackWindow uint64) uint64 { + m.floorCalls.Add(1) + return m.floor(rollbackWindow) } -func (m *mockStore) GetPruningBoundary(cutLine uint64) uint64 { - m.boundaryCalls.Add(1) - return m.pruningBoundary(cutLine) +func (m *mockStore) PruneHistory(blockNumber uint64) error { + m.historyPruned.Store(true) + m.prunedHistoryBelow.Store(blockNumber) + m.historyPruneCalls.Add(1) + return m.pruneErr } -func (m *mockStore) PruneBelow(blockNumber uint64) error { - m.pruneBelowCalled.Store(true) - m.prunedBelow.Store(blockNumber) - m.pruneBelowCalls.Add(1) +func (m *mockStore) PruneSnapshots(blockNumber uint64) error { + m.snapshotsPruned.Store(true) + m.prunedSnapshotsBelow.Store(blockNumber) return m.pruneErr } @@ -105,41 +119,47 @@ func prunableStores(list ...*mockStore) []PrunableStore { return result } -func testConfig(t *testing.T, rollbackWindow uint64) *StorageGarbageCollectorConfig { +func testConfig(t *testing.T, rollbackWindow uint64, lookbackWindow int64) *StorageGarbageCollectorConfig { t.Helper() config := &StorageGarbageCollectorConfig{ RollbackWindow: rollbackWindow, + LookbackWindow: lookbackWindow, PruneInterval: time.Minute, } require.NoError(t, config.Validate()) return config } -// TestPruneDecisions covers one prune cycle. wantPruneBelow == nil means no store is pruned. +// TestPruneDecisions covers one prune cycle. wantHistoryBelow == nil means no store is touched at +// all; otherwise every collector-managed store is pruned to that height. The two are separate +// because a cycle can reach one cut line and not the other: the lookback window can take the history +// cut line below genesis while snapshots are still prunable. // -// wantPruned is hardcoded per store rather than derived from getCutLine / GetPruningBoundary: deriving it from the -// helpers under test would let a sign error or off-by-one shift the expectation in lockstep with the bug. +// Expectations are hardcoded per case rather than derived from getHistoryCutLine / +// GetRollbackFloor: deriving them from the code under test would let a sign error or off-by-one +// shift the expectation in lockstep with the bug. func TestPruneDecisions(t *testing.T) { cases := []struct { - name string - rollbackWindow uint64 - stores []*mockStore - wantPruneBelow *uint64 - wantPruned []bool + name string + rollbackWindow uint64 + lookbackWindow int64 + stores []*mockStore + wantSnapshotsBelow *uint64 + wantHistoryBelow *uint64 }{ { - name: "SC and WAL both retention 0: min boundary wins", + name: "SC and WAL: min floor wins", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 100_000, 80_000, 85_000, 92_000), contiguousStore("stateWAL", 100_000), }, - // cutLine 90_000; sc keeps snapshot 85_000; WAL answers 90_000 → min 85_000. - wantPruneBelow: ptr(85_000), - wantPruned: []bool{true, true}, + // sc keeps snapshot 85_000; WAL answers 90_000; min 85_000, no lookback to subtract. + wantSnapshotsBelow: ptr(85_000), + wantHistoryBelow: ptr(85_000), }, { - name: "lowest boundary across many stores wins", + name: "lowest floor across many stores wins", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 100_000, 80_000, 85_000, 92_000), @@ -148,185 +168,228 @@ func TestPruneDecisions(t *testing.T) { contiguousStore("stateWAL", 100_000), }, // sc 85_000, ss 88_000, flatKV 90_000, WAL 90_000 → min 85_000. - wantPruneBelow: ptr(85_000), - wantPruned: []bool{true, true, true, true}, + wantSnapshotsBelow: ptr(85_000), + wantHistoryBelow: ptr(85_000), }, { - name: "RollbackWindow of 1 still leaves one block of margin", - rollbackWindow: 1, + // The lookback window deepens history without deepening the snapshot depth: restore + // points are only needed as far back as the rollback promise. + name: "lookback window deepens history only", + rollbackWindow: 10_000, + lookbackWindow: 5_000, stores: []*mockStore{ - snapshotStore("sc", 100_000, 80_000, 99_999), + snapshotStore("sc", 100_000, 95_000), contiguousStore("stateWAL", 100_000), }, - wantPruneBelow: ptr(99_999), - wantPruned: []bool{true, true}, + // sc's only snapshot is above head - 10_000, so it answers that snapshot, 95_000; WAL + // 90_000; min 90_000, and history goes 5_000 deeper. + wantSnapshotsBelow: ptr(90_000), + wantHistoryBelow: ptr(85_000), }, { - name: "RollbackWindow 0: cutLine equals head", - rollbackWindow: 0, + // The lookback window is subtracted from the minimum however deep that minimum already + // is, so it stacks below a snapshot floor rather than being capped by the head. + name: "lookback window stacks below the lowest floor", + rollbackWindow: 10_000, + lookbackWindow: 5_000, stores: []*mockStore{ - snapshotStore("sc", 100_000, 80_000, 90_000), + snapshotStore("sc", 100_000, 60_000), contiguousStore("stateWAL", 100_000), }, - // cutLine 100_000; sc 90_000; WAL 100_000 → min 90_000. - wantPruneBelow: ptr(90_000), - wantPruned: []bool{true, true}, + // sc holds a snapshot at 60_000, which binds; 5_000 blocks of lookback go below it. + wantSnapshotsBelow: ptr(60_000), + wantHistoryBelow: ptr(55_000), }, { - name: "positive contiguous retention deepens that store's cut line", - rollbackWindow: 10_000, + name: "lookback window alone bounds a contiguous fleet", + rollbackWindow: 1_000, + lookbackWindow: 50_000, stores: []*mockStore{ - snapshotStore("sc", 100_000, 80_000, 90_000), - withRetentionWindow(contiguousStore("stateWAL", 100_000), 5_000), + contiguousStore("blockDB", 100_000), + contiguousStore("receiptDB", 100_000), }, - // sc cutLine 90_000 → 90_000; WAL cutLine 85_000 → 85_000 → min 85_000. - wantPruneBelow: ptr(85_000), - wantPruned: []bool{true, true}, + // Both answer 99_000; history goes 50_000 deeper. + wantSnapshotsBelow: ptr(99_000), + wantHistoryBelow: ptr(49_000), }, { - name: "SS retention 0 behaves like SC", - rollbackWindow: 10_000, + name: "RollbackWindow of 1 still leaves one block of margin", + rollbackWindow: 1, stores: []*mockStore{ - snapshotStore("ss", 100_000, 80_000, 90_000), + snapshotStore("sc", 100_000, 80_000, 99_999), contiguousStore("stateWAL", 100_000), }, - wantPruneBelow: ptr(90_000), - wantPruned: []bool{true, true}, - }, - { - name: "infinite retention on every store skips pruning", - rollbackWindow: 10_000, - stores: []*mockStore{ - withRetentionWindow(contiguousStore("blockDB", 100_000), InfiniteRetentionWindow), - withRetentionWindow(contiguousStore("receiptDB", 100_000), InfiniteRetentionWindow), - }, - wantPruneBelow: nil, - wantPruned: []bool{false, false}, + wantSnapshotsBelow: ptr(99_999), + wantHistoryBelow: ptr(99_999), }, { - name: "infinite retention on one store leaves others free to prune", - rollbackWindow: 10_000, + name: "both windows 0: nothing is held back but the snapshot", + rollbackWindow: 0, stores: []*mockStore{ - withRetentionWindow(contiguousStore("archiveWAL", 100_000), InfiniteRetentionWindow), - snapshotStore("sc", 100_000, 80_000), + snapshotStore("sc", 100_000, 80_000, 90_000), contiguousStore("stateWAL", 100_000), }, - // archiveWAL skipped; sc 80_000; stateWAL 90_000 → min 80_000. - wantPruneBelow: ptr(80_000), - wantPruned: []bool{false, true, true}, + // sc 90_000; WAL 100_000 → min 90_000. + wantSnapshotsBelow: ptr(90_000), + wantHistoryBelow: ptr(90_000), }, { - name: "snapshot exactly at the cut line is kept", + name: "snapshot exactly at the window depth is kept", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 100_000, 50_000, 90_000), contiguousStore("stateWAL", 100_000), }, - wantPruneBelow: ptr(90_000), - wantPruned: []bool{true, true}, + wantSnapshotsBelow: ptr(90_000), + wantHistoryBelow: ptr(90_000), }, { - name: "all snapshots above cut line: store votes the cut line", + // The shortfall case: SC cannot restore as deep as the window asks, so it answers its + // oldest snapshot to keep that one replayable rather than waiving its claim. + name: "all snapshots above the window: store answers its oldest", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 100_000, 95_000, 97_000), contiguousStore("stateWAL", 100_000), }, - // Both answer 90_000 → shared prune 90_000 (sc's snapshots sit above it, untouched). - wantPruneBelow: ptr(90_000), - wantPruned: []bool{true, true}, + // sc 95_000; WAL 90_000 → min 90_000. + wantSnapshotsBelow: ptr(90_000), + wantHistoryBelow: ptr(90_000), }, { - // Regression: with no contiguous store to bound the min, an answer above cutLine - // would become pruneHeight and delete inside the rollback window. - name: "all snapshots above cut line with no contiguous store", + // The shortfall alone, with no contiguous store to bind the minimum. 95_000 sits above + // head - RollbackWindow, and nothing caps it: the store cannot restore below its oldest + // snapshot, so history below that buys a rollback no store could serve anyway. + name: "a shortfall answer alone sets the cut lines", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 100_000, 95_000, 97_000), }, - wantPruneBelow: ptr(90_000), - wantPruned: []bool{true}, + wantSnapshotsBelow: ptr(95_000), + wantHistoryBelow: ptr(95_000), }, { - name: "lagging store lowers the global head", + // Each store measures the window against its own head, so the WAL answers 90_000 off + // its own 100_000 while the lagging store answers 30_000 off its 50_000. The lagging + // store cannot be pruned past what it holds. + name: "each store answers from its own head", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("lagging", 50_000, 30_000, 50_000), contiguousStore("stateWAL", 100_000), }, - // head 50_000 → cutLine 40_000; lagging answers 30_000; WAL 40_000 → min 30_000. - wantPruneBelow: ptr(30_000), - wantPruned: []bool{true, true}, + wantSnapshotsBelow: ptr(30_000), + wantHistoryBelow: ptr(30_000), }, { - // A store with no snapshot will replay forward once its first one lands, so pruning - // the WAL to its own cut line now would delete the range it replays from. - name: "store with no snapshot blocks the whole cycle", + // A snapshot store with nothing to replay from cannot say which blocks the WAL may + // drop, so it answers 0 and the whole fleet stays where it is. + name: "store with no snapshot holds both cut lines at 0", rollbackWindow: 10_000, stores: []*mockStore{ snapshotStore("sc", 100_000), contiguousStore("stateWAL", 100_000), }, - wantPruneBelow: nil, - wantPruned: []bool{false, false}, }, { - // Same store set, blocker last. This ordering is the one that matters: the WAL votes - // 90_000, so a usable pruneHeight is sitting there when the loop ends and only the - // blocked check stops it from being issued. Ordering the blocker first would leave - // pruneHeight at 0, and the cycle would stall on that alone. - name: "blocker after a store that already voted", + // A store that has ingested nothing is the same case: it answers 0, the minimum carries + // that through, and nothing is deleted anywhere. + name: "empty store holds both cut lines at 0", rollbackWindow: 10_000, stores: []*mockStore{ + contiguousStore("blockDB", 0), contiguousStore("stateWAL", 100_000), - snapshotStore("sc", 100_000), }, - wantPruneBelow: nil, - wantPruned: []bool{false, false}, }, { - // Same store set, but RollbackWindow 0 waives the guarantee, so there is nothing - // left for the snapshot-less store to protect and the others prune. - name: "no snapshot does not block when RollbackWindow is 0", - rollbackWindow: 0, + // A snapshot on disk does not make a store prunable-around while its head is behind the + // window: it answers from the head, so a stalled store holds everything back. + name: "stalled store with a snapshot still answers 0", + rollbackWindow: 10_000, stores: []*mockStore{ - snapshotStore("sc", 100_000), + snapshotStore("stalled", 0, 50_000), contiguousStore("stateWAL", 100_000), }, - wantPruneBelow: ptr(100_000), - wantPruned: []bool{false, true}, }, { - name: "zero head ignored for global head; store still votes", + // Every store owes a rollback deeper than its whole history, so every one answers 0. + name: "head inside the rollback window: nothing deleted", rollbackWindow: 10_000, stores: []*mockStore{ - snapshotStore("stalled", 0, 50_000), + snapshotStore("sc", 5_000, 1_000, 2_000), + contiguousStore("stateWAL", 5_000), + }, + }, + { + // The one case where the cycle reaches one cut line and not the other: the rollback + // promise is satisfiable, so snapshots below 10_000 go, but the lookback window reaches + // below genesis from there and history is left alone. + name: "lookback window reaches below genesis: snapshots only", + rollbackWindow: 1_000, + lookbackWindow: 90_000, + stores: []*mockStore{ + snapshotStore("sc", 50_000, 10_000), + contiguousStore("stateWAL", 50_000), + }, + wantSnapshotsBelow: ptr(10_000), + }, + { + // -1 is infinite history retention: snapshots below the rollback floor are still + // reclaimed, but history is never pruned however far that floor advances. + name: "infinite lookback: snapshots only, history untouched", + rollbackWindow: 1_000, + lookbackWindow: -1, + stores: []*mockStore{ + snapshotStore("sc", 100_000, 10_000), contiguousStore("stateWAL", 100_000), }, - // head from WAL 100_000; stalled still answers 50_000 → min 50_000. - wantPruneBelow: ptr(50_000), - wantPruned: []bool{true, true}, + wantSnapshotsBelow: ptr(10_000), + }, + { + // Infinite lookback must not force a deletion on a chain younger than its rollback + // window. Every store owes a rollback deeper than its whole history, answers 0, and the + // minimum holds both cut lines there — the -1 that zeroes the history cut line finds it + // already 0, so nothing is pruned anywhere. + name: "infinite lookback on a young chain: nothing deleted", + rollbackWindow: 1_000, + lookbackWindow: -1, + stores: []*mockStore{ + snapshotStore("sc", 100, 50), + contiguousStore("blockDB", 100), + contiguousStore("stateWAL", 100), + }, }, { - name: "head inside retain window: no prune", + // A fleet that keeps no snapshots — blockDB, receiptDB, WAL — still has a snapshot cut + // line: the collector issues PruneSnapshots and each store no-ops it. Infinite lookback + // holds every block of history above genesis. + name: "infinite lookback on a contiguous fleet: history untouched", rollbackWindow: 10_000, + lookbackWindow: -1, stores: []*mockStore{ - snapshotStore("sc", 5_000, 1_000, 2_000), - contiguousStore("stateWAL", 5_000), + contiguousStore("blockDB", 100_000), + contiguousStore("receiptDB", 100_000), + contiguousStore("stateWAL", 100_000), }, - wantPruneBelow: nil, - wantPruned: []bool{false, false}, + // All three answer 90_000; the snapshot cut line fires as a no-op on stores that hold + // none, and history is never pruned. + wantSnapshotsBelow: ptr(90_000), }, { - name: "head inside one store's window skips only that store", - rollbackWindow: 60_000, + // The mixed fleet the collector actually runs: SC holds the snapshots and binds the + // snapshot cut line, while blockDB, receiptDB and the WAL answer from their own heads. + // Infinite lookback keeps all history across every one of them. + name: "infinite lookback across SC, blockDB, receiptDB and WAL", + rollbackWindow: 10_000, + lookbackWindow: -1, stores: []*mockStore{ - snapshotStore("sc", 100_000, 500, 1_000), - withRetentionWindow(contiguousStore("stateWAL", 100_000), 40_000), + snapshotStore("sc", 100_000, 50_000), + contiguousStore("blockDB", 100_000), + contiguousStore("receiptDB", 100_000), + contiguousStore("stateWAL", 100_000), }, - // WAL cutLine 0 (skipped); sc cutLine 40_000 → 1_000. - wantPruneBelow: ptr(1_000), - wantPruned: []bool{true, false}, + // sc 50_000; the contiguous three 90_000 → min 50_000. History is never pruned. + wantSnapshotsBelow: ptr(50_000), }, { name: "no store has a latest block", @@ -335,78 +398,119 @@ func TestPruneDecisions(t *testing.T) { snapshotStore("sc", 0), contiguousStore("stateWAL", 0), }, - wantPruneBelow: nil, - wantPruned: []bool{false, false}, }, { name: "no stores at all", rollbackWindow: 10_000, stores: nil, - wantPruneBelow: nil, - wantPruned: nil, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - require.Len(t, tc.wantPruned, len(tc.stores), "wantPruned must have one entry per store") - - require.NoError(t, prune(testConfig(t, tc.rollbackWindow), prunableStores(tc.stores...))) + config := testConfig(t, tc.rollbackWindow, tc.lookbackWindow) + require.NoError(t, prune(config, prunableStores(tc.stores...))) + + for _, store := range tc.stores { + if tc.wantSnapshotsBelow == nil { + require.Falsef(t, store.snapshotsPruned.Load(), "%s snapshots should not be pruned", store.name) + } else { + require.Truef(t, store.snapshotsPruned.Load(), "%s snapshots should be pruned", store.name) + require.Equalf(t, *tc.wantSnapshotsBelow, store.prunedSnapshotsBelow.Load(), + "%s snapshot cut line", store.name) + } - for i, store := range tc.stores { - if !tc.wantPruned[i] { - require.Falsef(t, store.pruneBelowCalled.Load(), "%s should not be pruned", store.name) + if tc.wantHistoryBelow == nil { + require.Falsef(t, store.historyPruned.Load(), "%s history should not be pruned", store.name) continue } - require.NotNil(t, tc.wantPruneBelow, "wantPruned expects a prune height") - require.Truef(t, store.pruneBelowCalled.Load(), "%s should be pruned", store.name) - require.Equalf(t, *tc.wantPruneBelow, store.prunedBelow.Load(), "%s prune height", store.name) + require.Truef(t, store.historyPruned.Load(), "%s history should be pruned", store.name) + require.Equalf(t, *tc.wantHistoryBelow, store.prunedHistoryBelow.Load(), + "%s history cut line", store.name) } }) } } -func TestPruneOnYoungChainWithDeepContiguousRetention(t *testing.T) { - // head 100 << WAL retain window (rollback 1_000 + retention 100_000) → both cutLines 0. +// The whole reason PruneSnapshots and PruneHistory are separate calls: they are handed cut lines one +// lookback window apart. Collapsing them onto one height would either strand the retained snapshot +// (snapshots down at the history cut line are the restore points nothing can ask for) or throw away +// the lookback window (history stopping at the snapshot cut line). +func TestPruneUsesDifferentDepthsForSnapshotsAndHistory(t *testing.T) { + sc := snapshotStore("sc", 100_000, 60_000) + wal := contiguousStore("stateWAL", 100_000) + + require.NoError(t, prune(testConfig(t, 10_000, 5_000), prunableStores(sc, wal))) + + require.Equal(t, uint64(60_000), sc.prunedSnapshotsBelow.Load(), + "sc's own snapshot binds the minimum, and it is never itself a candidate") + require.Equal(t, uint64(55_000), wal.prunedHistoryBelow.Load(), + "history goes a lookback window below that snapshot, keeping it replayable") + require.Equal(t, uint64(60_000), wal.prunedSnapshotsBelow.Load(), + "a contiguous store is still called; it holds no snapshots and no-ops") +} + +// One cut line reaching genesis must not silence the other. Here the rollback promise is satisfiable +// so snapshots below the floor are dead weight and go, while the lookback window from that floor +// lands below genesis and history is left untouched. +func TestPruneStillPrunesSnapshotsWhenHistoryIsHeldAtZero(t *testing.T) { + sc := snapshotStore("sc", 50_000, 10_000, 40_000) + wal := contiguousStore("stateWAL", 50_000) + + require.NoError(t, prune(testConfig(t, 1_000, 90_000), prunableStores(sc, wal))) + + require.False(t, sc.historyPruned.Load(), "the lookback window reaches below genesis") + require.True(t, sc.snapshotsPruned.Load(), "snapshots are still pruned") + require.Equal(t, uint64(40_000), sc.prunedSnapshotsBelow.Load(), + "sc's floor snapshot binds, so the 10_000 one below it goes") +} + +// A chain younger than its own rollback window must lose nothing: every store owes a rollback +// deeper than its whole history, so every one answers 0 and both cut lines stay there. The +// deletions are still issued — they are no-ops at 0 — which is what keeps the collector free of a +// special case for the young chain. +func TestPruneOnYoungChain(t *testing.T) { sc := snapshotStore("sc", 100, 50, 100) - wal := withRetentionWindow(contiguousStore("stateWAL", 100), 100_000) + wal := contiguousStore("stateWAL", 100) - require.NoError(t, prune(testConfig(t, 1_000), prunableStores(sc, wal))) + require.NoError(t, prune(testConfig(t, 1_000, 100_000), prunableStores(sc, wal))) - require.False(t, sc.pruneBelowCalled.Load()) - require.False(t, wal.pruneBelowCalled.Load()) + require.Equal(t, uint64(0), sc.prunedHistoryBelow.Load()) + require.Equal(t, uint64(0), wal.prunedHistoryBelow.Load()) } -// Answers are held positionally rather than keyed by Name(), so a store sharing a name with a -// participating one must not inherit its prune. The stake is data loss: an infinite-retention -// store exists precisely to keep what the others are dropping. -func TestPruneKeepsInfiniteRetentionStoreWithDuplicateName(t *testing.T) { - archive := withRetentionWindow(contiguousStore("ss", 100_000), InfiniteRetentionWindow) +// Answers are held positionally rather than keyed by Name(), so a store sharing a name with another +// must not inherit its decision. The stake is data loss: only the blocked store's own answer says +// whether the range it needs still exists. +func TestPruneKeepsAnswersPositionalWithDuplicateNames(t *testing.T) { + selfPruned := withSelfPruning(contiguousStore("ss", 100_000)) participating := snapshotStore("ss", 100_000, 80_000) wal := contiguousStore("stateWAL", 100_000) - require.NoError(t, prune(testConfig(t, 10_000), prunableStores(archive, participating, wal))) + require.NoError(t, prune(testConfig(t, 10_000, 0), prunableStores(selfPruned, participating, wal))) - require.False(t, archive.pruneBelowCalled.Load(), "an infinite-retention store must never be pruned") - require.True(t, participating.pruneBelowCalled.Load()) - require.Equal(t, uint64(80_000), participating.prunedBelow.Load()) - require.Equal(t, uint64(80_000), wal.prunedBelow.Load()) + require.False(t, selfPruned.historyPruned.Load(), "a self-pruning store must never be pruned") + require.True(t, participating.historyPruned.Load()) + require.Equal(t, uint64(80_000), participating.prunedHistoryBelow.Load()) + require.Equal(t, uint64(80_000), wal.prunedHistoryBelow.Load()) } -func TestPruneGetLatestBlockError(t *testing.T) { - sentinel := errors.New("boom") +// A store that cannot read its own head has no separate error path to the collector: it answers a +// floor of 0, which the minimum carries into both cut lines, and the cycle deletes nothing anywhere. +// Refusing to delete is the whole requirement, and it falls out of the ordinary reduction. +func TestPruneUnreadableStoreStopsTheCycle(t *testing.T) { sc := snapshotStore("sc", 100_000, 80_000) - broken := contiguousStore("brokenStore", 100_000) - broken.getErr = sentinel + unreadable := contiguousStore("brokenStore", 100_000) + unreadable.floor = func(uint64) uint64 { return 0 } - err := prune(testConfig(t, 10_000), prunableStores(sc, broken)) - require.ErrorIs(t, err, sentinel) - require.ErrorContains(t, err, "brokenStore") - require.False(t, sc.pruneBelowCalled.Load()) - require.False(t, broken.pruneBelowCalled.Load()) + require.NoError(t, prune(testConfig(t, 10_000, 0), prunableStores(sc, unreadable))) + + require.False(t, sc.historyPruned.Load()) + require.False(t, sc.snapshotsPruned.Load()) + require.False(t, unreadable.historyPruned.Load()) } -func TestPruneBelowErrorContinuesRemainingStores(t *testing.T) { +func TestPruneErrorContinuesRemainingStores(t *testing.T) { firstErr := errors.New("boom1") secondErr := errors.New("boom2") first := snapshotStore("first", 100_000, 80_000) @@ -415,178 +519,154 @@ func TestPruneBelowErrorContinuesRemainingStores(t *testing.T) { second.pruneErr = secondErr wal := contiguousStore("stateWAL", 100_000) - err := prune(testConfig(t, 10_000), prunableStores(first, second, wal)) + err := prune(testConfig(t, 10_000, 0), prunableStores(first, second, wal)) require.ErrorIs(t, err, firstErr) require.ErrorIs(t, err, secondErr) require.ErrorContains(t, err, "first") require.ErrorContains(t, err, "second") require.ErrorContains(t, err, "80000") - require.True(t, first.pruneBelowCalled.Load()) - require.True(t, second.pruneBelowCalled.Load()) - require.True(t, wal.pruneBelowCalled.Load()) - require.Equal(t, uint64(80_000), wal.prunedBelow.Load()) + require.True(t, first.historyPruned.Load()) + require.True(t, second.historyPruned.Load()) + require.True(t, wal.historyPruned.Load()) + require.Equal(t, uint64(80_000), wal.prunedHistoryBelow.Load()) } -func TestGetCutLine(t *testing.T) { - cases := []struct { - name string - head uint64 - rollbackWindow uint64 - retention int64 - want uint64 - }{ - {name: "rollback window only", head: 100_000, rollbackWindow: 10_000, want: 90_000}, - {name: "zero rollback window", head: 100_000, rollbackWindow: 0, want: 100_000}, - {name: "zero rollback with retention", head: 100_000, rollbackWindow: 0, retention: 10_000, want: 90_000}, - {name: "retention adds to rollback window", head: 100_000, rollbackWindow: 1, retention: 10_000, want: 89_999}, - {name: "the two windows add", head: 100_000, rollbackWindow: 10_000, retention: 5_000, want: 85_000}, - {name: "zero retention", head: 100_000, rollbackWindow: 10_000, want: 90_000}, - {name: "infinite retention", head: 100_000, rollbackWindow: 10_000, retention: InfiniteRetentionWindow, want: 0}, - {name: "any negative retention is infinite", head: 100_000, rollbackWindow: 10_000, retention: -99, want: 0}, - {name: "head one above the window", head: 10_001, rollbackWindow: 10_000, want: 1}, - {name: "head exactly at the window", head: 10_000, rollbackWindow: 10_000, want: 0}, - {name: "head one below the window", head: 9_999, rollbackWindow: 10_000, want: 0}, - {name: "head far below the window", head: 100, rollbackWindow: 1_000, retention: 100_000, want: 0}, - {name: "head at genesis", head: 0, rollbackWindow: 10_000, want: 0}, - {name: "rollback plus retention overflows uint64", head: math.MaxUint64, rollbackWindow: math.MaxUint64, retention: 1, want: 0}, - {name: "max rollback alone does not overflow", head: math.MaxUint64, rollbackWindow: math.MaxUint64, want: 0}, - } +// A failure pruning snapshots must not skip the history prune on the same store: the two are +// independent deletions, and permission-to-drop is not transactional. +func TestPruneHistoryStillRunsAfterSnapshotError(t *testing.T) { + sentinel := errors.New("boom") + sc := snapshotStore("sc", 100_000, 80_000) + sc.pruneErr = sentinel - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - require.Equal(t, tc.want, getCutLine(tc.head, tc.rollbackWindow, tc.retention)) - }) - } + err := prune(testConfig(t, 10_000, 0), prunableStores(sc)) + require.ErrorIs(t, err, sentinel) + require.ErrorContains(t, err, "snapshots below 80000") + require.ErrorContains(t, err, "history below 80000") + require.True(t, sc.historyPruned.Load()) } -func TestGetGlobalLatestBlock(t *testing.T) { - storesWithHeads := func(heads ...uint64) []PrunableStore { - list := make([]*mockStore, len(heads)) - for i, head := range heads { - list[i] = contiguousStore("store", head) - } - return prunableStores(list...) - } - +func TestGetHistoryCutLine(t *testing.T) { cases := []struct { - name string - heads []uint64 - want uint64 + name string + snapshotCutLine uint64 + lookbackWindow int64 + want uint64 }{ - {name: "single store", heads: []uint64{100}, want: 100}, - {name: "all agree", heads: []uint64{100, 100, 100}, want: 100}, - {name: "smallest first", heads: []uint64{80, 100}, want: 80}, - {name: "smallest last", heads: []uint64{100, 80}, want: 80}, - {name: "smallest in the middle", heads: []uint64{100, 50, 90}, want: 50}, - {name: "leading zero ignored", heads: []uint64{0, 100}, want: 100}, - {name: "trailing zero ignored", heads: []uint64{100, 0}, want: 100}, - {name: "zero among many ignored", heads: []uint64{100, 0, 80}, want: 80}, - {name: "every store reports zero", heads: []uint64{0, 0}, want: 0}, - {name: "no stores", heads: nil, want: 0}, + {name: "no lookback leaves the cut line alone", snapshotCutLine: 90_000, want: 90_000}, + {name: "lookback deepens the cut line", snapshotCutLine: 90_000, lookbackWindow: 5_000, want: 85_000}, + {name: "one block of lookback", snapshotCutLine: 90_000, lookbackWindow: 1, want: 89_999}, + {name: "lookback one below the cut line", snapshotCutLine: 5_001, lookbackWindow: 5_000, want: 1}, + {name: "lookback exactly at the cut line", snapshotCutLine: 5_000, lookbackWindow: 5_000, want: 0}, + {name: "lookback past genesis", snapshotCutLine: 100, lookbackWindow: 100_000, want: 0}, + {name: "a cut line of 0 stays 0", snapshotCutLine: 0, lookbackWindow: 10, want: 0}, + {name: "no cut line and no lookback", want: 0}, + {name: "max cut line", snapshotCutLine: math.MaxUint64, lookbackWindow: 1, want: math.MaxUint64 - 1}, + {name: "infinite lookback holds history at 0", snapshotCutLine: math.MaxUint64, lookbackWindow: -1, want: 0}, + {name: "infinite lookback with no cut line", snapshotCutLine: 0, lookbackWindow: -1, want: 0}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - head, err := getGlobalLatestBlock(storesWithHeads(tc.heads...)) - require.NoError(t, err) - require.Equal(t, tc.want, head) + require.Equal(t, tc.want, getHistoryCutLine(tc.snapshotCutLine, tc.lookbackWindow)) }) } } -// The prune log is the stated mechanism for reconstructing a deletion after the fact, so the three -// outcomes must not collapse onto the same rendering: never asked (cutLine 0), unable to serve a -// rollback, and asked with a boundary. The first two both carry boundary 0, so cutLine is the only -// thing separating them. -func TestDescribeDecisionsRendersEachOutcomeDistinctly(t *testing.T) { +// The prune log is the stated mechanism for reconstructing a deletion after the fact, so every store +// has to appear whatever it answered — including the one answering 0, which is the entry that explains +// a cycle that deleted nothing — and a store the collector does not prune has to be distinguishable +// from one it pruned to no effect. +func TestDescribeDecisionsRendersEveryStore(t *testing.T) { stores := prunableStores( - withRetentionWindow(contiguousStore("archiveWAL", 100_000), InfiniteRetentionWindow), - snapshotStore("sc", 100_000), + contiguousStore("blockDB", 0), contiguousStore("stateWAL", 100_000), + withSelfPruning(snapshotStore("selfSC", 100_000, 50_000)), ) decisions := []storeDecision{ - {cutLine: 0, boundary: CannotServeRollback}, // skipped before being asked - {cutLine: 90_000, boundary: CannotServeRollback}, // asked, cannot serve a rollback - {cutLine: 90_000, boundary: 90_000}, // asked, reported a boundary + {floor: 0, externalPruning: true}, // asked, needs everything kept + {floor: 90_000, externalPruning: true}, // asked, named a floor + {floor: 50_000}, // asked, but prunes itself } require.Equal(t, - "archiveWAL=notAsked sc=cannotServeRollback(cutLine=90000) "+ - "stateWAL=90000(cutLine=90000)", + "blockDB=0 stateWAL=90000 selfSC=50000(selfPruned)", describeDecisions(stores, decisions), ) } -// One blocker abandons the cycle, but every store is still asked so the blocked cycle can log a -// complete decision set. Under RollbackWindow 0 the guarantee is waived and the blocker is ignored -// outright, which is what lets the later stores prune. -func TestPruneAsksEveryStoreEvenWhenBlocked(t *testing.T) { - blocker := snapshotStore("sc", 100_000) // no snapshot -> CannotServeRollback - afterBlocker := contiguousStore("stateWAL", 100_000) - require.NoError(t, prune(testConfig(t, 1_000), prunableStores(blocker, afterBlocker))) - - require.Equal(t, uint64(1), blocker.boundaryCalls.Load()) - require.False(t, afterBlocker.pruneBelowCalled.Load(), "the cycle must be abandoned") - require.Equal(t, uint64(1), afterBlocker.boundaryCalls.Load(), - "a blocked cycle still asks every store, so its log names every blocker at once") +// The whole point of ExternalPruning: a self-pruning store is still a full participant in the +// decision — its floor drags the shared minimum down and protects the range it replays from — while +// never being pruned by the collector. Withdrawing its answer instead would prune the WAL to 99_000 +// and strand its snapshot at 50_000 with nothing to replay forward. +func TestPruneSkipsSelfPruningStoreButKeepsItsAnswer(t *testing.T) { + selfPruned := withSelfPruning(snapshotStore("sc", 100_000, 50_000)) + wal := contiguousStore("stateWAL", 100_000) - waived := snapshotStore("sc", 100_000) - afterWaived := contiguousStore("stateWAL", 100_000) - require.NoError(t, prune(testConfig(t, 0), prunableStores(waived, afterWaived))) + require.NoError(t, prune(testConfig(t, 1_000, 0), prunableStores(selfPruned, wal))) - require.Equal(t, uint64(1), afterWaived.boundaryCalls.Load(), - "RollbackWindow 0 ignores the blocker, so later stores must still be asked") - require.True(t, afterWaived.pruneBelowCalled.Load()) - require.Equal(t, uint64(100_000), afterWaived.prunedBelow.Load()) + require.Equal(t, uint64(1), selfPruned.floorCalls.Load(), "a self-pruning store is still asked") + require.False(t, selfPruned.historyPruned.Load(), "its own pruner enforces its retention") + require.False(t, selfPruned.snapshotsPruned.Load(), "including its snapshots") + require.True(t, wal.historyPruned.Load()) + require.Equal(t, uint64(50_000), wal.prunedHistoryBelow.Load(), + "the self-pruning store's floor must still hold the WAL back to its snapshot") } -// InfiniteRetentionWindow on a snapshot store is a footgun, and this pins which way it cuts. A -// store with no cut line is never asked, so it leaves the minimum entirely and the WAL holding its -// replay range is pruned to the WAL's own cut line — the old snapshot survives but can no longer be -// replayed forward. Retention 0 protects it strictly better, because then it answers its oldest -// needed snapshot and drags the WAL down to it. -// -// The other infinite-retention cases in TestPruneDecisions only cover the safe direction, where the -// infinite-retention store is the contiguous archive (a replay source, not a consumer). -func TestPruneInfiniteRetentionSnapshotStore(t *testing.T) { - archive := withRetentionWindow(snapshotStore("archiveSC", 100_000, 20_000), InfiniteRetentionWindow) +// A self-pruning store that has ingested nothing still holds history where it is. Who deletes its +// data is a separate question from whether the range it will need exists yet, and only the second is +// what its floor reports. +func TestPruneSelfPruningEmptyStoreStillHoldsHistory(t *testing.T) { + empty := withSelfPruning(contiguousStore("ss", 0)) wal := contiguousStore("stateWAL", 100_000) - require.NoError(t, prune(testConfig(t, 1_000), prunableStores(archive, wal))) - require.Zero(t, archive.boundaryCalls.Load(), "no cut line means it is never asked") - require.False(t, archive.pruneBelowCalled.Load(), "infinite retention is never pruned") - require.True(t, wal.pruneBelowCalled.Load()) - require.Equal(t, uint64(99_000), wal.prunedBelow.Load(), - "the archive contributes nothing, so the WAL only honors its own cut line") + require.NoError(t, prune(testConfig(t, 1_000, 0), prunableStores(empty, wal))) + + require.Equal(t, uint64(0), wal.prunedHistoryBelow.Load(), "history must not move past an empty store") + require.False(t, empty.historyPruned.Load(), "its own pruner enforces its retention") +} + +// Every store is asked exactly once per cycle, whatever the others answer, so the log carries a +// complete decision set and a store cannot be skipped on the strength of an earlier one's answer. +func TestPruneAsksEveryStoreOncePerCycle(t *testing.T) { + empty := contiguousStore("blockDB", 0) + sc := snapshotStore("sc", 100_000, 80_000) + wal := contiguousStore("stateWAL", 100_000) - // Same store, contract-sanctioned retention of 0: it participates and protects its snapshot. - compliant := snapshotStore("archiveSC", 100_000, 20_000) - wal2 := contiguousStore("stateWAL", 100_000) - require.NoError(t, prune(testConfig(t, 1_000), prunableStores(compliant, wal2))) + require.NoError(t, prune(testConfig(t, 1_000, 0), prunableStores(empty, sc, wal))) - require.Equal(t, uint64(20_000), wal2.prunedBelow.Load(), - "retention 0 pulls the WAL down to the snapshot, keeping it replayable") + for _, store := range []*mockStore{empty, sc, wal} { + require.Equalf(t, uint64(1), store.floorCalls.Load(), "%s asked exactly once", store.name) + } + require.Equal(t, uint64(0), wal.prunedHistoryBelow.Load(), "the empty store's answer binds the cycle") } func TestDefaultStorageGarbageCollectorConfig(t *testing.T) { cfg := DefaultStorageGarbageCollectorConfig() require.Equal(t, uint64(1_000), cfg.RollbackWindow) + require.Equal(t, int64(0), cfg.LookbackWindow) require.Equal(t, 5*time.Minute, cfg.PruneInterval) require.NoError(t, cfg.Validate()) } +// The windows are independent, so every combination of non-negative counts validates, as does the +// -1 infinite-retention sentinel on the lookback window. The interval and a lookback below -1 are +// the only rejections. func TestValidate(t *testing.T) { require.ErrorContains(t, (*StorageGarbageCollectorConfig)(nil).Validate(), "config is required") - require.NoError(t, (&StorageGarbageCollectorConfig{ - RollbackWindow: 0, - PruneInterval: time.Minute, - }).Validate()) + for _, windows := range [][2]int64{{0, 0}, {1, 0}, {0, 1}, {1_000, 50_000}, {50_000, 1_000}, {1_000, -1}} { + require.NoError(t, (&StorageGarbageCollectorConfig{ + RollbackWindow: uint64(windows[0]), + LookbackWindow: windows[1], + PruneInterval: time.Minute, + }).Validate(), "windows %v", windows) + } - require.NoError(t, (&StorageGarbageCollectorConfig{ + require.ErrorContains(t, (&StorageGarbageCollectorConfig{ RollbackWindow: 1, + LookbackWindow: -2, PruneInterval: time.Minute, - }).Validate()) + }).Validate(), "lookback window") require.ErrorContains(t, (&StorageGarbageCollectorConfig{ RollbackWindow: 1, @@ -665,13 +745,13 @@ func TestRunTickerDrivesPruneCycles(t *testing.T) { startCollector(t, 10*time.Millisecond, sc, wal) require.Eventually(t, func() bool { - return sc.pruneBelowCalled.Load() && wal.pruneBelowCalled.Load() + return sc.historyPruned.Load() && wal.historyPruned.Load() }, 2*time.Second, 5*time.Millisecond, "ticker should drive a prune cycle") - require.Equal(t, uint64(80_000), sc.prunedBelow.Load()) - require.Equal(t, uint64(80_000), wal.prunedBelow.Load()) + require.Equal(t, uint64(80_000), sc.prunedHistoryBelow.Load()) + require.Equal(t, uint64(80_000), wal.prunedHistoryBelow.Load()) } -// TestRunSurvivesPruneError covers run()'s logger.Error branch: a PruneBelow failure is logged and the loop keeps +// TestRunSurvivesPruneError covers run()'s logger.Error branch: a prune failure is logged and the loop keeps // ticking rather than exiting after the first error. func TestRunSurvivesPruneError(t *testing.T) { broken := snapshotStore("broken", 100_000, 80_000) @@ -680,7 +760,7 @@ func TestRunSurvivesPruneError(t *testing.T) { startCollector(t, 10*time.Millisecond, broken) require.Eventually(t, func() bool { - return broken.pruneBelowCalls.Load() > 1 + return broken.historyPruneCalls.Load() > 1 }, 2*time.Second, 5*time.Millisecond, "a failed prune must not kill the run loop") } diff --git a/sei-db/seiwal/seiwal.go b/sei-db/seiwal/seiwal.go index e0ce5f161a..497f579db1 100644 --- a/sei-db/seiwal/seiwal.go +++ b/sei-db/seiwal/seiwal.go @@ -18,7 +18,7 @@ var ErrIteratorRange = errors.New("invalid iterator range") // // A WAL instance is not safe for concurrent use: its methods must not be called from multiple // goroutines simultaneously. Callers that share a WAL across goroutines must serialize access -// themselves. +// themselves. PruneBefore is the sole exception; see its doc for the guarantee implementations owe. // // Slices are not copied at the call boundary. Any slice passed into a WAL method — the payload and every // slice reachable through it — must not be modified after the call: the WAL may retain it and read it @@ -64,6 +64,11 @@ type WAL[T any] interface { // async and lazy, and implementations are free to delay it arbitrarily long. Pruning removes whole // sealed files only, so records may survive above the requested threshold until their containing file // is fully below it. + // + // Unlike the other methods here, PruneBefore may be called from another goroutine, concurrently + // with any method including Append and Close, and implementations must support that without + // external serialization. Such a call is unordered with respect to appends: whether a record + // appended around the same instant is pruned is unspecified. PruneBefore(lowestIndexToKeep uint64) error // Iterator returns an iterator over the WAL across the inclusive index range [startIndex, endIndex]. diff --git a/sei-db/state_db/sc/flatkv/config/config.go b/sei-db/state_db/sc/flatkv/config/config.go index 764e3fba39..2a866a3f84 100644 --- a/sei-db/state_db/sc/flatkv/config/config.go +++ b/sei-db/state_db/sc/flatkv/config/config.go @@ -39,9 +39,22 @@ type Config struct { // SnapshotKeepRecent defines how many old snapshots to keep besides the // latest one. 0 means keep only the current snapshot (no old snapshots). + // Ignored entirely when ExternalPruning is set. // Default: 1 SnapshotKeepRecent uint32 `mapstructure:"snapshot-keep-recent"` + // ExternalPruning hands retention to the StorageGarbageCollector: the store stops pruning its + // own snapshots (SnapshotKeepRecent) and stops truncating the state WAL. + // + // Not read from app.toml. It is set by whatever constructs the collector, since it is only + // correct when this store is registered with a running one. + // + // With it on, snapshots are retained by height rather than by count, so the number kept becomes + // RollbackWindow / SnapshotInterval instead of SnapshotKeepRecent + 1. + // + // Default: false + ExternalPruning bool `mapstructure:"-"` + // EnablePebbleMetrics defines if the Pebble metrics should be enabled. // Default: true EnablePebbleMetrics bool `mapstructure:"enable-pebble-metrics"` diff --git a/sei-db/state_db/sc/flatkv/snapshot.go b/sei-db/state_db/sc/flatkv/snapshot.go index 9dd738f84d..8477aed430 100644 --- a/sei-db/state_db/sc/flatkv/snapshot.go +++ b/sei-db/state_db/sc/flatkv/snapshot.go @@ -511,7 +511,7 @@ func (s *CommitStore) WriteSnapshot(_ string) (err error) { logger.Error("failed to update SNAPSHOT_BASE", "err", err) } - pruned = s.pruneSnapshots(dir, version) + pruned = s.pruneSnapshotsByCount(dir, version) success = true s.lastSnapshotTime = time.Now() @@ -523,10 +523,17 @@ func (s *CommitStore) WriteSnapshot(_ string) (err error) { return nil } -// pruneSnapshots removes old snapshots beyond SnapshotKeepRecent, keeping +// pruneSnapshotsByCount removes old snapshots beyond SnapshotKeepRecent, keeping // the latest snapshot (currentVersion) plus the N most recent older ones. // Best-effort: errors are logged but do not fail the snapshot operation. -func (s *CommitStore) pruneSnapshots(dir string, currentVersion int64) int { +// +// Does nothing when config.ExternalPruning is set, which hands retention to the +// StorageGarbageCollector and its by-block-height PruneSnapshots. +func (s *CommitStore) pruneSnapshotsByCount(dir string, currentVersion int64) int { + if s.config.ExternalPruning { + return 0 + } + start := time.Now() defer func() { otelMetrics.SnapshotPruneLatency.Record(s.ctx, secondsSince(start)) @@ -722,12 +729,13 @@ func (s *CommitStore) Rollback(targetVersion int64) (err error) { return nil } -// tryTruncateWAL truncates WAL entries older than the earliest snapshot, keeping enough entries for rollback -// to any retained snapshot. Scheduling the truncation is best-effort in that it is skipped when there is -// nothing to prune against, but a prune that fails is not a benign outcome: it only fails when the WAL is -// already dead, which means commits will fail from that point on. +// tryTruncateWAL truncates WAL entries older than the earliest snapshot, keeping enough entries for +// rollback to any retained snapshot. Skipped when there is no snapshot to truncate against. +// +// Does nothing when config.ExternalPruning is set, under which the collector prunes the WAL as a +// managed store in its own right. func (s *CommitStore) tryTruncateWAL() { - if s.wal == nil { + if s.wal == nil || s.config.ExternalPruning { return } diff --git a/sei-db/state_db/sc/flatkv/store.go b/sei-db/state_db/sc/flatkv/store.go index 2adaf26976..5f7c07c0c9 100644 --- a/sei-db/state_db/sc/flatkv/store.go +++ b/sei-db/state_db/sc/flatkv/store.go @@ -58,44 +58,7 @@ const ( // dataDBDirs lists all data DB directory names (used for per-DB LtHash iteration). var dataDBDirs = []string{accountDBDir, codeDBDir, storageDBDir, miscDBDir} -// InitializeDataDirectories sets the DataDir for each nested PebbleDB config -// that does not already have one, using DataDir as the base path. The DBs live -// under the working directory: /working/. -func InitializeDataDirectories(c *config.Config) { - workDir := filepath.Join(c.DataDir, workingDirName) - if c.AccountDBConfig.DataDir == "" { - c.AccountDBConfig.DataDir = filepath.Join(workDir, accountDBDir) - } - if c.CodeDBConfig.DataDir == "" { - c.CodeDBConfig.DataDir = filepath.Join(workDir, codeDBDir) - } - if c.StorageDBConfig.DataDir == "" { - c.StorageDBConfig.DataDir = filepath.Join(workDir, storageDBDir) - } - if c.MiscDBConfig.DataDir == "" { - c.MiscDBConfig.DataDir = filepath.Join(workDir, miscDBDir) - } - if c.MetadataDBConfig.DataDir == "" { - c.MetadataDBConfig.DataDir = filepath.Join(workDir, metadataDir) - } - applyPebbleMetricsConfig(c) -} - -func applyPebbleMetricsConfig(c *config.Config) { - // Keep a single FlatKV-level knob for Pebble internal metrics. Per-DB - // EnableMetrics values are intentionally overwritten here. - c.AccountDBConfig.EnableMetrics = c.EnablePebbleMetrics - c.CodeDBConfig.EnableMetrics = c.EnablePebbleMetrics - c.StorageDBConfig.EnableMetrics = c.EnablePebbleMetrics - c.MiscDBConfig.EnableMetrics = c.EnablePebbleMetrics - c.MetadataDBConfig.EnableMetrics = c.EnablePebbleMetrics - - c.AccountDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics - c.CodeDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics - c.StorageDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics - c.MiscDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics - c.MetadataDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics -} +var _ Store = (*CommitStore)(nil) // CommitStore implements flatkv.Store for EVM state storage. // @@ -221,8 +184,6 @@ type CommitStore struct { ltCalc *lthash.HashCalculator } -var _ Store = (*CommitStore)(nil) - // dataDBs returns the four data PebbleDB instances in fixed iteration order: // accountDB, codeDB, storageDB, miscDB. metadataDB is excluded. func (s *CommitStore) dataDBs() []seidbtypes.KeyValueDB { @@ -909,3 +870,42 @@ func (s *CommitStore) reopenWAL() error { func (s *CommitStore) GetPhaseTimer() *metrics.PhaseTimer { return s.phaseTimer } + +// InitializeDataDirectories sets the DataDir for each nested PebbleDB config +// that does not already have one, using DataDir as the base path. The DBs live +// under the working directory: /working/. +func InitializeDataDirectories(c *config.Config) { + workDir := filepath.Join(c.DataDir, workingDirName) + if c.AccountDBConfig.DataDir == "" { + c.AccountDBConfig.DataDir = filepath.Join(workDir, accountDBDir) + } + if c.CodeDBConfig.DataDir == "" { + c.CodeDBConfig.DataDir = filepath.Join(workDir, codeDBDir) + } + if c.StorageDBConfig.DataDir == "" { + c.StorageDBConfig.DataDir = filepath.Join(workDir, storageDBDir) + } + if c.MiscDBConfig.DataDir == "" { + c.MiscDBConfig.DataDir = filepath.Join(workDir, miscDBDir) + } + if c.MetadataDBConfig.DataDir == "" { + c.MetadataDBConfig.DataDir = filepath.Join(workDir, metadataDir) + } + applyPebbleMetricsConfig(c) +} + +func applyPebbleMetricsConfig(c *config.Config) { + // Keep a single FlatKV-level knob for Pebble internal metrics. Per-DB + // EnableMetrics values are intentionally overwritten here. + c.AccountDBConfig.EnableMetrics = c.EnablePebbleMetrics + c.CodeDBConfig.EnableMetrics = c.EnablePebbleMetrics + c.StorageDBConfig.EnableMetrics = c.EnablePebbleMetrics + c.MiscDBConfig.EnableMetrics = c.EnablePebbleMetrics + c.MetadataDBConfig.EnableMetrics = c.EnablePebbleMetrics + + c.AccountDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics + c.CodeDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics + c.StorageDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics + c.MiscDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics + c.MetadataDBConfig.EnableReadWriteMetrics = c.EnableReadWriteMetrics +} diff --git a/sei-db/state_db/sc/flatkv/store_gc.go b/sei-db/state_db/sc/flatkv/store_gc.go new file mode 100644 index 0000000000..6e6e1b67a4 --- /dev/null +++ b/sei-db/state_db/sc/flatkv/store_gc.go @@ -0,0 +1,197 @@ +package flatkv + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/sei-protocol/sei-chain/sei-db/management/gc" +) + +// CommitStore participates in the shared prune cycle as a snapshot store: it restores only at a +// snapshot boundary, replaying the state WAL forward from there to reach any higher height. +// +// The state WAL must be managed by the same collector. This store reports a floor that keeps the WAL +// it replays from, but never prunes that WAL itself. +var _ gc.PrunableStore = (*CommitStore)(nil) + +func (s *CommitStore) Name() string { + return "FlatKV" +} + +// ExternalPruning reports config.ExternalPruning, the same field pruneSnapshotsByCount and +// tryTruncateWAL consult to stand down. It is fixed at construction. +func (s *CommitStore) ExternalPruning() bool { + return s.config.ExternalPruning +} + +// PruneHistory does nothing: the history this store replays over is the state WAL, which the +// collector manages as a store in its own right. +func (s *CommitStore) PruneHistory(uint64) error { + return nil +} + +// PruneSnapshots deletes every snapshot strictly below blockNumber, never the active snapshot. A +// snapshot that fails to delete is reported while the rest are still attempted; one that is already +// gone is not an error. +func (s *CommitStore) PruneSnapshots(blockNumber uint64) error { + if blockNumber == 0 { + return nil + } + blocks, err := s.snapshotBlocks() + if err != nil { + return fmt.Errorf("scan snapshots: %w", err) + } + if len(blocks) == 0 { + return nil + } + active, err := s.activeSnapshotHeight() + if err != nil { + return fmt.Errorf("prune snapshots below %d: %w", blockNumber, err) + } + cutLine := min(blockNumber, active) + + var errs error + pruned := 0 + for _, block := range blocks { + if block >= cutLine { + break // ascending, so nothing further is a candidate + } + removed, err := s.deleteSnapshot(block) + if err != nil { + errs = errors.Join(errs, err) + continue + } + if removed { + pruned++ + } + } + + if pruned > 0 { + logger.Info("pruned snapshots below the rollback cut line", "count", pruned, "cutLine", cutLine) + } + return errs +} + +// activeSnapshotHeight returns the height of the snapshot "current" points at — the one the next open +// clones and replays the state WAL forward from. It is usually the newest snapshot on disk, but a +// crash during WriteSnapshot or a partial Rollback can leave it lower. +func (s *CommitStore) activeSnapshotHeight() (uint64, error) { + _, version, err := currentSnapshotDir(s.flatkvDir()) + if err != nil { + return 0, fmt.Errorf("resolve active snapshot: %w", err) + } + return uint64(max(version, 0)), nil //nolint:gosec // clamped non-negative above +} + +// snapshotFloor returns the oldest snapshot that must survive a rollback of rollbackWindow blocks +// behind head: +// +// a snapshot at or below head - rollbackWindow → the newest such snapshot +// every snapshot above that height → the oldest snapshot, the deepest this store can +// restore to +// no snapshot, or a window deeper than head → 0, nothing here is eligible for pruning +// +// Version 0 is never the answer, as it restores to no committed height. blocks may be in any order. +func snapshotFloor(blocks []uint64, head uint64, rollbackWindow uint64) uint64 { + if head <= rollbackWindow { + return 0 + } + target := head - rollbackWindow + + var oldest, newestPastWindow uint64 + foundReal := false + for _, block := range blocks { + if block == 0 { + continue + } + if !foundReal || block < oldest { + oldest = block + } + foundReal = true + if block <= target && block > newestPastWindow { + newestPastWindow = block + } + } + + if !foundReal { + return 0 + } + if newestPastWindow > 0 { + return newestPastWindow + } + return oldest +} + +// snapshotBlocks returns the block number of every snapshot on disk, ascending. A missing snapshot +// directory yields no blocks rather than an error. +func (s *CommitStore) snapshotBlocks() ([]uint64, error) { + var blocks []uint64 + err := traverseSnapshots(s.flatkvDir(), true, func(version int64) (bool, error) { + if version >= 0 { + blocks = append(blocks, uint64(version)) + } + return false, nil + }) + if err != nil { + return nil, err + } + return blocks, nil +} + +// deleteSnapshot removes the snapshot directory for block, reporting whether this call is the one that +// removed it. An already-gone snapshot is not an error and reports false. +func (s *CommitStore) deleteSnapshot(block uint64) (bool, error) { + path := filepath.Join(s.flatkvDir(), snapshotName(int64(block))) //nolint:gosec // block numbers are bounded well below 2^63 + if err := atomicRemoveDir(path); err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, fmt.Errorf("remove snapshot %d: %w", block, err) + } + return true, nil +} + +// GetRollbackFloor returns the oldest snapshot this store must keep to serve a rollback of +// rollbackWindow blocks behind its own committed head, bounded by the active snapshot. See +// snapshotFloor for the outcomes. +// +// It returns 0 — nothing here is eligible for pruning — when there is no snapshot to name, when the +// window is deeper than the head, or when the snapshot layout cannot be read. +func (s *CommitStore) GetRollbackFloor(rollbackWindow uint64) uint64 { + blocks, err := s.snapshotBlocks() + if err != nil { + logger.Error("failed to scan snapshots for the rollback floor; holding it at 0", + "rollbackWindow", rollbackWindow, "err", err) + return 0 + } + head, err := s.GetLatestBlock() + if err != nil { + logger.Error("failed to read the committed version for the rollback floor; holding it at 0", + "rollbackWindow", rollbackWindow, "err", err) + return 0 + } + floor := snapshotFloor(blocks, head, rollbackWindow) + if floor == 0 { + return 0 + } + active, err := s.activeSnapshotHeight() + if err != nil { + logger.Error("failed to resolve the active snapshot for the rollback floor; holding it at 0", + "rollbackWindow", rollbackWindow, "err", err) + return 0 + } + return min(floor, active) +} + +// GetLatestBlock returns the highest committed version, or 0 when nothing has been committed. This is +// the store's ingest position, not its newest snapshot. +func (s *CommitStore) GetLatestBlock() (uint64, error) { + s.mu.RLock() + defer s.mu.RUnlock() + if s.committedVersion <= 0 { + return 0, nil + } + return uint64(s.committedVersion), nil //nolint:gosec // guarded positive above +} diff --git a/sei-db/state_db/sc/flatkv/store_gc_test.go b/sei-db/state_db/sc/flatkv/store_gc_test.go new file mode 100644 index 0000000000..5001b4e6e7 --- /dev/null +++ b/sei-db/state_db/sc/flatkv/store_gc_test.go @@ -0,0 +1,513 @@ +package flatkv + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/management/gc" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" + "github.com/sei-protocol/sei-chain/sei-db/state_db/statewal" +) + +// The GC surface reads the snapshot directory and two plain fields, so most of it can be exercised +// without opening the five PebbleDBs a real store carries. Only the tests that assert against +// snapshots WriteSnapshot produced, or against a concurrent committer, pay for a live store. +func gcStore(t *testing.T, dir string) (*CommitStore, gc.PrunableStore) { + t.Helper() + s := &CommitStore{config: config.Config{DataDir: dir}} + return s, s +} + +// gcStoreAtHead is gcStore with a committed head. The head is what the rollback window is resolved +// against, so a test that wants a particular depth sets one rather than passing a height in. +func gcStoreAtHead(t *testing.T, dir string, head int64) (*CommitStore, gc.PrunableStore) { + t.Helper() + s, store := gcStore(t, dir) + s.committedVersion = head + return s, store +} + +// mkSnapshots creates snapshot directories and points "current" at the newest one on disk, which is +// the shape production leaves behind: WriteSnapshot repoints the symlink at each snapshot it writes. +// The GC surface is bounded by the active snapshot, so a test that left "current" unset would be +// measuring a state a live store never reaches. The tests that want a stale or missing symlink say so +// after calling this. +func mkSnapshots(t *testing.T, dir string, versions ...int64) { + t.Helper() + for _, v := range versions { + require.NoError(t, os.MkdirAll(filepath.Join(dir, snapshotName(v)), 0750)) + } + onDisk := snapshotVersions(t, dir) + if len(onDisk) == 0 { + return + } + require.NoError(t, updateCurrentSymlink(dir, snapshotName(onDisk[len(onDisk)-1]))) +} + +func snapshotVersions(t *testing.T, dir string) []int64 { + t.Helper() + var found []int64 + require.NoError(t, traverseSnapshots(dir, true, func(v int64) (bool, error) { + found = append(found, v) + return false, nil + })) + return found +} + +// This store's history is the state WAL, which the collector manages as a store in its own right and +// prunes to the shared minimum. So PruneHistory here is a no-op, and must not reach the snapshots +// PruneSnapshots owns. +func TestGCPruneHistoryIsANoOp(t *testing.T) { + s, store := gcStore(t, t.TempDir()) + dir := s.flatkvDir() + mkSnapshots(t, dir, 5, 10, 20) + + require.NoError(t, store.PruneHistory(20)) + require.Equal(t, []int64{5, 10, 20}, snapshotVersions(t, dir)) +} + +// The head is the committed version, not the newest snapshot: it is this store's ingest position, and +// what GetRollbackFloor measures the rollback window against. +func TestGCLatestBlockIsCommittedVersion(t *testing.T) { + s, store := gcStore(t, t.TempDir()) + + latest, err := store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(0), latest, "nothing committed yet") + + // Snapshots do not move the head; commits do. + mkSnapshots(t, s.flatkvDir(), 10, 20) + latest, err = store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(0), latest) + + s.committedVersion = 42 + latest, err = store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(42), latest) +} + +// snapshotFloor is fed a directory listing that happens to be sorted today, but it must not depend on +// that: choosing the floor by iteration position rather than by value would answer too high on an +// unsorted slice, and too high is the one direction that lets the WAL be pruned past blocks a restore +// replays. Every case runs its blocks in a scrambled order and expects the same answer as the sorted +// one would give. +func TestSnapshotFloorIsOrderIndependent(t *testing.T) { + const head = 100 + for _, tc := range []struct { + name string + blocks []uint64 + rollbackWindow uint64 + want uint64 + }{ + {name: "newest at or below the window", blocks: []uint64{20, 5, 10}, rollbackWindow: 85, want: 10}, + {name: "shortfall answers the oldest", blocks: []uint64{97, 90, 95}, rollbackWindow: 10, want: 90}, + {name: "version 0 is never the floor", blocks: []uint64{30, 0, 10}, rollbackWindow: 95, want: 10}, + {name: "only version 0 reads as none", blocks: []uint64{0}, rollbackWindow: 10, want: 0}, + {name: "window past genesis", blocks: []uint64{40, 10}, rollbackWindow: 100, want: 0}, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, snapshotFloor(tc.blocks, head, tc.rollbackWindow)) + }) + } +} + +// Restoring to a height inside the rollback window means starting at the newest snapshot at or below +// that depth and replaying the WAL forward, so that snapshot is the oldest thing this store must keep. +// The depth comes from its own head, which is why every case here fixes a head of 100. +func TestGCRollbackFloorIsNewestSnapshotPastTheWindow(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 100) + mkSnapshots(t, s.flatkvDir(), 5, 10, 20) + + for _, tc := range []struct { + rollbackWindow uint64 + want uint64 + why string + }{ + {rollbackWindow: 85, want: 10, why: "newest at or below head - window"}, + {rollbackWindow: 80, want: 20, why: "a snapshot exactly at that depth serves it"}, + {rollbackWindow: 0, want: 20, why: "a window of 0 reaches the newest snapshot"}, + {rollbackWindow: 95, want: 5, why: "the oldest snapshot exactly at that depth"}, + } { + require.Equal(t, tc.want, store.GetRollbackFloor(tc.rollbackWindow), tc.why) + } +} + +// Whatever the window, the answer is either a snapshot that exists or 0. That is what lets the +// collector hold history at a floor this store can restore from, and it is the property a store must +// never break — a floor naming a height with no snapshot on it would be a restore point the store +// cannot serve. +func TestGCRollbackFloorIsAlwaysAnExistingSnapshot(t *testing.T) { + const head = 40 + s, store := gcStoreAtHead(t, t.TempDir(), head) + mkSnapshots(t, s.flatkvDir(), 7, 13, 29) + + for rollbackWindow := uint64(0); rollbackWindow <= 60; rollbackWindow++ { + if rollbackWindow >= head { + require.Equal(t, uint64(0), store.GetRollbackFloor(rollbackWindow), + "a window deeper than the head leaves nothing eligible at rollbackWindow=%d", rollbackWindow) + continue + } + require.Contains(t, []uint64{7, 13, 29}, store.GetRollbackFloor(rollbackWindow), + "floor is not a snapshot at rollbackWindow=%d", rollbackWindow) + } +} + +// Every snapshot above the window means none can be dropped, and the store reports its oldest — the +// deepest it can actually restore to. That is a snapshot-retention shortfall (the snapshot depth is +// shallower than RollbackWindow), and naming that snapshot is what keeps it replayable rather than +// letting history be pruned out from under it. +func TestGCRollbackFloorAllSnapshotsInsideTheWindow(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 1_000) + mkSnapshots(t, s.flatkvDir(), 500, 900) + + require.Equal(t, uint64(500), store.GetRollbackFloor(900), "head - window is 100, below both snapshots") + require.Equal(t, uint64(0), store.GetRollbackFloor(2_000), + "a window deeper than the head leaves nothing eligible for pruning") +} + +// With no snapshot there is none to name, so nothing here is eligible for pruning: a store that +// cannot restore anywhere cannot say which blocks the WAL may drop. +func TestGCRollbackFloorWithoutSnapshots(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 100) + require.Equal(t, uint64(0), store.GetRollbackFloor(10)) + + // A directory that does not exist yet reads the same way. + require.NoError(t, os.RemoveAll(s.flatkvDir())) + require.Equal(t, uint64(0), store.GetRollbackFloor(10)) + + // So does a store that has committed nothing. + _, fresh := gcStore(t, t.TempDir()) + require.Equal(t, uint64(0), fresh.GetRollbackFloor(10)) +} + +// The initial empty snapshot restores to no committed height, so it is not a restore point and cannot +// be a floor. A store holding only that one is read as holding none. +func TestGCRollbackFloorIgnoresInitialSnapshot(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 100) + mkSnapshots(t, s.flatkvDir(), 0) + + require.Equal(t, uint64(0), store.GetRollbackFloor(10), "version 0 does not count as a snapshot") + + // With a real snapshot present, version 0 still contributes nothing — not even as the oldest. + mkSnapshots(t, s.flatkvDir(), 30) + require.Equal(t, uint64(30), store.GetRollbackFloor(0)) + require.Equal(t, uint64(30), store.GetRollbackFloor(90), + "head - window is 10; the oldest real snapshot answers, not version 0") +} + +// Not knowing which snapshots exist means not knowing which blocks are needed to replay from them, and +// the WAL would be pruned on the strength of that answer. So a failed scan reports 0, holding the +// fleet's history where it is. +func TestGCRollbackFloorScanFailureHoldsHistory(t *testing.T) { + dir := filepath.Join(t.TempDir(), "flatkv") + require.NoError(t, os.WriteFile(dir, []byte("not a directory"), 0600)) + + _, store := gcStoreAtHead(t, dir, 100) + require.Equal(t, uint64(0), store.GetRollbackFloor(10)) +} + +// PruneSnapshots drops everything strictly below the height it is given, and the height need not +// have a snapshot on it. +func TestGCPruneSnapshotsDeletesBelowTheCutLine(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 30) + dir := s.flatkvDir() + mkSnapshots(t, dir, 0, 5, 10, 20, 30) + require.NoError(t, updateCurrentSymlink(dir, snapshotName(30))) + + // A cut line of 25 has no snapshot on it; 20 is the newest below it and goes with the rest. + require.NoError(t, store.PruneSnapshots(25)) + require.Equal(t, []int64{30}, snapshotVersions(t, dir)) + + // Idempotent: the same cut line twice deletes nothing more. + require.NoError(t, store.PruneSnapshots(25)) + require.Equal(t, []int64{30}, snapshotVersions(t, dir)) +} + +// The snapshot the store reported must survive the cycle that follows, which is what makes the +// collector's minimum meaningful: history is held at a floor the store still has the snapshot for. +// The collector's cut line is the minimum across stores and so is at or below this store's own +// answer, which the loop walks the tightest case of. +func TestGCPruneSnapshotsKeepsTheSnapshotItReported(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 40) + dir := s.flatkvDir() + mkSnapshots(t, dir, 5, 10, 20, 30) + require.NoError(t, updateCurrentSymlink(dir, snapshotName(30))) + + for rollbackWindow := uint64(0); rollbackWindow <= 40; rollbackWindow++ { + floor := store.GetRollbackFloor(rollbackWindow) + before := snapshotVersions(t, dir) + require.NoError(t, store.PruneSnapshots(floor)) + remaining := snapshotVersions(t, dir) + + if floor == 0 { + require.Equal(t, before, remaining, + "nothing is eligible at rollbackWindow=%d, so nothing may be deleted", rollbackWindow) + continue + } + require.NotEmpty(t, remaining, "a store that had a snapshot must still have one") + require.Contains(t, remaining, int64(floor), //nolint:gosec // bounded by the loop + "the reported floor must survive at rollbackWindow=%d", rollbackWindow) + } +} + +// A cut line at or below the oldest snapshot leaves the lot. It must not be read as "delete +// everything below the newest". +func TestGCPruneSnapshotsBelowTheOldestSnapshotIsNoOp(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 1_000) + dir := s.flatkvDir() + mkSnapshots(t, dir, 500, 900) + require.NoError(t, updateCurrentSymlink(dir, snapshotName(900))) + + require.NoError(t, store.PruneSnapshots(500)) + require.Equal(t, []int64{500, 900}, snapshotVersions(t, dir)) +} + +// The collector never passes 0 — it means nothing is eligible — but the store absorbs it as a no-op +// rather than reading it as an empty range to delete against. +func TestGCPruneSnapshotsAtZeroIsNoOp(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 10) + dir := s.flatkvDir() + mkSnapshots(t, dir, 5, 10) + require.NoError(t, updateCurrentSymlink(dir, snapshotName(10))) + + require.NoError(t, store.PruneSnapshots(0)) + require.Equal(t, []int64{5, 10}, snapshotVersions(t, dir)) +} + +// A store with no snapshots has nothing to prune, which is not a failure — and notably not an error +// about the missing active symlink, since there is no deletion to protect. +func TestGCPruneSnapshotsWithoutSnapshots(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 100) + require.NoError(t, store.PruneSnapshots(10)) + + require.NoError(t, os.RemoveAll(s.flatkvDir())) + require.NoError(t, store.PruneSnapshots(10)) +} + +// The active snapshot is what the next open resolves to, so the cut line stops there even when it is +// asked to go deeper. This is the shape a crash between WriteSnapshot's rename and its symlink update +// leaves behind: snapshot 10 is on disk while "current" is still 5. Deleting 5 would leave a dangling +// symlink, which os.Readlink resolves happily, so the store would open against a directory that is +// not there. +func TestGCPruneSnapshotsKeepsActiveSnapshotBelowTheCutLine(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 10) + dir := s.flatkvDir() + mkSnapshots(t, dir, 3, 5, 10) + require.NoError(t, updateCurrentSymlink(dir, snapshotName(5))) + + require.NoError(t, store.PruneSnapshots(10)) + require.Equal(t, []int64{5, 10}, snapshotVersions(t, dir), + "the cut line stops at the active snapshot, so 3 goes and 5 stays") +} + +// Without a resolvable active snapshot there is nothing to bound the deletion by, so the prune is +// refused rather than run blind. +func TestGCPruneSnapshotsRefusesWithoutActiveSnapshot(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 10) + dir := s.flatkvDir() + mkSnapshots(t, dir, 5, 10) + require.NoError(t, os.Remove(currentPath(dir))) + + require.Error(t, store.PruneSnapshots(10)) + require.Equal(t, []int64{5, 10}, snapshotVersions(t, dir)) +} + +// The same refusal on the reporting side: a floor above the snapshot this store would resume from +// would let the WAL be pruned past the blocks that resume replays, so an unresolvable "current" holds +// the fleet's history where it is. +func TestGCRollbackFloorHoldsHistoryWithoutActiveSnapshot(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 100) + dir := s.flatkvDir() + mkSnapshots(t, dir, 10, 20) + require.Equal(t, uint64(20), store.GetRollbackFloor(80), "the newest snapshot past the window") + + require.NoError(t, os.Remove(currentPath(dir))) + require.Equal(t, uint64(0), store.GetRollbackFloor(80)) +} + +// A snapshot newer than "current" is what a crash between WriteSnapshot's rename and its symlink +// update leaves behind, and the next open takes the symlink rather than adopting the orphan. So the +// floor stops at the active snapshot: reporting the orphan would hold the WAL only from there, and +// the replay that starts at the active snapshot needs the blocks below it. +func TestGCRollbackFloorStopsAtTheActiveSnapshot(t *testing.T) { + s, store := gcStoreAtHead(t, t.TempDir(), 100) + dir := s.flatkvDir() + mkSnapshots(t, dir, 10, 50) + require.NoError(t, updateCurrentSymlink(dir, snapshotName(10))) + + require.Equal(t, uint64(10), store.GetRollbackFloor(20), + "head - window is 80, so the orphan at 50 would answer if the active snapshot did not bound it") +} + +// ExternalPruning is off unless asked for: a store built without a collector must keep pruning +// itself, since standing down with nothing to replace it grows snapshots without bound. +func TestGCExternalPruningDefaultsOff(t *testing.T) { + _, store := gcStore(t, t.TempDir()) + require.False(t, store.ExternalPruning()) + + s := &CommitStore{config: config.Config{DataDir: t.TempDir(), ExternalPruning: true}} + require.True(t, s.ExternalPruning()) +} + +// The guarantee the flag exists for: the count-based pruner and the collector are never both +// enforcing retention, because both read this one field. SnapshotKeepRecent 0 is the most +// destructive setting there is — delete every snapshot but the current one — so if anything can +// defeat the stand-down, it shows here. +func TestGCExternalPruningStandsDownSnapshotPruner(t *testing.T) { + run := func(external bool) []int64 { + dir := t.TempDir() + s := &CommitStore{ + ctx: t.Context(), + config: config.Config{ + DataDir: dir, + SnapshotKeepRecent: 0, + ExternalPruning: external, + }, + } + mkSnapshots(t, dir, 5, 10, 15) + s.pruneSnapshotsByCount(dir, 15) + return snapshotVersions(t, dir) + } + + require.Equal(t, []int64{15}, run(false), + "left to itself the count-based pruner takes everything below the current snapshot") + require.Equal(t, []int64{5, 10, 15}, run(true), + "under the collector it must not delete the snapshot being held for the rollback window") +} + +// pruneSpyWAL is a statewal.StateWAL that records the one call tryTruncateWAL makes on it. The other +// interface methods are never reached on this path, so the embedded nil satisfies the type and panics +// loudly if that ever stops being true. +type pruneSpyWAL struct { + statewal.StateWAL + pruned bool + prunedTo uint64 +} + +func (w *pruneSpyWAL) Prune(lowestBlockNumberToKeep uint64) error { + w.pruned = true + w.prunedTo = lowestBlockNumberToKeep + return nil +} + +// tryTruncateWAL is the second mechanism ExternalPruning stands down, and by the config doc the +// higher-stakes of the pair: left running under the collector it would truncate the state WAL to +// this store's own earliest snapshot while the collector holds the fleet's floor lower — pruning the +// WAL out from under SS, which replays it. Its sibling pruneSnapshotsByCount is pinned by the test +// above; this pins the WAL half so a later change cannot silently re-enable it. +// +// A real snapshot above version 0 is laid down so tryTruncateWAL has a floor to truncate to (version +// 0 short-circuits it before the guard). The observable is whether it schedules a WAL prune: it must +// with retention its own, and must not under ExternalPruning. +func TestGCExternalPruningStandsDownWALTruncation(t *testing.T) { + prune := func(external bool) *pruneSpyWAL { + s, _ := gcStore(t, t.TempDir()) + s.config.ExternalPruning = external + spy := &pruneSpyWAL{} + s.wal = spy + mkSnapshots(t, s.flatkvDir(), 5, 10) + + s.tryTruncateWAL() + return spy + } + + self := prune(false) + require.True(t, self.pruned, "with retention its own, tryTruncateWAL must prune the WAL") + require.Equal(t, uint64(5), self.prunedTo, "it truncates up to the earliest snapshot") + + require.False(t, prune(true).pruned, "under ExternalPruning tryTruncateWAL must not touch the WAL") +} + +// End to end against snapshots WriteSnapshot actually produced, including that the store still opens +// afterwards — the prune must not disturb what the next open resolves through. +func TestGCPrunesRealSnapshotsAndStoreStillOpens(t *testing.T) { + dir := t.TempDir() + cfg := config.DefaultTestConfig(t) + cfg.DataDir = filepath.Join(dir, flatkvRootDir) + cfg.SnapshotInterval = 2 + // Configured the way a collector-managed store actually is. SnapshotKeepRecent is deliberately + // left at its default of 1 to show ExternalPruning overrides it rather than cooperating with it: + // under that count, snapshots 0 and 2 would be gone before the collector ever saw them. + cfg.ExternalPruning = true + + s, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + require.NoError(t, s.LoadLatest()) + + for range 6 { + commitAndCheck(t, s) + } + // A fresh store starts at the initial snapshot; the interval adds one every other block. + require.Equal(t, []int64{0, 2, 4, 6}, snapshotVersions(t, cfg.DataDir)) + + store, ok := any(s).(gc.PrunableStore) + require.True(t, ok, "a FlatKV commit store must satisfy gc.PrunableStore") + + latest, err := store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(6), latest) + + floor := store.GetRollbackFloor(1) + require.Equal(t, uint64(4), floor, "newest snapshot at or below head - 1") + + require.NoError(t, store.PruneSnapshots(floor)) + require.Equal(t, []int64{4, 6}, snapshotVersions(t, cfg.DataDir)) + + require.NoError(t, s.Close()) + + cfg2 := config.DefaultTestConfig(t) + cfg2.DataDir = cfg.DataDir + s2, err := newCommitStoreWithWAL(t.Context(), cfg2) + require.NoError(t, err) + require.NoError(t, s2.LoadLatest()) + require.Equal(t, int64(6), s2.Version(), "the pruned store reopens at its committed version") + require.NoError(t, s2.Close()) +} + +// The collector runs on its own goroutine while the store commits on another. Only the race detector +// can judge this: CommitStore keeps its committed version in a plain field that Commit advances under +// the write lock, and WriteSnapshot rewrites the snapshot directory the other two methods scan. +func TestGCConcurrentWithCommitter(t *testing.T) { + dir := t.TempDir() + cfg := config.DefaultTestConfig(t) + cfg.DataDir = filepath.Join(dir, flatkvRootDir) + cfg.SnapshotInterval = 5 + cfg.SnapshotKeepRecent = 1_000 // the collector is the only deleter here + + s, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + require.NoError(t, s.LoadLatest()) + defer func() { require.NoError(t, s.Close()) }() + + store := gc.PrunableStore(s) + + const blocks = 30 + done := make(chan struct{}) + go func() { + defer close(done) + for version := int64(1); version <= blocks; version++ { + // Assertions are not allowed off the main test goroutine; fail loudly instead. + if _, err := s.Commit(version); err != nil { + panic(err) + } + } + }() + + // Drive prune cycles exactly as the collector does, off the committer's goroutine. + for range blocks { + head, err := store.GetLatestBlock() + require.NoError(t, err) + require.LessOrEqual(t, head, uint64(blocks)) + // No assertion on the floor itself: the committer advances the head between these calls, + // so any relation to the head read above is racy by construction. What is under test is + // that reading and pruning concurrently with a committer is safe at all. + floor := store.GetRollbackFloor(10) + require.NoError(t, store.PruneSnapshots(floor)) + require.NoError(t, store.PruneHistory(floor)) + } + <-done +} diff --git a/sei-db/state_db/statewal/state_wal_gc.go b/sei-db/state_db/statewal/state_wal_gc.go new file mode 100644 index 0000000000..5fc16eab86 --- /dev/null +++ b/sei-db/state_db/statewal/state_wal_gc.go @@ -0,0 +1,62 @@ +package statewal + +import ( + "fmt" + + "github.com/sei-protocol/sei-chain/sei-db/management/gc" +) + +// stateWALImpl joins the shared prune cycle as a contiguous store: it holds every block from its +// retention floor up to its head, so any height in between can serve a rollback. +// +// This is the only surface of stateWALImpl usable from another goroutine (see the type doc). Every +// method here reads a constant, a single atomic, or the WAL underneath. +// +// SC and SS must be managed by the same collector, since they replay from this WAL and it is their +// reported floors that keep the blocks they need. +var _ gc.PrunableStore = (*stateWALImpl)(nil) + +func (w *stateWALImpl) Name() string { + return "StateWAL" +} + +// ExternalPruning is unconditionally true: the WAL prunes only when told to, so it has no pruner of +// its own for the collector to collide with. +func (w *stateWALImpl) ExternalPruning() bool { + return true +} + +// PruneHistory schedules removal of the blocks below blockNumber, going straight to the underlying +// WAL. Prune is the equivalent for the WAL's own owner; this one is safe to call from the collector's +// goroutine, and leaves the WAL usable on failure rather than bricking it. +func (w *stateWALImpl) PruneHistory(blockNumber uint64) error { + if err := w.wal.PruneBefore(blockNumber); err != nil { + return fmt.Errorf("failed to prune state WAL below block %d: %w", blockNumber, err) + } + return nil +} + +// PruneSnapshots does nothing: the WAL keeps no snapshots. +func (w *stateWALImpl) PruneSnapshots(uint64) error { + return nil +} + +// GetRollbackFloor returns head - rollbackWindow, measured against this WAL's own head. It returns +// 0 — nothing here is eligible for pruning — when the window is deeper than the whole WAL, which +// includes a freshly created one, or when the head cannot be read. +func (w *stateWALImpl) GetRollbackFloor(rollbackWindow uint64) uint64 { + head, err := w.GetLatestBlock() + if err != nil { + return 0 + } + if head <= rollbackWindow { + return 0 + } + return head - rollbackWindow +} + +// GetLatestBlock returns the highest block ended by SignalEndOfBlock, or 0 when none has been. A +// block written but not yet ended is excluded: it is still buffered rather than a record. +func (w *stateWALImpl) GetLatestBlock() (uint64, error) { + return w.lastCompletedBlock.Load(), nil +} diff --git a/sei-db/state_db/statewal/state_wal_gc_test.go b/sei-db/state_db/statewal/state_wal_gc_test.go new file mode 100644 index 0000000000..4e4acad450 --- /dev/null +++ b/sei-db/state_db/statewal/state_wal_gc_test.go @@ -0,0 +1,253 @@ +package statewal + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/management/gc" + "github.com/sei-protocol/sei-chain/sei-db/proto" +) + +// openWALForGC opens a WAL and returns it as the collector sees it, closing it on cleanup. +func openWALForGC(t *testing.T, cfg *Config) (StateWAL, gc.PrunableStore) { + t.Helper() + w := openWAL(t, cfg) + t.Cleanup(func() { require.NoError(t, w.Close()) }) + store, ok := w.(gc.PrunableStore) + require.True(t, ok, "a state WAL must satisfy gc.PrunableStore") + return w, store +} + +// The head is the last block ended by SignalEndOfBlock. A block that has been written but not ended +// is still buffered rather than a record, so counting it would put this store's head — and with it +// the floor it reports — one block above what the WAL can replay. +func TestGCLatestBlockCountsOnlyCompletedBlocks(t *testing.T) { + w, store := openWALForGC(t, testConfig(t.TempDir())) + + latest, err := store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(0), latest, "an empty WAL has no head") + + writeBlock(t, w, 1) + writeBlock(t, w, 2) + latest, err = store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(2), latest) + + // Block 3 is written but not ended, so it is not yet in the WAL. + require.NoError(t, w.Write(3, []*proto.NamedChangeSet{makeChangeSet("evm", []byte{3}, []byte{3})})) + latest, err = store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(2), latest, "a block in progress must not count as the head") + + require.NoError(t, w.SignalEndOfBlock()) + latest, err = store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(3), latest) +} + +// The head survives a reopen: a WAL that reported 0 after a restart would report a floor of 0 with +// it, holding the whole fleet's history where it is until the head is rebuilt. +func TestGCLatestBlockRecoveredOnOpen(t *testing.T) { + cfg := testConfig(t.TempDir()) + + w := openWAL(t, cfg) + for block := uint64(1); block <= 5; block++ { + writeBlock(t, w, block) + } + require.NoError(t, w.Flush()) + require.NoError(t, w.Close()) + + _, store := openWALForGC(t, cfg) + latest, err := store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(5), latest) +} + +// The WAL keeps no snapshots — it is what snapshots replay forward from — so its half of the split +// contract is a no-op that must not touch the stored range. Its depth is instead set by SC/SS +// answering their oldest live snapshot as a rollback floor, which reaches the WAL as the shared +// minimum PruneHistory receives. +func TestGCPruneSnapshotsIsANoOp(t *testing.T) { + cfg := testConfig(t.TempDir()) + cfg.TargetFileSize = 1 + w, store := openWALForGC(t, cfg) + + for block := uint64(1); block <= 10; block++ { + writeBlock(t, w, block) + } + require.NoError(t, w.Flush()) + + require.NoError(t, store.PruneSnapshots(8)) + require.NoError(t, w.Flush()) + + ok, first, last, err := w.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(1), first, "a snapshot prune must not move the WAL floor") + require.Equal(t, uint64(10), last) +} + +// A contiguous store resolves the window against its own head, and reports 0 where the window is +// deeper than that head — including on an empty WAL, where the answer holds the fleet's history +// where it is. +func TestGCRollbackFloorComesFromOwnHead(t *testing.T) { + w, store := openWALForGC(t, testConfig(t.TempDir())) + + require.Equal(t, uint64(0), store.GetRollbackFloor(42), "an empty WAL needs every block kept") + + for block := uint64(1); block <= 5; block++ { + writeBlock(t, w, block) + } + require.Equal(t, uint64(5), store.GetRollbackFloor(0)) + require.Equal(t, uint64(2), store.GetRollbackFloor(3)) + require.Equal(t, uint64(0), store.GetRollbackFloor(1_000), + "a window deeper than the head leaves nothing eligible for pruning") +} + +// PruneHistory goes straight to the WAL, so reclamation does not wait on write traffic. A WAL that has +// stopped receiving blocks is exactly where a deferred prune would strand history indefinitely, so no +// further block is written here before the result is checked. +func TestGCPruneHistoryDoesNotWaitForTheNextBlock(t *testing.T) { + cfg := testConfig(t.TempDir()) + cfg.TargetFileSize = 1 // seal after every block, so whole-file pruning can act per block + w, store := openWALForGC(t, cfg) + + for block := uint64(1); block <= 10; block++ { + writeBlock(t, w, block) + } + require.NoError(t, w.Flush()) + + require.NoError(t, store.PruneHistory(6)) + require.NoError(t, w.Flush()) // the prune is async; order behind it + + ok, first, last, err := w.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(6), first) + require.Equal(t, uint64(10), last) + + require.Equal(t, []uint64{6, 7, 8, 9, 10}, collectBlocks(t, w, 6, 10)) +} + +// Cycles are independent, so a floor lower than one already applied must not walk back the prune it +// performed. Nothing here enforces that — a prune only ever deletes — but the collector is free to +// issue a lower floor after a rollback, and this pins that doing so is harmless rather than a way to +// resurrect blocks or corrupt the range. +func TestGCPruneHistoryIgnoresALowerFloor(t *testing.T) { + cfg := testConfig(t.TempDir()) + cfg.TargetFileSize = 1 + w, store := openWALForGC(t, cfg) + + for block := uint64(1); block <= 10; block++ { + writeBlock(t, w, block) + } + require.NoError(t, w.Flush()) + + require.NoError(t, store.PruneHistory(8)) + require.NoError(t, store.PruneHistory(3)) + require.NoError(t, w.Flush()) + + ok, first, _, err := w.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(8), first, "a later, lower floor must not undo the higher one") +} + +// The collector runs on its own goroutine while the WAL's owner writes blocks on another, which is +// the whole reason this surface is separate from the writer-facing one. Only the race detector can +// judge it: stateWALImpl keeps its state in plain fields on the assumption of a single caller, and +// what makes the prune safe to issue from here is seiwal.WAL's concurrency carve-out for PruneBefore. +func TestGCConcurrentWithWriter(t *testing.T) { + cfg := testConfig(t.TempDir()) + cfg.TargetFileSize = 1 + w, store := openWALForGC(t, cfg) + + const blocks = 60 + done := make(chan struct{}) + go func() { + defer close(done) + for block := uint64(1); block <= blocks; block++ { + // Assertions are not allowed off the main test goroutine; fail loudly instead. + cs := []*proto.NamedChangeSet{makeChangeSet("evm", []byte{byte(block)}, []byte{byte(block)})} + if err := w.Write(block, cs); err != nil { + panic(err) + } + if err := w.SignalEndOfBlock(); err != nil { + panic(err) + } + } + }() + + // Drive prune cycles exactly as the collector does, off the writer's goroutine. + for range blocks { + head, err := store.GetLatestBlock() + require.NoError(t, err) + require.LessOrEqual(t, head, uint64(blocks)) + if head > 20 { + require.NoError(t, store.PruneHistory(store.GetRollbackFloor(20))) + } + } + <-done + + ok, first, last, err := w.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(blocks), last) + require.LessOrEqual(t, first, uint64(blocks-20), "pruning must never outrun the floor it was given") +} + +// A prune issued before a close is ordered ahead of it rather than dropped, and survives into the next +// session. What must not happen is the close failing, or the WAL bricking, because of it. +func TestGCPruneHistoryBeforeClose(t *testing.T) { + cfg := testConfig(t.TempDir()) + cfg.TargetFileSize = 1 + + w := openWAL(t, cfg) + for block := uint64(1); block <= 5; block++ { + writeBlock(t, w, block) + } + require.NoError(t, w.(gc.PrunableStore).PruneHistory(4)) + require.NoError(t, w.Close()) + + w2, store := openWALForGC(t, cfg) + ok, first, last, err := w2.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(4), first) + require.Equal(t, uint64(5), last) + + latest, err := store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(5), latest) +} + +// The collector can still be mid-cycle when the node shuts down, so a prune arriving after the close +// must be reported rather than panic, and must not brick the WAL on the way out — PruneHistory declines +// to write fatalErr precisely because the writer reads it unsynchronized. +func TestGCPruneHistoryAfterClose(t *testing.T) { + cfg := testConfig(t.TempDir()) + cfg.TargetFileSize = 1 + + w := openWAL(t, cfg) + for block := uint64(1); block <= 5; block++ { + writeBlock(t, w, block) + } + store, ok := w.(gc.PrunableStore) + require.True(t, ok) + require.NoError(t, w.Close()) + + require.Error(t, store.PruneHistory(4)) + + // The head is still readable, and the close committed what was written. + latest, err := store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(5), latest) + + w2, _ := openWALForGC(t, cfg) + _, first, last, err := w2.GetStoredRange() + require.NoError(t, err) + require.Equal(t, uint64(1), first) + require.Equal(t, uint64(5), last) +} diff --git a/sei-db/state_db/statewal/state_wal_impl.go b/sei-db/state_db/statewal/state_wal_impl.go index 5dbc9e7de1..416ec606dd 100644 --- a/sei-db/state_db/statewal/state_wal_impl.go +++ b/sei-db/state_db/statewal/state_wal_impl.go @@ -3,6 +3,7 @@ package statewal import ( "errors" "fmt" + "sync/atomic" "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/seiwal" @@ -12,7 +13,9 @@ var _ StateWAL = (*stateWALImpl)(nil) // A WAL for storing state changesets by block number. // -// Not safe for concurrent use; see the StateWAL interface doc. +// Not safe for concurrent use; see the StateWAL interface doc. The gc.PrunableStore surface in +// state_wal_gc.go is the one exception: it runs on the collector's goroutine, and touches only +// lastCompletedBlock and the WAL underneath. type stateWALImpl struct { // The underlying generic WAL, keyed by block number, whose payload is a block's changesets. wal seiwal.WAL[[]*proto.NamedChangeSet] @@ -39,6 +42,14 @@ type stateWALImpl struct { // end-of-block. Ownership is handed to the WAL at end-of-block and a fresh buffer starts for the next // block, so the serialization goroutine never races the wrapper over the backing array. buf []*proto.NamedChangeSet + + // The highest block that has been ended by SignalEndOfBlock, and so is actually a record in the WAL. + // Distinct from currentBlock, which may name a block still accumulating in buf. Atomic because the + // garbage collector reads it off-goroutine (GetLatestBlock); the writer is its only mutator. + // + // 0 also means no block has completed yet, so a WAL whose only completed block is block 0 is + // indistinguishable from an empty one. + lastCompletedBlock atomic.Uint64 } // New opens (or creates) a state WAL in the configured directory, recovering any files left behind by a @@ -113,6 +124,7 @@ func newStateWAL(wal seiwal.WAL[[]*proto.NamedChangeSet]) (StateWAL, error) { w.currentBlock = last w.currentBlockEnded = true w.hasCurrentBlock = true + w.lastCompletedBlock.Store(last) } return w, nil } @@ -156,6 +168,7 @@ func (w *stateWALImpl) SignalEndOfBlock() error { return w.fail(fmt.Errorf("failed to append block %d: %w", w.currentBlock, err)) } w.currentBlockEnded = true + w.lastCompletedBlock.Store(w.currentBlock) w.buf = nil // hand ownership to the WAL; the next block starts a fresh buffer return nil } diff --git a/sei-tendermint/config/autobahn.go b/sei-tendermint/config/autobahn.go index c042a9afc1..b4c7b97042 100644 --- a/sei-tendermint/config/autobahn.go +++ b/sei-tendermint/config/autobahn.go @@ -125,16 +125,16 @@ func (c AutobahnBlockDBConfig) Validate() error { // LittBlockConfig returns littblock.DefaultConfig(dir) with this config's // optional overrides applied. Fsync is always forced on. -func (c AutobahnBlockDBConfig) LittBlockConfig(dir string) (littblock.LittBlockConfig, error) { +func (c AutobahnBlockDBConfig) LittBlockConfig(dir string) (littblock.BlockDBConfig, error) { if err := c.Validate(); err != nil { - return littblock.LittBlockConfig{}, err + return littblock.BlockDBConfig{}, err } cfg, err := littblock.DefaultConfig(dir) if err != nil { - return littblock.LittBlockConfig{}, fmt.Errorf("littblock.DefaultConfig: %w", err) + return littblock.BlockDBConfig{}, fmt.Errorf("littblock.DefaultConfig: %w", err) } if r, ok := c.Retention.Get(); ok { - cfg.Retention = r.Duration() + cfg.RetentionTime = r.Duration() } if p, ok := c.GCPeriod.Get(); ok { cfg.Litt.GCPeriod = p.Duration() diff --git a/sei-tendermint/config/autobahn_test.go b/sei-tendermint/config/autobahn_test.go index b6e4b815a7..d9bf43f679 100644 --- a/sei-tendermint/config/autobahn_test.go +++ b/sei-tendermint/config/autobahn_test.go @@ -38,7 +38,7 @@ func TestAutobahnBlockDBConfig_LittBlockConfig(t *testing.T) { GCPeriod: utils.Some(utils.Duration(wantGCPeriod)), }).LittBlockConfig(dir) require.NoError(t, err) - require.Equal(t, wantRetention, cfg.Retention) + require.Equal(t, wantRetention, cfg.RetentionTime) require.Equal(t, wantGCPeriod, cfg.Litt.GCPeriod) require.True(t, cfg.Litt.Fsync) } diff --git a/sei-tendermint/internal/autobahn/data/state_recovery_test.go b/sei-tendermint/internal/autobahn/data/state_recovery_test.go index 6e7ebe9e31..251c1b51c0 100644 --- a/sei-tendermint/internal/autobahn/data/state_recovery_test.go +++ b/sei-tendermint/internal/autobahn/data/state_recovery_test.go @@ -440,7 +440,7 @@ func TestRecoveryAfterPruneNoGC(t *testing.T) { // Write both QCs and all their blocks to the DB. cfg1 := utils.OrPanic1(littblock.DefaultConfig(dir)) - cfg1.Retention = time.Nanosecond + cfg1.RetentionTime = time.Nanosecond db1 := utils.OrPanic1(littblock.NewBlockDB(cfg1)) writeToBlockDB(t, db1, []*types.FullCommitQC{qc1, qc2}, [][]*types.Block{blocks1, blocks2}) @@ -450,7 +450,7 @@ func TestRecoveryAfterPruneNoGC(t *testing.T) { // Reopen the same dir without ForceGC — pruned entries may still be present. cfg2 := utils.OrPanic1(littblock.DefaultConfig(dir)) - cfg2.Retention = time.Nanosecond + cfg2.RetentionTime = time.Nanosecond db2 := utils.OrPanic1(littblock.NewBlockDB(cfg2)) t.Cleanup(func() { _ = db2.Close() }) diff --git a/sei-tendermint/internal/autobahn/data/state_test.go b/sei-tendermint/internal/autobahn/data/state_test.go index 23ae335fb7..d1893ef689 100644 --- a/sei-tendermint/internal/autobahn/data/state_test.go +++ b/sei-tendermint/internal/autobahn/data/state_test.go @@ -52,7 +52,7 @@ func snapshot(s *State) Snapshot { func newTestBlockDB(t *testing.T, dir string) types.BlockDB { t.Helper() cfg := utils.OrPanic1(littblock.DefaultConfig(dir)) - cfg.Retention = time.Nanosecond + cfg.RetentionTime = time.Nanosecond db := utils.OrPanic1(littblock.NewBlockDB(cfg)) t.Cleanup(func() { _ = db.Close() }) return db diff --git a/sei-tendermint/node/setup_test.go b/sei-tendermint/node/setup_test.go index 62668485a0..8aed49ae95 100644 --- a/sei-tendermint/node/setup_test.go +++ b/sei-tendermint/node/setup_test.go @@ -119,7 +119,7 @@ func TestBuildGigaConfig_BlockDBOverrides(t *testing.T) { littCfg, err := fc.BlockDB.LittBlockConfig(filepath.Join(dir, "blockdb")) require.NoError(t, err) require.NotNil(t, littCfg.Litt) - assert.Equal(t, 30*time.Second, littCfg.Retention) + assert.Equal(t, 30*time.Second, littCfg.RetentionTime) assert.Equal(t, 5*time.Second, littCfg.Litt.GCPeriod) assert.True(t, littCfg.Litt.Fsync) } @@ -141,7 +141,7 @@ func TestBuildGigaConfig_BlockDBOmittedKeepsDefaults(t *testing.T) { littCfg, err := config.AutobahnBlockDBConfig{}.LittBlockConfig(filepath.Join(dir, "blockdb")) require.NoError(t, err) require.NotNil(t, littCfg.Litt) - assert.Equal(t, 24*time.Hour, littCfg.Retention) + assert.Equal(t, time.Hour, littCfg.RetentionTime) assert.True(t, littCfg.Litt.Fsync) }