[Storehouse] 007 Payloadless Checkpoint (v7)

cmd/execution_builder.go

@@ -1484,6 +1484,51 @@ func (exeNode *ExecutionNode) LoadBootstrapper(node *NodeConfig) error {
 			return fmt.Errorf("could not load bootstrap state from checkpoint file: %w", err)
 		}
 
+		// In payloadless (V7) mode the spork only produces a V6 root.checkpoint.
+		// Convert it to a V7 root checkpoint here so the payloadless ledger can
+		// seed its forest from it on first boot; later restarts reuse this file
+		// (or a newer numbered V7 checkpoint written by the compactor). The
+		// HasRootCheckpointV7 guard keeps a re-entry after an interrupted
+		// bootstrap from hitting ConvertCheckpointV6ToV7's "output exists" check.
+		//
+		// TODO: ConvertCheckpointV6ToV7 reads the entire V6 forest into memory
+		// before emitting V7, a memory/time spike at first boot for mainnet-scale
+		// root checkpoints. A future optimization is to convert subtrie-by-subtrie
+		// without loading the whole forest.
+		if exeNode.exeConf.payloadless {
+			triedir := exeNode.exeConf.triedir
+			v7RootFileName := modelbootstrap.FilenameWALRootCheckpoint + wal.V7FileSuffix
+			hasV7Root, err := wal.HasRootCheckpointV7(triedir)
+			if err != nil {
+				return fmt.Errorf("could not check for V7 root checkpoint: %w", err)
+			}
+			if !hasV7Root {
+				// HasRootCheckpointV7 only looks for the header file, which the writer emits
+				// last. If a previous attempt died mid-conversion — a realistic outcome, since
+				// this is the memory-heavy step of bootstrap — its part files are still on
+				// disk and would trip ConvertCheckpointV6ToV7's refusal to clobber existing
+				// output, leaving the node unable to boot without manual cleanup. Discarding
+				// that partial output is always safe: the V6 source is untouched, and a
+				// checkpoint whose header was never written is unusable anyway.
+				err = wal.DeleteCheckpointFiles(triedir, v7RootFileName)
+				if err != nil {
+					return fmt.Errorf("could not remove partially converted V7 root checkpoint: %w", err)
+				}
+
+				err = wal.ConvertCheckpointV6ToV7(
+					triedir,
+					modelbootstrap.FilenameWALRootCheckpoint,
+					triedir,
+					v7RootFileName,
+					node.Logger,
+					16,
+				)
+				if err != nil {
+					return fmt.Errorf("could not convert V6 root checkpoint to V7 for payloadless node: %w", err)
+				}
+			}
+		}
+
 		err = bootstrapper.BootstrapExecutionDatabase(node.StorageLockMgr, node.ProtocolDB, node.RootSeal)
 		if err != nil {
 			return fmt.Errorf("could not bootstrap execution database: %w", err)

cmd/util/cmd/checkpoint-convert-v7/cmd.go

@@ -0,0 +1,100 @@
+package checkpoint_convert_v7
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+
+	"github.com/onflow/flow-go/ledger/complete/wal"
+)
+
+var (
+	flagCheckpointDir string
+	flagCheckpoint    string
+	flagOutputDir     string
+	flagOutput        string
+	flagNWorker       uint
+)
+
+// Cmd converts a V6 checkpoint to a V7 (payloadless) checkpoint by reading
+// the V6 part files, projecting every leaf into a payload-hash leaf, and
+// re-encoding with the V7 (payloadless) writer.
+var Cmd = &cobra.Command{
+	Use:   "checkpoint-convert-v7",
+	Short: "Convert a V6 checkpoint to a V7 (payloadless) checkpoint.",
+	Long: `Convert a V6 checkpoint to a V7 (payloadless) checkpoint.
+
+The V6 checkpoint header file and its 17 part files (subtrie + top-trie) must
+all be present. The V7 output uses the same checkpoint number with the
+".v7" suffix (e.g. "checkpoint.00000100" -> "checkpoint.00000100.v7") so the
+two formats can coexist in the same directory.
+
+Conversion preserves trie root hashes: every V7 trie produced has the same
+root hash as the corresponding V6 trie. The 16 V7 subtrie part files are
+encoded in parallel using --nworker goroutines.`,
+	Run: run,
+}
+
+func init() {
+	Cmd.Flags().StringVar(&flagCheckpointDir, "checkpoint-dir", "",
+		"directory containing the V6 checkpoint files (required)")
+	_ = Cmd.MarkFlagRequired("checkpoint-dir")
+
+	Cmd.Flags().StringVar(&flagCheckpoint, "checkpoint", "",
+		"V6 checkpoint header filename, e.g. \"checkpoint.00000100\" (required)")
+	_ = Cmd.MarkFlagRequired("checkpoint")
+
+	Cmd.Flags().StringVar(&flagOutputDir, "output-dir", "",
+		"directory to write the V7 checkpoint files to (default: --checkpoint-dir)")
+
+	Cmd.Flags().StringVar(&flagOutput, "output", "",
+		"V7 output filename. Default: input filename + \".v7\".")
+
+	Cmd.Flags().UintVar(&flagNWorker, "nworker", 16,
+		"number of subtrie files to encode in parallel (valid range [1, 16])")
+}
+
+func run(*cobra.Command, []string) {
+	outputDir := flagOutputDir
+	if outputDir == "" {
+		outputDir = flagCheckpointDir
+	}
+
+	outputFile := flagOutput
+	if outputFile == "" {
+		outputFile = defaultV7Filename(flagCheckpoint)
+	}
+
+	log.Info().
+		Str("checkpoint_dir", flagCheckpointDir).
+		Str("checkpoint", flagCheckpoint).
+		Str("output_dir", outputDir).
+		Str("output", outputFile).
+		Uint("nworker", flagNWorker).
+		Msg("converting V6 checkpoint to V7")
+
+	err := wal.ConvertCheckpointV6ToV7(
+		flagCheckpointDir,
+		flagCheckpoint,
+		outputDir,
+		outputFile,
+		log.Logger,
+		flagNWorker,
+	)
+	if err != nil {
+		log.Fatal().Err(err).Msg("checkpoint conversion failed")
+	}
+
+	log.Info().Msgf("wrote V7 checkpoint to %s", filepath.Join(outputDir, outputFile))
+}
+
+// defaultV7Filename returns the default V7 output filename for a given V6
+// checkpoint filename: append ".v7" unless it already carries the suffix.
+func defaultV7Filename(v6Name string) string {
+	if strings.HasSuffix(v6Name, wal.V7FileSuffix) {
+		return v6Name
+	}
+	return v6Name + wal.V7FileSuffix
+}

cmd/util/cmd/root.go

@@ -15,6 +15,7 @@ import (
 	bootstrap_execution_state_payloads "github.com/onflow/flow-go/cmd/util/cmd/bootstrap-execution-state-payloads"
 	check_storage "github.com/onflow/flow-go/cmd/util/cmd/check-storage"
 	checkpoint_collect_stats "github.com/onflow/flow-go/cmd/util/cmd/checkpoint-collect-stats"
+	checkpoint_convert_v7 "github.com/onflow/flow-go/cmd/util/cmd/checkpoint-convert-v7"
 	checkpoint_list_tries "github.com/onflow/flow-go/cmd/util/cmd/checkpoint-list-tries"
 	checkpoint_trie_stats "github.com/onflow/flow-go/cmd/util/cmd/checkpoint-trie-stats"
 	compare_debug_tx "github.com/onflow/flow-go/cmd/util/cmd/compare-debug-tx"
@@ -107,6 +108,7 @@ func addCommands() {
 	rootCmd.AddCommand(checkpoint_list_tries.Cmd)
 	rootCmd.AddCommand(checkpoint_trie_stats.Cmd)
 	rootCmd.AddCommand(checkpoint_collect_stats.Cmd)
+	rootCmd.AddCommand(checkpoint_convert_v7.Cmd)
 	rootCmd.AddCommand(read_badger.RootCmd)
 	rootCmd.AddCommand(read_protocol_state.RootCmd)
 	rootCmd.AddCommand(ledger_json_exporter.Cmd)

cmd/util/common/checkpoint.go

@@ -3,12 +3,14 @@ package common
 import (
 	"fmt"
 	"path/filepath"
+	"strings"
 
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 
 	"github.com/onflow/flow-go/ledger"
 	"github.com/onflow/flow-go/ledger/complete/wal"
+	modelbootstrap "github.com/onflow/flow-go/model/bootstrap"
 	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/state/protocol"
 	"github.com/onflow/flow-go/storage"
@@ -32,7 +34,13 @@ func FindHeightsByCheckpoints(
 
 	// find all trie root hashes in the checkpoint file
 	dir, fileName := filepath.Split(checkpointFilePath)
-	hashes, err := wal.ReadTriesRootHash(logger, dir, fileName)
+	var hashes []ledger.RootHash
+	var err error
+	if strings.HasSuffix(fileName, wal.V7FileSuffix) {
+		hashes, err = wal.ReadTriesRootHashV7(logger, dir, fileName)
+	} else {
+		hashes, err = wal.ReadTriesRootHash(logger, dir, fileName)
+	}
 	if err != nil {
 		return 0, flow.DummyStateCommitment, 0,
 			fmt.Errorf("could not read trie root hashes from checkpoint file %v: %w",
@@ -130,15 +138,36 @@ func GenerateProtocolSnapshotForCheckpoint(
 
 // findLatestCheckpointFilePath finds the latest checkpoint file in the given directory
 // it returns the header file name of the latest checkpoint file
+//
+// The returned name is version-specific: a V7 (payloadless) checkpoint carries the
+// [wal.V7FileSuffix], a V6 checkpoint does not. Rendering the wrong version's name would point at a
+// file that does not exist, since the two versions coexist in a payloadless triedir.
+//
+// No error returns are expected during normal operation.
 func findLatestCheckpointFilePath(checkpointDir string) (string, error) {
-	_, last, err := wal.ListCheckpoints(checkpointDir)
+	_, last, err := wal.ListCheckpointsWithInfo(checkpointDir)
 	if err != nil {
 		return "", fmt.Errorf("could not list checkpoints in directory %v: %w", checkpointDir, err)
 	}
 
-	fileName := wal.NumberToFilename(last)
-	if last < 0 {
-		fileName = "root.checkpoint"
+	// No numbered checkpoint: fall back to the root checkpoint, which the listing above excludes.
+	// Prefer the V7 root checkpoint, because a payloadless triedir has both (the V7 one is converted
+	// from the V6 one at bootstrap) while a full triedir only has the V6 one.
+	if last == nil {
+		hasV7Root, err := wal.HasRootCheckpointV7(checkpointDir)
+		if err != nil {
+			return "", fmt.Errorf("could not check for V7 root checkpoint in directory %v: %w", checkpointDir, err)
+		}
+		fileName := modelbootstrap.FilenameWALRootCheckpoint
+		if hasV7Root {
+			fileName += wal.V7FileSuffix
+		}
+		return filepath.Join(checkpointDir, fileName), nil
+	}
+
+	fileName := wal.NumberToFilename(last.Number)
+	if last.Version == wal.VersionV7 {
+		fileName = wal.NumberToFilenameV7(last.Number)
 	}
 
 	checkpointFilePath := filepath.Join(checkpointDir, fileName)

ledger/complete/compactor.go

@@ -184,7 +184,7 @@ func (c *Compactor) run() {
 		activeSegmentNum = -1
 	}
 
-	lastCheckpointNum, err := c.checkpointer.LatestCheckpoint()
+	lastCheckpointNum, err := c.checkpointer.LatestCheckpointV6()
 	if err != nil {
 		c.logger.Error().Err(err).Msg("compactor failed to get last checkpoint number")
 		lastCheckpointNum = -1
@@ -311,7 +311,7 @@ func (c *Compactor) checkpoint(ctx context.Context, tries []*trie.MTrie, checkpo
 	default:
 	}
 
-	err = cleanupCheckpoints(c.checkpointer, int(c.checkpointsToKeep))
+	err = cleanupCheckpointsV6(c.checkpointer, int(c.checkpointsToKeep))
 	if err != nil {
 		return &removeCheckpointError{err: err}
 	}
@@ -361,25 +361,30 @@ func createCheckpoint(checkpointer *realWAL.Checkpointer, logger zerolog.Logger,
 	return nil
 }
 
-// cleanupCheckpoints deletes prior checkpoint files if needed.
-// Since the function is side-effect free, all failures are simply a no-op.
-func cleanupCheckpoints(checkpointer *realWAL.Checkpointer, checkpointsToKeep int) error {
+// cleanupCheckpointsV6 deletes prior V6 checkpoint files if needed.
+//
+// Retention is applied per checkpoint type: this V6 compactor only counts and
+// removes V6 checkpoints, leaving any V7 (payloadless) files in the same
+// directory to be governed by the payloadless compactor's own retention. A
+// `checkpointsToKeep` of N therefore permits N V6 and N V7 checkpoints to
+// coexist.
+func cleanupCheckpointsV6(checkpointer *realWAL.Checkpointer, checkpointsToKeep int) error {
 	// Don't list checkpoints if we keep them all
 	if checkpointsToKeep == 0 {
 		return nil
 	}
-	checkpoints, err := checkpointer.Checkpoints()
+	checkpoints, err := checkpointer.CheckpointsV6()
 	if err != nil {
-		return fmt.Errorf("cannot list checkpoints: %w", err)
+		return fmt.Errorf("cannot list V6 checkpoints: %w", err)
 	}
 	if len(checkpoints) > int(checkpointsToKeep) {
 		// if condition guarantees this never fails
 		checkpointsToRemove := checkpoints[:len(checkpoints)-int(checkpointsToKeep)]
 
 		for _, checkpoint := range checkpointsToRemove {
-			err := checkpointer.RemoveCheckpoint(checkpoint)
+			err := checkpointer.RemoveCheckpointV6(checkpoint)
 			if err != nil {
-				return fmt.Errorf("cannot remove checkpoint %d: %w", checkpoint, err)
+				return fmt.Errorf("cannot remove V6 checkpoint %d: %w", checkpoint, err)
 			}
 		}
 	}