feat(consensus): Implement and wire halt-height and halt-time

.github/workflows/nightly.yml

@@ -77,9 +77,39 @@ jobs:
         env:
           GOPATH: /home/runner/go
 
+  # Halt/swap/resume upgrade smoke test: a multi-validator devnet halts (by
+  # height, then by time in a second run), restarts from the same data dirs,
+  # and must resume with tx load flowing. The local build serves as both the
+  # old and new beacond, gating the halt/swap/restart mechanics.
+  test-halt-swap-resume:
+    runs-on:
+      labels: ubuntu-24.04-beacon-kit
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          submodules: recursive
+      - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
+        with:
+          go-version-file: go.mod
+          check-latest: true
+          cache-dependency-path: "**/*.sum"
+      - name: Install Foundry (cast drives the tx load)
+        uses: foundry-rs/foundry-toolchain@c7450ba673e133f5ee30098b3b54f444d3a2ca2d # v1.8.0
+      - run: make test-halt-swap-resume test-halt-swap-resume-time
+        env:
+          GOPATH: /home/runner/go
+      - name: Upload test logs
+        if: failure()
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
+        with:
+          name: upgrade-test-logs
+          path: .tmp/upgrade-test/logs
+
   # Post to Slack only when the nightly run fails.
   notify-slack:
-    needs: [pipeline, test-deps]
+    needs: [pipeline, test-deps, test-halt-swap-resume]
     if: failure()
     runs-on:
       labels: ubuntu-24.04-beacon-kit
@@ -89,7 +119,7 @@ jobs:
       - name: Post failure to Slack
         env:
           CORE_SLACK_WEBHOOK_URL: ${{ secrets.CORE_SLACK_WEBHOOK_URL }}
-          TEXT: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} nightly CI failed (pipeline=${{ needs.pipeline.result }}, test-deps=${{ needs.test-deps.result }})
+          TEXT: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} nightly CI failed (pipeline=${{ needs.pipeline.result }}, test-deps=${{ needs.test-deps.result }}, test-halt-swap-resume=${{ needs.test-halt-swap-resume.result }})
         run: |
           if [ -z "$CORE_SLACK_WEBHOOK_URL" ]; then
             echo "CORE_SLACK_WEBHOOK_URL not set; skipping Slack notification."

Makefile

@@ -8,6 +8,7 @@ include scripts/build/linting.mk
 include scripts/build/protobuf.mk
 include scripts/build/release.mk
 include scripts/build/testing.mk
+include scripts/build/halt-upgrade-test.mk
 include contracts/Makefile
 include kurtosis/Makefile
 include scripts/build/help.mk

consensus/cometbft/service/abci.go

@@ -55,6 +55,14 @@ func (s *Service) PrepareProposal(
 	_ context.Context,
 	req *cmtabci.PrepareProposalRequest,
 ) (*cmtabci.PrepareProposalResponse, error) {
+	// Once halted, propose nothing so consensus cannot decide a block past the halt point while the halt
+	// shutdown completes.
+	if s.ensureNotHalted() != nil {
+		s.logger.Info("halt point reached, returning an empty proposal", "height", req.Height)
+		//nolint:nilerr // a halted node proposes nothing instead of erroring
+		return &cmtabci.PrepareProposalResponse{Txs: [][]byte{}}, nil
+	}
+
 	// Check if ctx is still good. CometBFT does not check this.
 	if s.ctx.Err() != nil {
 		// If the context is getting cancelled, we are shutting down.
@@ -94,6 +102,14 @@ func (s *Service) ProcessProposal(
 	_ context.Context,
 	req *cmtabci.ProcessProposalRequest,
 ) (*cmtabci.ProcessProposalResponse, error) {
+	// Once halted, prevote nil on every proposal so no block past the halt point can be decided. An error
+	// return would make CometBFT panic.
+	if s.ensureNotHalted() != nil {
+		s.logger.Info("halt point reached, rejecting proposal", "height", req.Height)
+		//nolint:nilerr // a halted node votes nil instead of erroring
+		return &cmtabci.ProcessProposalResponse{Status: cmtabci.PROCESS_PROPOSAL_STATUS_REJECT}, nil
+	}
+
 	// Check if ctx is still good. CometBFT does not check this.
 	if s.ctx.Err() != nil {
 		// Node will panic on context cancel with "CONSENSUS FAILURE!!!" due to
@@ -109,6 +125,16 @@ func (s *Service) FinalizeBlock(
 	_ context.Context,
 	req *cmtabci.FinalizeBlockRequest,
 ) (*cmtabci.FinalizeBlockResponse, error) {
+	// Never execute a block past the halt point. An error return would make CometBFT panic with CONSENSUS
+	// FAILURE (consensus and blocksync both escalate FinalizeBlock errors), so park until the halt shutdown
+	// exits the process underneath this call. Checked before the ctx guard below so a post-halt block arriving
+	// mid-shutdown parks instead of panicking.
+	if err := s.ensureNotHalted(); err != nil {
+		s.logger.Info("halt point reached, refusing to finalize block", "height", req.Height)
+		s.waitForHaltShutdown()
+		return nil, err
+	}
+
 	// Check if ctx is still good. CometBFT does not check this.
 	if s.ctx.Err() != nil {
 		// Node will panic on context cancel with "CONSENSUS FAILURE!!!" due to error.
@@ -122,10 +148,9 @@ func (s *Service) FinalizeBlock(
 // Commit implements the ABCI interface. It will commit all state that exists in
 // the deliver state's multi-store and includes the resulting commit ID in the
 // returned cmtabci.ResponseCommit. Commit will set the check state based on the
-// latest header and reset the deliver state. Also, if a non-zero halt height is
-// defined in config, Commit will execute a deferred function call to check
-// against that height and gracefully halt if it matches the latest committed
-// height.
+// latest header and reset the deliver state. Also, if a non-zero halt height
+// or halt time is configured, Commit gracefully shuts the node down once the
+// committed block reaches it.
 func (s *Service) Commit(
 	_ context.Context, req *cmtabci.CommitRequest,
 ) (*cmtabci.CommitResponse, error) {

consensus/cometbft/service/commit.go

@@ -23,43 +23,120 @@ package cometbft
 
 import (
 	"fmt"
+	"os"
+	"syscall"
+	"time"
 
 	"cosmossdk.io/store/rootmulti"
 	cmtabci "github.com/cometbft/cometbft/abci/types"
+	cmtproto "github.com/cometbft/cometbft/api/cometbft/types/v1"
 )
 
 func (s *Service) commit(
 	*cmtabci.CommitRequest,
 ) (*cmtabci.CommitResponse, error) {
-	_, finalState, err := s.cachedStates.GetFinal()
-	if err != nil {
+	if _, _, err := s.cachedStates.GetFinal(); err != nil {
 		// This is unexpected since CometBFT should call Commit only
 		// after FinalizeBlock has been called. Panic appeases nilaway.
 		panic(fmt.Errorf("commit: %w", err))
 	}
 
-	header := finalState.Context().BlockHeader()
-	retainHeight := s.GetBlockRetentionHeight(header.Height)
+	// The cached state context carries an empty block header, so use the height and time captured from FinalizeBlock instead.
+	retainHeight := s.GetBlockRetentionHeight(s.finalizedHeight)
 
 	rms, ok := s.sm.GetCommitMultiStore().(*rootmulti.Store)
 	if ok {
-		rms.SetCommitHeader(header)
+		rms.SetCommitHeader(cmtproto.Header{ChainID: s.chainID, Height: s.finalizedHeight, Time: s.finalizedTime})
 	}
 	s.sm.GetCommitMultiStore().Commit()
 
 	s.cachedStates.Reset()
 
 	if s.blockDelay != nil {
-		if err = s.sm.SaveBlockDelay(s.blockDelay.ToBytes()); err != nil {
+		if err := s.sm.SaveBlockDelay(s.blockDelay.ToBytes()); err != nil {
 			panic(fmt.Errorf("failed to save block delay: %w", err))
 		}
 	}
 
+	s.haltIfReached()
+
 	return &cmtabci.CommitResponse{
 		RetainHeight: retainHeight,
 	}, nil
 }
 
+// haltPointReached reports whether a block at the given height and time has reached the configured halt-height
+// or halt-time. It is the single halt predicate, applied to the last finalized block by ensureNotHalted and haltIfReached.
+func haltPointReached(haltHeight, haltTime uint64, height int64, blockTime time.Time) bool {
+	unixTime := blockTime.Unix()
+	switch {
+	case haltHeight > 0 && height >= 0 && uint64(height) >= haltHeight:
+		return true
+	case haltTime > 0 && unixTime >= 0 && uint64(unixTime) >= haltTime:
+		return true
+	default:
+		return false
+	}
+}
+
+// ensureNotHalted returns an error once the last finalized block has reached the halt point. Service start
+// refuses to run with it and the ABCI handlers gate on it (see abci.go), so a node with the halt flags still
+// set neither advances state past the halt block nor creeps one block per restart.
+func (s *Service) ensureNotHalted() error {
+	if !haltPointReached(s.haltHeight, s.haltTime, s.finalizedHeight, s.finalizedTime) {
+		return nil
+	}
+	return fmt.Errorf(
+		"chain reached the configured halt point (halt-height %d, halt-time %d) at committed height %d, unset the halt flags to resume",
+		s.haltHeight, s.haltTime, s.finalizedHeight,
+	)
+}
+
+// haltShutdownSlack bounds how long a parked ABCI call waits after the app context is cancelled. The halt
+// shutdown normally exits the process within it, so the caller's error return (and the CometBFT panic it
+// causes) only happens on a wedged shutdown.
+//
+//nolint:gochecknoglobals // var instead of const so tests can shorten it
+var haltShutdownSlack = 30 * time.Second
+
+// waitForHaltShutdown parks an ABCI call that must not proceed past the halt point while the halt shutdown
+// brings the process down.
+func (s *Service) waitForHaltShutdown() {
+	<-s.ctx.Done()
+	time.Sleep(haltShutdownSlack)
+}
+
+// haltGracePeriod keeps Commit blocked after the halt block so vote gossip can deliver the halt-block
+// precommits to validators still one short. Exiting immediately can wedge those peers at the previous height,
+// and a wedged set larger than 1/3 cannot recover after the restart (individual precommit signatures do not
+// survive commit aggregation).
+const haltGracePeriod = 5 * time.Second
+
+// haltIfReached gracefully shuts down the node once the committed block reaches the configured halt-height or
+// halt-time. It runs after the block has been fully committed, so a restarted node resumes consensus at the
+// next height with no replay needed.
+func (s *Service) haltIfReached() {
+	if !haltPointReached(s.haltHeight, s.haltTime, s.finalizedHeight, s.finalizedTime) {
+		return
+	}
+
+	s.logger.Info("halting node per configuration",
+		"halt_height", s.haltHeight, "halt_time", s.haltTime, "committed_height", s.finalizedHeight, "grace_period", haltGracePeriod)
+
+	// Sleeping here blocks the consensus state machine inside Commit, so no halting node can advance to the
+	// next height while its peer gossip routines keep serving the halt-block precommits from the live vote set.
+	time.Sleep(haltGracePeriod)
+
+	// Signal our own process so the node's regular shutdown path runs, the same mechanism a cosmos-sdk baseapp
+	// uses for halt-height. FindProcess never fails on Unix.
+	p, _ := os.FindProcess(os.Getpid())
+	if err := p.Signal(syscall.SIGINT); err != nil {
+		if err = p.Signal(syscall.SIGTERM); err != nil {
+			os.Exit(0)
+		}
+	}
+}
+
 // GetBlockRetentionHeight returns the height for which all blocks below this
 // height
 // are pruned from CometBFT. Given a commitment height and a non-zero local

consensus/cometbft/service/commit_internal_test.go

@@ -0,0 +1,148 @@
+// SPDX-License-Identifier: BUSL-1.1
+//
+// Copyright (C) 2025, Berachain Foundation. All rights reserved.
+// Use of this software is governed by the Business Source License included
+// in the LICENSE file of this repository and at www.mariadb.com/bsl11.
+//
+// ANY USE OF THE LICENSED WORK IN VIOLATION OF THIS LICENSE WILL AUTOMATICALLY
+// TERMINATE YOUR RIGHTS UNDER THIS LICENSE FOR THE CURRENT AND ALL OTHER
+// VERSIONS OF THE LICENSED WORK.
+//
+// THIS LICENSE DOES NOT GRANT YOU ANY RIGHT IN ANY TRADEMARK OR LOGO OF
+// LICENSOR OR ITS AFFILIATES (PROVIDED THAT YOU MAY USE A TRADEMARK OR LOGO OF
+// LICENSOR AS EXPRESSLY REQUIRED BY THIS LICENSE).
+//
+// TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
+// AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
+// EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
+// TITLE.
+//
+
+package cometbft
+
+import (
+	"context"
+	"io"
+	"testing"
+	"time"
+
+	"github.com/berachain/beacon-kit/log/phuslu"
+	cmtabci "github.com/cometbft/cometbft/abci/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHaltPointReached(t *testing.T) {
+	t.Parallel()
+
+	blockTime := time.Unix(1700000000, 0)
+
+	tests := []struct {
+		name       string
+		haltHeight uint64
+		haltTime   uint64
+		height     int64
+		blockTime  time.Time
+		want       bool
+	}{
+		{name: "disabled", height: 100, blockTime: blockTime, want: false},
+		{name: "below halt height", haltHeight: 100, height: 99, blockTime: blockTime, want: false},
+		{name: "at halt height", haltHeight: 100, height: 100, blockTime: blockTime, want: true},
+		{name: "past halt height", haltHeight: 100, height: 101, blockTime: blockTime, want: true},
+		{name: "before halt time", haltTime: 1700000001, height: 100, blockTime: blockTime, want: false},
+		{name: "at halt time", haltTime: 1700000000, height: 100, blockTime: blockTime, want: true},
+		{name: "past halt time", haltTime: 1699999999, height: 100, blockTime: blockTime, want: true},
+		{name: "zero block time does not halt", haltTime: 1700000000, height: 100, blockTime: time.Time{}, want: false},
+		{name: "halt time reached before halt height", haltHeight: 200, haltTime: 1700000000, height: 100, blockTime: blockTime, want: true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			require.Equal(t, tt.want, haltPointReached(tt.haltHeight, tt.haltTime, tt.height, tt.blockTime))
+		})
+	}
+}
+
+// TestEnsureNotHalted pins ensureNotHalted to the last finalized block. FinalizeBlock at height N runs it before
+// updating the finalized fields, so state held at N-1 means "may block N be finalized".
+func TestEnsureNotHalted(t *testing.T) {
+	t.Parallel()
+
+	s := &Service{}
+	require.NoError(t, s.ensureNotHalted(), "halt disabled")
+
+	s.haltHeight = 10
+	s.finalizedHeight = 9
+	require.NoError(t, s.ensureNotHalted(), "halt block itself must finalize")
+
+	s.finalizedHeight = 10
+	require.Error(t, s.ensureNotHalted(), "block past halt height must be refused")
+
+	s = &Service{haltTime: 1700000000}
+	require.NoError(t, s.ensureNotHalted(), "unseeded finalized time must not refuse")
+
+	s.finalizedTime = time.Unix(1700000000, 0)
+	require.Error(t, s.ensureNotHalted(), "block after halt time must be refused")
+}
+
+// haltedService returns a Service whose halt point has been reached.
+func haltedService(ctx context.Context) *Service {
+	return &Service{
+		logger:          phuslu.NewLogger(io.Discard, nil),
+		ctx:             ctx,
+		haltHeight:      10,
+		finalizedHeight: 10,
+	}
+}
+
+// TestFinalizeBlockParksAtHaltPoint pins the FinalizeBlock halt gate. A block past the halt point must park
+// until shutdown rather than return an error, which CometBFT escalates to a CONSENSUS FAILURE panic.
+func TestFinalizeBlockParksAtHaltPoint(t *testing.T) {
+	t.Parallel() // safe, no other test reads haltShutdownSlack
+
+	restore := haltShutdownSlack
+	haltShutdownSlack = 50 * time.Millisecond
+	t.Cleanup(func() { haltShutdownSlack = restore })
+
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+	s := haltedService(ctx)
+
+	done := make(chan error, 1)
+	go func() {
+		_, err := s.FinalizeBlock(ctx, &cmtabci.FinalizeBlockRequest{Height: 11})
+		done <- err
+	}()
+
+	select {
+	case err := <-done:
+		t.Fatalf("FinalizeBlock returned instead of parking until shutdown: %v", err)
+	case <-time.After(250 * time.Millisecond):
+	}
+
+	cancel()
+	select {
+	case err := <-done:
+		require.ErrorContains(t, err, "halt point")
+	case <-time.After(5 * time.Second):
+		t.Fatal("FinalizeBlock did not return after context cancel plus slack")
+	}
+}
+
+// TestProposalHandlersGateAtHaltPoint pins the non-panicking proposal gates,
+// which keep a halting validator from helping decide a post-halt block.
+func TestProposalHandlersGateAtHaltPoint(t *testing.T) {
+	t.Parallel()
+
+	ctx := t.Context()
+	s := haltedService(ctx)
+
+	prepResp, err := s.PrepareProposal(ctx, &cmtabci.PrepareProposalRequest{Height: 11})
+	require.NoError(t, err)
+	require.Empty(t, prepResp.Txs, "halted node must propose nothing")
+
+	procResp, err := s.ProcessProposal(ctx, &cmtabci.ProcessProposalRequest{Height: 11})
+	require.NoError(t, err)
+	require.Equal(t, cmtabci.PROCESS_PROPOSAL_STATUS_REJECT, procResp.Status)
+}