diff --git a/pkg/ddl/backfilling_dist_executor.go b/pkg/ddl/backfilling_dist_executor.go
index 08406f8e93f0b..0ad12e60484cd 100644
--- a/pkg/ddl/backfilling_dist_executor.go
+++ b/pkg/ddl/backfilling_dist_executor.go
@@ -33,6 +33,12 @@ import (
 	"go.uber.org/zap"
 )
 
+var errBackfillTaskMetaOutdated = errors.New("backfill task meta is outdated")
+
+func isBackfillTaskMetaOutdatedErr(err error) bool {
+	return errors.Cause(err) == errBackfillTaskMetaOutdated
+}
+
 // Version constants for BackfillTaskMeta.
 const (
 	BackfillTaskMetaVersion0 = iota
@@ -164,7 +170,9 @@ func (s *backfillDistExecutor) newBackfillStepExecutor(
 			logutil.DDLIngestLogger().Warn("index info not found",
 				zap.Int64("table ID", tbl.Meta().ID),
 				zap.Int64("index ID", eid))
-			return nil, errors.Errorf("index info not found: %d", eid)
+			return nil, errors.Annotatef(errBackfillTaskMetaOutdated,
+				"index info not found: %d, table ID: %d, job ID: %d",
+				eid, tbl.Meta().ID, jobMeta.ID)
 		}
 		indexInfos = append(indexInfos, indexInfo)
 	}
@@ -246,6 +254,9 @@ func (*backfillDistExecutor) IsIdempotent(*proto.Subtask) bool {
 }
 
 func (*backfillDistExecutor) IsRetryableError(err error) bool {
+	if isBackfillTaskMetaOutdatedErr(err) {
+		return false
+	}
 	return common.IsRetryableError(err) || isRetryableError(err, true)
 }
 
diff --git a/pkg/ddl/backfilling_test.go b/pkg/ddl/backfilling_test.go
index 0e341f7e59386..8451f667215d4 100644
--- a/pkg/ddl/backfilling_test.go
+++ b/pkg/ddl/backfilling_test.go
@@ -21,6 +21,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/pingcap/errors"
 	"github.com/pingcap/tidb/pkg/ddl/copr"
 	"github.com/pingcap/tidb/pkg/ddl/ingest"
 	distsqlctx "github.com/pingcap/tidb/pkg/distsql/context"
@@ -64,6 +65,12 @@ func TestDoneTaskKeeper(t *testing.T) {
 	require.True(t, bytes.Equal(n.nextKey, kv.Key("h")))
 }
 
+func TestOutdatedBackfillTaskMetaIsNonRetryable(t *testing.T) {
+	err := errors.Annotatef(errBackfillTaskMetaOutdated, "index info not found: %d", 1)
+	require.False(t, isRetryableError(err, true))
+	require.False(t, (&backfillDistExecutor{}).IsRetryableError(err))
+}
+
 func TestPickBackfillType(t *testing.T) {
 	ingest.LitDiskRoot = ingest.NewDiskRootImpl(t.TempDir())
 	ingest.LitMemRoot = ingest.NewMemRootImpl(math.MaxInt64)
diff --git a/pkg/ddl/index.go b/pkg/ddl/index.go
index c4beac45b4976..01afeca465e58 100644
--- a/pkg/ddl/index.go
+++ b/pkg/ddl/index.go
@@ -1935,6 +1935,9 @@ func isRetryableJobError(err error, jobErrCnt int64) bool {
 }
 
 func isRetryableError(err error, retryUnknown bool) bool {
+	if isBackfillTaskMetaOutdatedErr(err) {
+		return false
+	}
 	errMsg := err.Error()
 	for _, m := range dbterror.ReorgRetryableErrMsgs {
 		if strings.Contains(errMsg, m) {