planner, meta: avoid unsafe index on virtual generated temporal columns

pkg/executor/builder.go

@@ -533,11 +533,16 @@ func buildIndexLookUpChecker(b *executorBuilder, p *physicalop.PhysicalIndexLook
 
 func (b *executorBuilder) buildCheckTable(v *plannercore.CheckTable) exec.Executor {
 	canUseFastCheck := true
+	tblInfo := v.Table.Meta()
 	for _, idx := range v.IndexInfos {
 		if idx.MVIndex || idx.IsColumnarIndex() {
 			canUseFastCheck = false
 			break
 		}
+		if idx.ContainsVirtualGeneratedTemporalWithDateColumn(tblInfo) {
+			canUseFastCheck = false
+			break
+		}
 		for _, col := range idx.Columns {
 			if col.Length != types.UnspecifiedLength {
 				canUseFastCheck = false

pkg/executor/check_table_index.go

@@ -896,6 +896,9 @@ func getCheckSum(ctx context.Context, se sessionctx.Context, sql string) ([]grou
 
 func getGlobalCheckSum(ctx context.Context, se sessionctx.Context, sql string) (checksum uint64, count int64, err error) {
 	ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnAdmin)
+	// Create a new ExecDetails for the internal SQL to avoid data race with the parent statement.
+	// The parent context may carry an ExecDetails that is still being written to by concurrent goroutines.
+	ctx = execdetails.ContextWithInitializedExecDetails(ctx)
 	rs, err := se.GetSQLExecutor().ExecuteInternal(ctx, sql)
 	if err != nil {
 		return 0, 0, err

pkg/executor/test/admintest/admin_test.go

@@ -2293,13 +2293,27 @@ func TestFastAdminCheckWithError(t *testing.T) {
 	// And the admin check shouldn't be blocked when meeting error.
 	tk := testkit.NewTestKit(t, store)
 	tk.MustExec("use test")
+	tk.MustExec("set tidb_enable_fast_table_check = 1")
 	tk.MustExec("drop table if exists admin_test")
 	tk.MustExec(`
 		create table admin_test (c1 int, c2 int,
 		key idx1(c1), key idx2(c1), key idx3(c1), key idx4(c1), key idx5(c1),
 		key idx6(c1), key idx7(c1), key idx8(c1), key idx9(c1), key idx10(c1))
 	`)
 	tk.MustExecToErr("admin check table admin_test")
+
+	tk.MustExec("drop table if exists admin_test_generated")
+	tk.MustExec(`
+		create table admin_test_generated (
+			payload json,
+			f date as (JSON_EXTRACT(payload, '$.f')),
+			key idx_f(f)
+		)
+	`)
+	tk.MustExec(`insert into admin_test_generated set payload='{"f":"2018-09-28"}'`)
+	// mockFastCheckTableError is still active here. This only succeeds because
+	// buildCheckTable falls back to CheckTableExec for idx_f instead of using FastCheckTableExec.
+	tk.MustExec("admin check table admin_test_generated")
 }
 
 func TestFastAdminCheckQuickPassSkipBucketed(t *testing.T) {
@@ -2488,7 +2502,7 @@ func TestAdminCheckTableWithEnumAndPointGet(t *testing.T) {
 func TestFastCheckTableConcurrent(t *testing.T) {
 	// This test verifies that concurrent execution of admin check table works correctly.
 	// Note: The data race in ExecDetails (fixed by using ContextWithInitializedExecDetails
-	// in getCheckSum) cannot be detected in unit tests because mocktikv doesn't trigger
+	// in getCheckSum/getGlobalCheckSum) cannot be detected in unit tests because mocktikv doesn't trigger
 	// the network traffic writes to ExecDetails that happen in real TiKV environments.
 	store := testkit.CreateMockStore(t)
 	tk := testkit.NewTestKit(t, store)

pkg/executor/test/tiflashtest/tiflash_test.go

@@ -1811,9 +1811,9 @@ func TestMPP47766(t *testing.T) {
 	tk.MustQuery("explain select date(test_time), count(1) as test_date from `traces` group by 1").Check(testkit.Rows(
 		"Projection_4 8000.00 root  test.traces.test_time_gen->Column#6, Column#5",
 		"└─HashAgg_8 8000.00 root  group by:test.traces.test_time_gen, funcs:count(1)->Column#5, funcs:firstrow(test.traces.test_time_gen)->test.traces.test_time_gen",
-		"  └─TableReader_20 10000.00 root  MppVersion: 3, data:ExchangeSender_19",
-		"    └─ExchangeSender_19 10000.00 mpp[tiflash]  ExchangeType: PassThrough",
-		"      └─TableFullScan_18 10000.00 mpp[tiflash] table:traces keep order:false, stats:pseudo"))
+		"  └─TableReader_18 10000.00 root  MppVersion: 3, data:ExchangeSender_17",
+		"    └─ExchangeSender_17 10000.00 mpp[tiflash]  ExchangeType: PassThrough",
+		"      └─TableFullScan_16 10000.00 mpp[tiflash] table:traces keep order:false, stats:pseudo"))
 	tk.MustQuery("explain select /*+ read_from_storage(tiflash[traces]) */ date(test_time) as test_date, count(1) from `traces` group by 1").
 		Check(testkit.Rows(
 			"TableReader_31 8000.00 root  MppVersion: 3, data:ExchangeSender_30",

pkg/importsdk/BUILD.bazel

@@ -54,6 +54,7 @@ go_test(
         "//pkg/lightning/config",
         "//pkg/lightning/log",
         "//pkg/lightning/mydump",
+        "//pkg/parser/ast",
         "//pkg/parser/mysql",
         "//pkg/util/table-filter",
         "@com_github_data_dog_go_sqlmock//:go-sqlmock",

pkg/meta/model/index.go

@@ -25,6 +25,7 @@ import (
 	"github.com/pingcap/tidb/pkg/parser/mysql"
 	"github.com/pingcap/tidb/pkg/parser/types"
 	"github.com/pingcap/tidb/pkg/planner/cascades/base"
+	tidbtypes "github.com/pingcap/tidb/pkg/types"
 )
 
 // DistanceMetric is the distance metric used by the vector index.
@@ -365,6 +366,21 @@ func (index *IndexInfo) HasPrefixIndex() bool {
 	return false
 }
 
+// ContainsVirtualGeneratedTemporalWithDateColumn reports whether the index contains
+// a virtual generated temporal-with-date column.
+func (index *IndexInfo) ContainsVirtualGeneratedTemporalWithDateColumn(tblInfo *TableInfo) bool {
+	for _, idxCol := range index.Columns {
+		if idxCol.Offset < 0 || idxCol.Offset >= len(tblInfo.Columns) {
+			continue
+		}
+		col := tblInfo.Columns[idxCol.Offset]
+		if col.IsVirtualGenerated() && tidbtypes.IsTemporalWithDate(col.GetType()) {
+			return true
+		}
+	}
+	return false
+}
+
 // HasColumnInIndexColumns checks whether the index contains the column with the specified ID.
 func (index *IndexInfo) HasColumnInIndexColumns(tblInfo *TableInfo, colID int64) bool {
 	for _, ic := range index.Columns {

pkg/planner/core/casetest/index/BUILD.bazel

@@ -9,7 +9,7 @@ go_test(
     ],
     data = glob(["testdata/**"]),
     flaky = True,
-    shard_count = 12,
+    shard_count = 13,
     deps = [
         "//pkg/domain",
         "//pkg/domain/infosync",

pkg/planner/core/casetest/index/index_test.go

@@ -107,6 +107,44 @@ func TestInvisibleIndex(t *testing.T) {
 	})
 }
 
+func TestVirtualGeneratedTemporalWithDateIndex(t *testing.T) {
+	testkit.RunTestUnderCascades(t, func(t *testing.T, tk *testkit.TestKit, cascades, caller string) {
+		tk.MustExec("use test")
+		tk.MustExec("drop table if exists t_issue_52520")
+		tk.MustExec("create table t_issue_52520 (a varchar(32), b date as (a), key idx_b(b))")
+		tk.MustExec("set @@sql_mode = ''")
+		tk.MustExec("insert into t_issue_52520(a) values ('2020-02-31')")
+		tk.MustExec("set @@sql_mode = 'ALLOW_INVALID_DATES'")
+
+		tk.MustNoIndexUsed("select /* issue:52520 */ b from t_issue_52520 use index(idx_b)")
+		tk.MustQuery("show warnings").CheckContain("virtual generated temporal column")
+		tk.MustQuery("select /* issue:52520 */ b from t_issue_52520 use index(idx_b)").Check(testkit.Rows("2020-02-31"))
+		tk.MustQuery("show warnings").CheckContain("virtual generated temporal column")
+
+		tk.MustNoIndexUsed("select /*+ USE_INDEX(t_issue_52520, idx_b) */ /* issue:52520 */ b from t_issue_52520")
+		tk.MustQuery("show warnings").CheckContain("virtual generated temporal column")
+		tk.MustQuery("select /*+ USE_INDEX(t_issue_52520, idx_b) */ /* issue:52520 */ b from t_issue_52520").Check(testkit.Rows("2020-02-31"))
+		tk.MustQuery("show warnings").CheckContain("virtual generated temporal column")
+
+		tk.MustNoIndexUsed("select /* issue:52520 */ b from t_issue_52520 use index(idx_b) where b = '2020-02-31'")
+		tk.MustQuery("show warnings").CheckContain("virtual generated temporal column")
+		tk.MustQuery("select /* issue:52520 */ b from t_issue_52520 use index(idx_b) where b = '2020-02-31'").Check(testkit.Rows("2020-02-31"))
+		tk.MustQuery("show warnings").CheckContain("virtual generated temporal column")
+
+		tk.MustNoIndexUsed("select /*+ USE_INDEX(t_issue_52520, idx_b) */ /* issue:52520 */ b from t_issue_52520 where b = '2020-02-31'")
+		tk.MustQuery("show warnings").CheckContain("virtual generated temporal column")
+		tk.MustQuery("select /*+ USE_INDEX(t_issue_52520, idx_b) */ /* issue:52520 */ b from t_issue_52520 where b = '2020-02-31'").Check(testkit.Rows("2020-02-31"))
+		tk.MustQuery("show warnings").CheckContain("virtual generated temporal column")
+
+		tk.MustNoIndexUsed("select /* issue:52520 */ b from t_issue_52520 ignore index(idx_b) where b = '2020-02-31'")
+		tk.MustQuery("show warnings").Check(testkit.Rows())
+
+		tk.MustExec("drop table if exists t_issue_52520_prefix")
+		tk.MustExec("create table t_issue_52520_prefix (a varchar(32), c int, b date as (a), key idx_safe(c), key idx_unsafe(b))")
+		tk.MustContainErrMsg("select /* issue:52520 */ c from t_issue_52520_prefix use index(idx) where c = 1", "Key 'idx' doesn't exist")
+	})
+}
+
 func TestRangeDerivation(t *testing.T) {
 	testkit.RunTestUnderCascades(t, func(t *testing.T, testKit *testkit.TestKit, cascades, caller string) {
 		testKit.MustExec("use test")
@@ -297,6 +335,14 @@ func TestInvertedIndex(t *testing.T) {
 		testKit.MustNoIndexUsed("select * from t ignore index(idx_b) where b = 2")
 		testKit.MustNoIndexUsed("select * from t ignore index(idx_c) where c = 3")
 		testKit.MustNoIndexUsed("select * from t ignore index(idx_d) where d < 1")
+
+		testKit.MustExec("drop table if exists t_issue_52520_columnar")
+		testKit.MustExec("create table t_issue_52520_columnar (a varchar(32), b date as (a))")
+		testKit.MustExec("alter table t_issue_52520_columnar set tiflash replica 1")
+		testKit.MustExec("alter table t_issue_52520_columnar add columnar index idx_b (b) using inverted")
+		testkit.SetTiFlashReplica(t, dom, "test", "t_issue_52520_columnar")
+		testKit.MustNoIndexUsed("select /* issue:52520 */ b from t_issue_52520_columnar use index(idx_b)")
+		testKit.MustQuery("show warnings").CheckContain("virtual generated temporal column")
 	})
 }
 

pkg/planner/core/logical_plan_builder.go

@@ -5034,10 +5034,12 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName, as
 		return nil, plannererrors.ErrPartitionClauseOnNonpartitioned
 	}
 
-	possiblePaths, err := getPossibleAccessPaths(b.ctx, b.TableHints(), tn.IndexHints, tbl, dbName, tblName, b.isForUpdateRead, b.optFlag&rule.FlagPartitionProcessor > 0)
+	possiblePathInfo, err := getPossibleAccessPathInfo(b.ctx, b.TableHints(), tn.IndexHints, tbl, dbName, tblName, b.isForUpdateRead, b.optFlag&rule.FlagPartitionProcessor > 0)
 	if err != nil {
 		return nil, err
 	}
+	possiblePaths := possiblePathInfo.paths
+	gcSubstituteExtraPaths := possiblePathInfo.gcSubstituteExtraPaths
 
 	if tableInfo.IsView() {
 		if tn.TableSample != nil {
@@ -5091,6 +5093,10 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName, as
 			if err != nil {
 				return nil, err
 			}
+			gcSubstituteExtraPaths, err = util.FilterPathByIsolationRead(b.ctx, gcSubstituteExtraPaths, tblName, dbName)
+			if err != nil && len(gcSubstituteExtraPaths) != 0 {
+				return nil, err
+			}
 		}
 	}
 
@@ -5178,6 +5184,7 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName, as
 		IndexMergeHints:        indexMergeHints,
 		PossibleAccessPaths:    possiblePaths,
 		AllPossibleAccessPaths: allPaths,
+		GcSubstituteExtraPaths: gcSubstituteExtraPaths,
 		Columns:                make([]*model.ColumnInfo, 0, countCnt),
 		PartitionNames:         tn.PartitionNames,
 		TblCols:                make([]*expression.Column, 0, countCnt),

pkg/planner/core/operator/logicalop/logical_datasource.go

@@ -77,6 +77,9 @@ type DataSource struct {
 
 	// AllPossibleAccessPaths stores all the possible access path from build datasource phase.
 	AllPossibleAccessPaths []*util.AccessPath
+	// GcSubstituteExtraPaths stores filtered index paths that should still participate in
+	// generated-column substitution. They are not available for access-path selection.
+	GcSubstituteExtraPaths []*util.AccessPath
 	// PossibleAccessPaths stores all the possible access path for one specific logical alternative.
 	// because different logical alternative may have different filter condition, so the possible access path may be different.
 	// like: