From 039d6fc5c70054683007232ef03de66b0bb73c0a Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Mon, 26 Jan 2026 22:39:59 +0900 Subject: [PATCH] This is an automated cherry-pick of #65051 Signed-off-by: ti-chi-bot --- pkg/expression/util.go | 12 + pkg/planner/core/casetest/index/BUILD.bazel | 14 + pkg/planner/core/casetest/index/index_test.go | 99 +++ pkg/planner/core/exhaust_physical_plans.go | 161 +++- pkg/planner/core/indexmerge_path.go | 10 + .../core/operator/logicalop/BUILD.bazel | 1 + .../operator/logicalop/logical_datasource.go | 79 ++ .../physicalop/physical_index_scan.go | 739 ++++++++++++++++++ pkg/planner/core/partidx/BUILD.bazel | 16 + pkg/planner/core/partidx/check_constraint.go | 221 ++++++ pkg/planner/core/plan_cacheable_checker.go | 18 + pkg/planner/core/planbuilder.go | 13 +- pkg/planner/core/point_get_plan.go | 32 +- pkg/planner/core/rule/rule_prune_indexes.go | 577 ++++++++++++++ pkg/planner/core/stats.go | 7 + pkg/planner/util/path.go | 56 ++ pkg/util/ranger/types.go | 27 + .../core/casetest/index/partialindex.result | 110 +++ .../core/casetest/index/partialindex.test | 39 + 19 files changed, 2211 insertions(+), 20 deletions(-) create mode 100644 pkg/planner/core/operator/physicalop/physical_index_scan.go create mode 100644 pkg/planner/core/partidx/BUILD.bazel create mode 100644 pkg/planner/core/partidx/check_constraint.go create mode 100644 pkg/planner/core/rule/rule_prune_indexes.go create mode 100644 tests/integrationtest/r/planner/core/casetest/index/partialindex.result create mode 100644 tests/integrationtest/t/planner/core/casetest/index/partialindex.test diff --git a/pkg/expression/util.go b/pkg/expression/util.go index 610551de3d8e3..54ee10e3e5cc7 100644 --- a/pkg/expression/util.go +++ b/pkg/expression/util.go @@ -782,6 +782,18 @@ var symmetricOp = map[opcode.Op]opcode.Op{ opcode.NullEQ: opcode.NullEQ, } +// CompareOpMap records all comparison operators. +var CompareOpMap = map[string]struct{}{ + ast.LT: {}, + ast.GE: {}, + ast.GT: {}, + ast.LE: {}, + ast.EQ: {}, + ast.NE: {}, + ast.NullEQ: {}, + ast.In: {}, +} + func pushNotAcrossArgs(ctx BuildContext, exprs []Expression, not bool) ([]Expression, bool) { newExprs := make([]Expression, 0, len(exprs)) flag := false diff --git a/pkg/planner/core/casetest/index/BUILD.bazel b/pkg/planner/core/casetest/index/BUILD.bazel index e4bb65791f767..149a7e557047e 100644 --- a/pkg/planner/core/casetest/index/BUILD.bazel +++ b/pkg/planner/core/casetest/index/BUILD.bazel @@ -9,18 +9,32 @@ go_test( ], data = glob(["testdata/**"]), flaky = True, +<<<<<<< HEAD shard_count = 8, deps = [ "//pkg/domain", "//pkg/domain/infosync", "//pkg/parser/model", +======= + shard_count = 11, + deps = [ + "//pkg/domain", + "//pkg/domain/infosync", + "//pkg/parser/ast", + "//pkg/planner/util", + "//pkg/session/sessmgr", +>>>>>>> 959bf330874 (planner: support basic usage of partial index (#65051)) "//pkg/store/mockstore", "//pkg/testkit", "//pkg/testkit/testdata", "//pkg/testkit/testfailpoint", "//pkg/testkit/testmain", "//pkg/testkit/testsetup", +<<<<<<< HEAD "//pkg/util", +======= + "@com_github_pingcap_failpoint//:failpoint", +>>>>>>> 959bf330874 (planner: support basic usage of partial index (#65051)) "@com_github_stretchr_testify//require", "@org_uber_go_goleak//:goleak", ], diff --git a/pkg/planner/core/casetest/index/index_test.go b/pkg/planner/core/casetest/index/index_test.go index 5f7946c7def53..2cc47f6ebf73c 100644 --- a/pkg/planner/core/casetest/index/index_test.go +++ b/pkg/planner/core/casetest/index/index_test.go @@ -20,9 +20,16 @@ import ( "testing" "time" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/domain" "github.com/pingcap/tidb/pkg/domain/infosync" +<<<<<<< HEAD "github.com/pingcap/tidb/pkg/parser/model" +======= + "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/planner/util" + "github.com/pingcap/tidb/pkg/session/sessmgr" +>>>>>>> 959bf330874 (planner: support basic usage of partial index (#65051)) "github.com/pingcap/tidb/pkg/store/mockstore" "github.com/pingcap/tidb/pkg/testkit" "github.com/pingcap/tidb/pkg/testkit/testdata" @@ -346,3 +353,95 @@ func TestAnalyzeVectorIndex(t *testing.T) { "Warning 1105 analyzing vector index is not supported, skip idx2", "Warning 1681 ANALYZE with tidb_analyze_version=1 is deprecated and will be removed in a future release.")) } + +func TestPartialIndexWithPlanCache(t *testing.T) { + testkit.RunTestUnderCascades(t, func(t *testing.T, tk *testkit.TestKit, cascades, caller string) { + tk.MustExec(`set tidb_enable_prepared_plan_cache=1`) + tk.MustExec("use test") + tk.MustExec("set @@tidb_enable_collect_execution_info=0;") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int, b int, index idx1(a) where a is not null, index idx2(b) where b > 10)") + + tk.MustExec("prepare stmt from 'select * from t where a = ?'") + tk.MustExec("set @a = 123") + + // IS NOT NULL pre condition can use plan cache. + tk.MustExec("execute stmt using @a") + tk.MustExec("execute stmt using @a") + tkProcess := tk.Session().ShowProcess() + ps := []*sessmgr.ProcessInfo{tkProcess} + tk.Session().SetSessionManager(&testkit.MockSessionManager{PS: ps}) + tk.MustQuery(fmt.Sprintf("explain for connection %d", tkProcess.ID)).CheckContain("idx1") + tk.MustExec("execute stmt using @a") + tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("1")) + + // Normal pre condition can not use plan cache. + tk.MustExec("prepare stmt from 'select * from t where b = ?'") + tk.MustExec("set @a = 20") + tk.MustExec("execute stmt using @a") + tk.MustExec("execute stmt using @a") + tkProcess = tk.Session().ShowProcess() + ps[0] = tkProcess + tk.Session().SetSessionManager(&testkit.MockSessionManager{PS: ps}) + tk.MustQuery(fmt.Sprintf("explain for connection %d", tkProcess.ID)).CheckContain("idx2") + tk.MustExec("execute stmt using @a") + tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("0")) + }) +} + +func TestPartialIndexWithIndexPrune(t *testing.T) { + testkit.RunTestUnderCascades(t, func(t *testing.T, tk *testkit.TestKit, cascades, caller string) { + tk.MustExec("use test") + tk.MustExec("set @@tidb_enable_collect_execution_info=0;") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int, b int, index idx1(a) where a is not null, index idx2(b) where b > 10)") + tk.MustQuery("explain select * from t use index(idx1) where a > 1").CheckContain("idx1") + + // Set the prune behavior to prune all non interesting ones. + tk.MustExec("set @@tidb_opt_index_prune_threshold=0") + + // The failpoint will check whether all partial indexes are pruned. + fpName := "github.com/pingcap/tidb/pkg/planner/core/rule/InjectCheckForIndexPrune" + require.NoError(t, failpoint.EnableCall(fpName, func(paths []*util.AccessPath) { + for _, path := range paths { + if path != nil && path.Index != nil && path.Index.ConditionExprString != "" { + require.True(t, false, "Partial index should be pruned") + } + } + })) + tk.MustQuery("select * from t") + + // idx1 is pruned because a is not referenced as interesting one. + // idx2 is kept though its constraint is not matched. + require.NoError(t, failpoint.EnableCall(fpName, func(paths []*util.AccessPath) { + idx2Found := false + for _, path := range paths { + if path != nil && path.Index != nil && path.Index.Name.L == "idx1" { + require.True(t, false, "Partial index idx1 should be pruned") + } + if path != nil && path.Index != nil && path.Index.Name.L == "idx2" { + idx2Found = true + } + } + require.True(t, idx2Found, "Partial index idx2 should not be pruned") + })) + tk.MustQuery("explain select * from t order by b").CheckNotContain("idx2") + + // idx2 is pruned because b is not referenced as interesting one. + // idx1 is kept though its constraint is not matched. + require.NoError(t, failpoint.EnableCall(fpName, func(paths []*util.AccessPath) { + idx1Found := false + for _, path := range paths { + if path != nil && path.Index != nil && path.Index.Name.L == "idx2" { + require.True(t, false, "Partial index idx2 should be pruned") + } + if path != nil && path.Index != nil && path.Index.Name.L == "idx1" { + idx1Found = true + } + } + require.True(t, idx1Found, "Partial index idx1 should not be pruned") + })) + tk.MustQuery("explain select * from t where a is null").CheckNotContain("idx1") + require.NoError(t, failpoint.Disable(fpName)) + }) +} diff --git a/pkg/planner/core/exhaust_physical_plans.go b/pkg/planner/core/exhaust_physical_plans.go index d999bcdf9973a..f09dbb2105af9 100644 --- a/pkg/planner/core/exhaust_physical_plans.go +++ b/pkg/planner/core/exhaust_physical_plans.go @@ -796,6 +796,165 @@ childLoop: return wrapper } +<<<<<<< HEAD +======= +// buildDataSource2IndexScanByIndexJoinProp builds an IndexScan as the inner child for an +// IndexJoin based on IndexJoinProp included in prop if possible. +// +// buildDataSource2IndexScanByIndexJoinProp differs with buildIndexJoinInner2IndexScan in that +// the first one is try to build a single table scan as the inner child of an index join then return +// this inner task(raw table scan) bottom-up, which will be attached with other inner parents of an +// index join in attach2Task when bottom-up of enumerating the physical plans; +// +// while the second is try to build a table scan as the inner child of an index join, then build +// entire inner subtree of a index join out as innerTask instantly according those validated and +// zipped inner patterns with calling constructInnerIndexScanTask. That's not done yet, it also +// tries to enumerate kinds of index join operators based on the finished innerTask and un-decided +// outer child which will be physical-ed in the future. +func buildDataSource2IndexScanByIndexJoinProp( + ds *logicalop.DataSource, + prop *property.PhysicalProperty) base.Task { + indexValid := func(path *util.AccessPath) bool { + if path.IsTablePath() { + return false + } + // if path is index path. index path currently include two kind of, one is normal, and the other is mv index. + // for mv index like mvi(a, json, b), if driving condition is a=1, and we build a prefix scan with range [1,1] + // on mvi, it will return many index rows which breaks handle-unique attribute here. + // + // the basic rule is that: mv index can be and can only be accessed by indexMerge operator. (embedded handle duplication) + if !path.IsIndexJoinUnapplicable() { + return true // not a MVIndex path, it can successfully be index join probe side. + } + return false + } + indexJoinResult, keyOff2IdxOff := getBestIndexJoinPathResultByProp(ds, prop.IndexJoinProp, indexValid) + if indexJoinResult == nil { + return base.InvalidTask + } + rangeInfo, maxOneRow := indexJoinPathGetRangeInfoAndMaxOneRow(ds.SCtx(), prop.IndexJoinProp.OuterJoinKeys, indexJoinResult) + var innerTask base.Task + if !prop.IsSortItemEmpty() && matchProperty(ds, indexJoinResult.chosenPath, prop) == property.PropMatched { + innerTask = constructDS2IndexScanTask(ds, indexJoinResult.chosenPath, indexJoinResult.chosenRanges.Range(), indexJoinResult.chosenRemained, indexJoinResult.idxOff2KeyOff, rangeInfo, true, prop.SortItems[0].Desc, prop.IndexJoinProp.AvgInnerRowCnt, maxOneRow) + } else { + innerTask = constructDS2IndexScanTask(ds, indexJoinResult.chosenPath, indexJoinResult.chosenRanges.Range(), indexJoinResult.chosenRemained, indexJoinResult.idxOff2KeyOff, rangeInfo, false, false, prop.IndexJoinProp.AvgInnerRowCnt, maxOneRow) + } + // since there is a possibility that inner task can't be built and the returned value is nil, we just return base.InvalidTask. + if innerTask == nil { + return base.InvalidTask + } + // prepare the index path chosen information and wrap them as IndexJoinInfo and fill back to CopTask. + // here we don't need to construct physical index join here anymore, because we will encapsulate it bottom-up. + // chosenPath and lastColManager of indexJoinResult should be returned to the caller (seen by index join to keep + // index join aware of indexColLens and compareFilters). + completeIndexJoinFeedBackInfo(innerTask.(*physicalop.CopTask), indexJoinResult, indexJoinResult.chosenRanges, keyOff2IdxOff) + return innerTask +} + +// buildDataSource2TableScanByIndexJoinProp builds a TableScan as the inner child for an +// IndexJoin if possible. +// If the inner side of an index join is a TableScan, only one tuple will be +// fetched from the inner side for every tuple from the outer side. This will be +// promised to be no worse than building IndexScan as the inner child. +func buildDataSource2TableScanByIndexJoinProp( + ds *logicalop.DataSource, + prop *property.PhysicalProperty) base.Task { + var tblPath *util.AccessPath + for _, path := range ds.PossibleAccessPaths { + if path.IsTablePath() && path.StoreType == kv.TiKV { // old logic + tblPath = path + break + } + } + if tblPath == nil { + return base.InvalidTask + } + var keyOff2IdxOff []int + var ranges ranger.MutableRanges = ranger.Ranges{} + var innerTask base.Task + var indexJoinResult *indexJoinPathResult + if ds.TableInfo.IsCommonHandle { + // for the leaf datasource, we use old logic to get the indexJoinResult, which contain the chosen path and ranges. + indexJoinResult, keyOff2IdxOff = getBestIndexJoinPathResultByProp(ds, prop.IndexJoinProp, func(path *util.AccessPath) bool { return path.IsCommonHandlePath }) + // if there is no chosen info, it means the leaf datasource couldn't even leverage this indexJoinProp, return InvalidTask. + if indexJoinResult == nil { + return base.InvalidTask + } + // prepare the range info with outer join keys, it shows like: [xxx] decided by: + rangeInfo, maxOneRow := indexJoinPathGetRangeInfoAndMaxOneRow(ds.SCtx(), prop.IndexJoinProp.OuterJoinKeys, indexJoinResult) + // construct the inner task with chosen path and ranges, note: it only for this leaf datasource. + // like the normal way, we need to check whether the chosen path is matched with the prop, if so, we will set the `keepOrder` to true. + if matchProperty(ds, indexJoinResult.chosenPath, prop) == property.PropMatched { + innerTask = constructDS2TableScanTask(ds, indexJoinResult.chosenRanges.Range(), rangeInfo, true, !prop.IsSortItemEmpty() && prop.SortItems[0].Desc, prop.IndexJoinProp.AvgInnerRowCnt, maxOneRow) + } else { + innerTask = constructDS2TableScanTask(ds, indexJoinResult.chosenRanges.Range(), rangeInfo, false, false, prop.IndexJoinProp.AvgInnerRowCnt, maxOneRow) + } + ranges = indexJoinResult.chosenRanges + } else { + var ( + ok bool + chosenPath *util.AccessPath + newOuterJoinKeys []*expression.Column + // note: pk col doesn't have mutableRanges, the global var(ranges) which will be handled as empty range in constructIndexJoin. + localRanges ranger.Ranges + ) + keyOff2IdxOff, newOuterJoinKeys, localRanges, chosenPath, ok = getIndexJoinIntPKPathInfo(ds, prop.IndexJoinProp.InnerJoinKeys, prop.IndexJoinProp.OuterJoinKeys, func(path *util.AccessPath) bool { return path.IsIntHandlePath }) + if !ok { + return base.InvalidTask + } + // For IntHandle (integer primary key), it's always a unique match. + maxOneRow := true + rangeInfo := indexJoinIntPKRangeInfo(ds.SCtx().GetExprCtx().GetEvalCtx(), newOuterJoinKeys) + if !prop.IsSortItemEmpty() && matchProperty(ds, chosenPath, prop) == property.PropMatched { + innerTask = constructDS2TableScanTask(ds, localRanges, rangeInfo, true, prop.SortItems[0].Desc, prop.IndexJoinProp.AvgInnerRowCnt, maxOneRow) + } else { + innerTask = constructDS2TableScanTask(ds, localRanges, rangeInfo, false, false, prop.IndexJoinProp.AvgInnerRowCnt, maxOneRow) + } + } + // since there is a possibility that inner task can't be built and the returned value is nil, we just return base.InvalidTask. + if innerTask == nil { + return base.InvalidTask + } + // prepare the index path chosen information and wrap them as IndexJoinInfo and fill back to CopTask. + // here we don't need to construct physical index join here anymore, because we will encapsulate it bottom-up. + // chosenPath and lastColManager of indexJoinResult should be returned to the caller (seen by index join to keep + // index join aware of indexColLens and compareFilters). + completeIndexJoinFeedBackInfo(innerTask.(*physicalop.CopTask), indexJoinResult, ranges, keyOff2IdxOff) + return innerTask +} + +// completeIndexJoinFeedBackInfo completes the IndexJoinInfo for the innerTask. +// indexJoin +// +// +--- outer child +// +--- inner child (say: projection ------------> unionScan -------------> ds) +// <-------RootTask(IndexJoinInfo) <--RootTask(IndexJoinInfo) <--copTask(IndexJoinInfo) +// +// when we build the underlying datasource as table-scan, we will return wrap it and +// return as a CopTask, inside which the index join contains some index path chosen +// information which will be used in indexJoin execution runtime: ref IndexJoinInfo +// declaration for more information. +// the indexJoinInfo will be filled back to the innerTask, passed upward to RootTask +// once this copTask is converted to RootTask type, and finally end up usage in the +// indexJoin's attach2Task with calling completePhysicalIndexJoin. +func completeIndexJoinFeedBackInfo(innerTask *physicalop.CopTask, indexJoinResult *indexJoinPathResult, ranges ranger.MutableRanges, keyOff2IdxOff []int) { + info := innerTask.IndexJoinInfo + if info == nil { + info = &physicalop.IndexJoinInfo{} + } + if indexJoinResult != nil { + if indexJoinResult.chosenPath != nil { + info.IdxColLens = indexJoinResult.chosenPath.IdxColLens + } + info.CompareFilters = indexJoinResult.lastColManager + } + info.Ranges = ranges + info.KeyOff2IdxOff = keyOff2IdxOff + // fill it back to the bottom-up Task. + innerTask.IndexJoinInfo = info +} + +>>>>>>> 959bf330874 (planner: support basic usage of partial index (#65051)) // buildIndexJoinInner2TableScan builds a TableScan as the inner child for an // IndexJoin if possible. // If the inner side of a index join is a TableScan, only one tuple will be @@ -915,7 +1074,7 @@ func buildIndexJoinInner2IndexScan( // on mvi, it will return many index rows which breaks handle-unique attribute here. // // the basic rule is that: mv index can be and can only be accessed by indexMerge operator. (embedded handle duplication) - if !isMVIndexPath(path) { + if !path.IsIndexJoinUnapplicable() { return true // not a MVIndex path, it can successfully be index join probe side. } return false diff --git a/pkg/planner/core/indexmerge_path.go b/pkg/planner/core/indexmerge_path.go index dadabce2bd657..fc45c776d5109 100644 --- a/pkg/planner/core/indexmerge_path.go +++ b/pkg/planner/core/indexmerge_path.go @@ -135,11 +135,21 @@ func generateIndexMergePath(ds *logicalop.DataSource) error { func generateNormalIndexPartialPaths4DNF( ds *logicalop.DataSource, +<<<<<<< HEAD dnfItems []expression.Expression, candidatePaths []*util.AccessPath, ) (paths []*util.AccessPath, needSelection bool, usedMap []bool) { paths = make([]*util.AccessPath, 0, len(dnfItems)) usedMap = make([]bool, len(dnfItems)) +======= + item expression.Expression, + candidatePath *util.AccessPath, +) (paths *util.AccessPath, needSelection bool) { + // Reject partial index first. + if candidatePath.Index != nil && candidatePath.Index.HasCondition() { + return nil, false + } +>>>>>>> 959bf330874 (planner: support basic usage of partial index (#65051)) pushDownCtx := util.GetPushDownCtx(ds.SCtx()) for offset, item := range dnfItems { cnfItems := expression.SplitCNFItems(item) diff --git a/pkg/planner/core/operator/logicalop/BUILD.bazel b/pkg/planner/core/operator/logicalop/BUILD.bazel index 027064cf6c38f..444ae87851317 100644 --- a/pkg/planner/core/operator/logicalop/BUILD.bazel +++ b/pkg/planner/core/operator/logicalop/BUILD.bazel @@ -53,6 +53,7 @@ go_library( "//pkg/planner/core/constraint", "//pkg/planner/core/cost", "//pkg/planner/core/operator/baseimpl", + "//pkg/planner/core/partidx", "//pkg/planner/core/resolve", "//pkg/planner/core/rule/util", "//pkg/planner/funcdep", diff --git a/pkg/planner/core/operator/logicalop/logical_datasource.go b/pkg/planner/core/operator/logicalop/logical_datasource.go index 63a6298ba89ac..b977c5b28e64a 100644 --- a/pkg/planner/core/operator/logicalop/logical_datasource.go +++ b/pkg/planner/core/operator/logicalop/logical_datasource.go @@ -17,6 +17,7 @@ package logicalop import ( "bytes" "fmt" + "slices" "github.com/pingcap/errors" "github.com/pingcap/tidb/pkg/expression" @@ -28,7 +29,11 @@ import ( "github.com/pingcap/tidb/pkg/parser/mysql" base2 "github.com/pingcap/tidb/pkg/planner/cascades/base" "github.com/pingcap/tidb/pkg/planner/core/base" +<<<<<<< HEAD "github.com/pingcap/tidb/pkg/planner/core/constraint" +======= + "github.com/pingcap/tidb/pkg/planner/core/partidx" +>>>>>>> 959bf330874 (planner: support basic usage of partial index (#65051)) ruleutil "github.com/pingcap/tidb/pkg/planner/core/rule/util" fd "github.com/pingcap/tidb/pkg/planner/funcdep" "github.com/pingcap/tidb/pkg/planner/property" @@ -685,3 +690,77 @@ func (ds *DataSource) AppendTableCol(col *expression.Column) { ds.TblCols = append(ds.TblCols, col) ds.TblColsByID[col.ID] = col } + +// CheckPartialIndexes checks and removes the partial indexes that cannot be used according to the pushed down conditions. +// It will go through each partial index to see whether it's condition constraints are all satisfied by the pushed down conditions. +// Detailed checking can be found in the comment of `CheckConstraints`. +// And we specially implement a `AlwaysMeetConstraints` function for IS NOT NULL constraint to make it suitable for plan cache. +// It's a special handler now, and it's not easy to extend to other constraints. +func (ds *DataSource) CheckPartialIndexes() { + var columnNames types.NameSlice + var removedPaths map[int64]struct{} + partialIndexUsedHint, hasPartialIndex := false, false + for _, path := range ds.PossibleAccessPaths { + // If there is no condition expression, it is not a partial index. + // So we skip it directly. + if path.Index == nil || path.Index.ConditionExprString == "" { + continue + } + hasPartialIndex = true + if columnNames == nil { + columnNames = make(types.NameSlice, 0, ds.schema.Len()) + for i := range ds.Schema().Columns { + columnNames = append(columnNames, &types.FieldName{ + TblName: ds.TableInfo.Name, + ColName: ds.Columns[i].Name, + }) + } + } + // Convert the raw string expression to Expression. + expr, err := expression.ParseSimpleExpr(ds.SCtx().GetExprCtx(), path.Index.ConditionExprString, expression.WithInputSchemaAndNames(ds.schema, columnNames, ds.TableInfo)) + cnfExprs := expression.SplitCNFItems(expr) + if err != nil || !partidx.CheckConstraints(ds.SCtx(), cnfExprs, ds.PushedDownConds) { + if removedPaths == nil { + removedPaths = make(map[int64]struct{}) + } + removedPaths[path.Index.ID] = struct{}{} + continue + } + if path.Forced { + partialIndexUsedHint = true + } + // A special handler for plan cache. + // We only do it for single IS NOT NULL constraint now. + if ds.SCtx().GetSessionVars().StmtCtx.UseCache() { + path.PartIdxCondNotAlwaysValid = !partidx.AlwaysMeetConstraints(ds.SCtx(), cnfExprs, ds.PushedDownConds) + } + } + // 1. No partial index, + // 2. Or no partial index is removed and no partial index is used by hint. + // In these cases, we don't need to do anything. + if !hasPartialIndex || (len(removedPaths) == 0 && !partialIndexUsedHint) { + return + } + checkIndex := func(path *util.AccessPath, checkForced bool) bool { + isRemoved := false + if path.Index != nil { + _, isRemoved = removedPaths[path.Index.ID] + } + return isRemoved || (checkForced && !path.Forced) + } + if partialIndexUsedHint { + ds.AllPossibleAccessPaths = slices.DeleteFunc(ds.AllPossibleAccessPaths, func(path *util.AccessPath) bool { + return checkIndex(path, true) + }) + ds.PossibleAccessPaths = slices.DeleteFunc(ds.PossibleAccessPaths, func(path *util.AccessPath) bool { + return checkIndex(path, true) + }) + } else { + ds.AllPossibleAccessPaths = slices.DeleteFunc(ds.AllPossibleAccessPaths, func(path *util.AccessPath) bool { + return checkIndex(path, false) + }) + ds.PossibleAccessPaths = slices.DeleteFunc(ds.PossibleAccessPaths, func(path *util.AccessPath) bool { + return checkIndex(path, false) + }) + } +} diff --git a/pkg/planner/core/operator/physicalop/physical_index_scan.go b/pkg/planner/core/operator/physicalop/physical_index_scan.go new file mode 100644 index 0000000000000..3569920824b1d --- /dev/null +++ b/pkg/planner/core/operator/physicalop/physical_index_scan.go @@ -0,0 +1,739 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package physicalop + +import ( + "fmt" + "strconv" + "strings" + "unsafe" + + "github.com/pingcap/errors" + "github.com/pingcap/tidb/pkg/expression" + "github.com/pingcap/tidb/pkg/kv" + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/planner/cardinality" + "github.com/pingcap/tidb/pkg/planner/core/access" + "github.com/pingcap/tidb/pkg/planner/core/base" + "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop" + "github.com/pingcap/tidb/pkg/planner/property" + "github.com/pingcap/tidb/pkg/planner/util" + "github.com/pingcap/tidb/pkg/planner/util/costusage" + "github.com/pingcap/tidb/pkg/planner/util/partitionpruning" + "github.com/pingcap/tidb/pkg/planner/util/utilfuncp" + "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" + "github.com/pingcap/tidb/pkg/statistics" + "github.com/pingcap/tidb/pkg/table/tables" + "github.com/pingcap/tidb/pkg/types" + pkgutil "github.com/pingcap/tidb/pkg/util" + "github.com/pingcap/tidb/pkg/util/intest" + "github.com/pingcap/tidb/pkg/util/plancodec" + "github.com/pingcap/tidb/pkg/util/ranger" + "github.com/pingcap/tidb/pkg/util/size" + sliceutil "github.com/pingcap/tidb/pkg/util/slice" + "github.com/pingcap/tidb/pkg/util/stringutil" + "github.com/pingcap/tipb/go-tipb" +) + +// PhysicalIndexScan represents an index scan plan. +type PhysicalIndexScan struct { + PhysicalSchemaProducer + + // AccessCondition is used to calculate range. + AccessCondition []expression.Expression + + Table *model.TableInfo `plan-cache-clone:"shallow"` // please see comment on genPlanCloneForPlanCacheCode. + Index *model.IndexInfo `plan-cache-clone:"shallow"` + IdxCols []*expression.Column + IdxColLens []int + Ranges []*ranger.Range `plan-cache-clone:"shallow"` + Columns []*model.ColumnInfo `plan-cache-clone:"shallow"` + DBName ast.CIStr `plan-cache-clone:"shallow"` + + TableAsName *ast.CIStr `plan-cache-clone:"shallow"` + + // dataSourceSchema is the original schema of DataSource. The schema of index scan in KV and index reader in TiDB + // will be different. The schema of index scan will decode all columns of index but the TiDB only need some of them. + DataSourceSchema *expression.Schema `plan-cache-clone:"shallow"` + + RangeInfo string + + // The index scan may be on a partition. + PhysicalTableID int64 + + GenExprs map[model.TableItemID]expression.Expression `plan-cache-clone:"must-nil"` + + IsPartition bool + Desc bool + KeepOrder bool + // ByItems only for partition table with orderBy + pushedLimit + ByItems []*util.ByItems + + // DoubleRead means if the index executor will read kv two times. + // If the query requires the columns that don't belong to index, DoubleRead will be true. + DoubleRead bool + + NeedCommonHandle bool + + // required by cost model + // tblColHists contains all columns before pruning, which are used to calculate row-size + TblColHists *statistics.HistColl `plan-cache-clone:"shallow"` + PkIsHandleCol *expression.Column + + // ConstColsByCond records the constant part of the index columns caused by the access conds. + // e.g. the index is (a, b, c) and there's filter a = 1 and b = 2, then the column a and b are const part. + ConstColsByCond []bool + + Prop *property.PhysicalProperty `plan-cache-clone:"shallow"` + + // UsedStatsInfo records stats status of this physical table. + // It's for printing stats related information when display execution plan. + UsedStatsInfo *stmtctx.UsedStatsInfoForTable `plan-cache-clone:"shallow"` + + // For GroupedRanges and GroupByColIdxs, please see comments in struct AccessPath. + + GroupedRanges [][]*ranger.Range `plan-cache-clone:"shallow"` + GroupByColIdxs []int `plan-cache-clone:"shallow"` + + NotAlwaysValid bool +} + +// FullRange represent used all partitions. +const FullRange = -1 + +// Clone implements op.PhysicalPlan interface. +func (p *PhysicalIndexScan) Clone(newCtx base.PlanContext) (base.PhysicalPlan, error) { + cloned := new(PhysicalIndexScan) + *cloned = *p + cloned.SetSCtx(newCtx) + base, err := p.PhysicalSchemaProducer.CloneWithSelf(newCtx, cloned) + if err != nil { + return nil, err + } + cloned.PhysicalSchemaProducer = *base + cloned.AccessCondition = util.CloneExprs(p.AccessCondition) + if p.Table != nil { + cloned.Table = p.Table.Clone() + } + if p.Index != nil { + cloned.Index = p.Index.Clone() + } + cloned.IdxCols = util.CloneCols(p.IdxCols) + cloned.IdxColLens = make([]int, len(p.IdxColLens)) + copy(cloned.IdxColLens, p.IdxColLens) + cloned.Ranges = sliceutil.DeepClone(p.Ranges) + cloned.Columns = sliceutil.DeepClone(p.Columns) + if p.DataSourceSchema != nil { + cloned.DataSourceSchema = p.DataSourceSchema.Clone() + } + + return cloned, nil +} + +// ExtractCorrelatedCols implements op.PhysicalPlan interface. +func (p *PhysicalIndexScan) ExtractCorrelatedCols() []*expression.CorrelatedColumn { + corCols := make([]*expression.CorrelatedColumn, 0, len(p.AccessCondition)) + for _, expr := range p.AccessCondition { + corCols = append(corCols, expression.ExtractCorColumns(expr)...) + } + return corCols +} + +const emptyPhysicalIndexScanSize = int64(unsafe.Sizeof(PhysicalIndexScan{})) + +// MemoryUsage return the memory usage of PhysicalIndexScan +func (p *PhysicalIndexScan) MemoryUsage() (sum int64) { + if p == nil { + return + } + + sum = emptyPhysicalIndexScanSize + p.PhysicalSchemaProducer.MemoryUsage() + int64(cap(p.IdxColLens))*size.SizeOfInt + + p.DBName.MemoryUsage() + int64(len(p.RangeInfo)) + int64(len(p.Columns))*model.EmptyColumnInfoSize + if p.TableAsName != nil { + sum += p.TableAsName.MemoryUsage() + } + if p.PkIsHandleCol != nil { + sum += p.PkIsHandleCol.MemoryUsage() + } + if p.Prop != nil { + sum += p.Prop.MemoryUsage() + } + if p.DataSourceSchema != nil { + sum += p.DataSourceSchema.MemoryUsage() + } + // slice memory usage + for _, cond := range p.AccessCondition { + sum += cond.MemoryUsage() + } + for _, col := range p.IdxCols { + sum += col.MemoryUsage() + } + for _, rang := range p.Ranges { + sum += rang.MemUsage() + } + for iid, expr := range p.GenExprs { + sum += int64(unsafe.Sizeof(iid)) + expr.MemoryUsage() + } + return +} + +// Init initializes PhysicalIndexScan. +func (p PhysicalIndexScan) Init(ctx base.PlanContext, offset int) *PhysicalIndexScan { + p.BasePhysicalPlan = NewBasePhysicalPlan(ctx, plancodec.TypeIdxScan, &p, offset) + return &p +} + +// AccessObject implements DataAccesser interface. +func (p *PhysicalIndexScan) AccessObject() base.AccessObject { + res := &access.ScanAccessObject{ + Database: p.DBName.O, + } + tblName := p.Table.Name.O + if p.TableAsName != nil && p.TableAsName.O != "" { + tblName = p.TableAsName.O + } + res.Table = tblName + if p.IsPartition { + pi := p.Table.GetPartitionInfo() + if pi != nil { + partitionName := pi.GetNameByID(p.PhysicalTableID) + res.Partitions = []string{partitionName} + } + } + if len(p.Index.Columns) > 0 { + index := access.IndexAccess{ + Name: p.Index.Name.O, + } + for _, idxCol := range p.Index.Columns { + if tblCol := p.Table.Columns[idxCol.Offset]; tblCol.Hidden { + index.Cols = append(index.Cols, tblCol.GeneratedExprString) + } else { + index.Cols = append(index.Cols, idxCol.Name.O) + } + } + res.Indexes = []access.IndexAccess{index} + } + return res +} + +// ExplainID overrides the ExplainID in order to match different range. +func (p *PhysicalIndexScan) ExplainID(_ ...bool) fmt.Stringer { + return stringutil.MemoizeStr(func() string { + if p.SCtx() != nil && p.SCtx().GetSessionVars().StmtCtx.IgnoreExplainIDSuffix { + return p.TP() + } + return p.TP() + "_" + strconv.Itoa(p.ID()) + }) +} + +// TP overrides the TP in order to match different range. +func (p *PhysicalIndexScan) TP(_ ...bool) string { + if p.IsFullScan() { + return plancodec.TypeIndexFullScan + } + return plancodec.TypeIndexRangeScan +} + +// ExplainInfo implements Plan interface. +func (p *PhysicalIndexScan) ExplainInfo() string { + return p.AccessObject().String() + ", " + p.OperatorInfo(false) +} + +// ExplainNormalizedInfo implements Plan interface. +func (p *PhysicalIndexScan) ExplainNormalizedInfo() string { + return p.AccessObject().NormalizedString() + ", " + p.OperatorInfo(true) +} + +// OperatorInfo implements DataAccesser interface. +func (p *PhysicalIndexScan) OperatorInfo(normalized bool) string { + ectx := p.SCtx().GetExprCtx().GetEvalCtx() + redact := p.SCtx().GetSessionVars().EnableRedactLog + var buffer strings.Builder + if len(p.RangeInfo) > 0 { + if !normalized { + buffer.WriteString("range: decided by ") + buffer.WriteString(p.RangeInfo) + buffer.WriteString(", ") + } + } else if p.haveCorCol() { + if normalized { + buffer.WriteString("range: decided by ") + buffer.Write(expression.SortedExplainNormalizedExpressionList(p.AccessCondition)) + buffer.WriteString(", ") + } else { + buffer.WriteString("range: decided by [") + for i, expr := range p.AccessCondition { + if i != 0 { + buffer.WriteString(" ") + } + buffer.WriteString(expr.StringWithCtx(ectx, redact)) + } + buffer.WriteString("], ") + } + } else if len(p.Ranges) > 0 { + if normalized { + buffer.WriteString("range:[?,?], ") + } else if !p.IsFullScan() { + buffer.WriteString("range:") + for _, idxRange := range p.Ranges { + buffer.WriteString(idxRange.Redact(redact)) + buffer.WriteString(", ") + } + } + } + buffer.WriteString("keep order:") + buffer.WriteString(strconv.FormatBool(p.KeepOrder)) + if p.Desc { + buffer.WriteString(", desc") + } + if !normalized { + if p.UsedStatsInfo != nil { + str := p.UsedStatsInfo.FormatForExplain() + if len(str) > 0 { + buffer.WriteString(", ") + buffer.WriteString(str) + } + } else if p.StatsInfo().StatsVersion == statistics.PseudoVersion { + // This branch is not needed in fact, we add this to prevent test result changes under planner/cascades/ + buffer.WriteString(", stats:pseudo") + } + } + return buffer.String() +} + +func (p *PhysicalIndexScan) haveCorCol() bool { + for _, cond := range p.AccessCondition { + if len(expression.ExtractCorColumns(cond)) > 0 { + return true + } + } + return false +} + +// IsFullScan checks whether the index scan covers the full range of the index. +func (p *PhysicalIndexScan) IsFullScan() bool { + if len(p.RangeInfo) > 0 || p.haveCorCol() { + return false + } + for _, ran := range p.Ranges { + if !ran.IsFullRange(false) { + return false + } + } + return true +} + +// GetScanRowSize calculates the average row size for the index scan operation. +func (p *PhysicalIndexScan) GetScanRowSize() float64 { + idx := p.Index + scanCols := make([]*expression.Column, 0, len(idx.Columns)+1) + // If `initSchema` has already appended the handle column in schema, just use schema columns, otherwise, add extra handle column. + if len(idx.Columns) == len(p.Schema().Columns) { + scanCols = append(scanCols, p.Schema().Columns...) + handleCol := p.PkIsHandleCol + if handleCol != nil { + scanCols = append(scanCols, handleCol) + } + } else { + scanCols = p.Schema().Columns + } + return cardinality.GetIndexAvgRowSize(p.SCtx(), p.TblColHists, scanCols, p.Index.Unique) +} + +// InitSchema is used to set the schema of PhysicalIndexScan. Before calling this, +// make sure the following field of PhysicalIndexScan are initialized: +// +// PhysicalIndexScan.Table *model.TableInfo +// PhysicalIndexScan.Index *model.IndexInfo +// PhysicalIndexScan.Index.Columns []*IndexColumn +// PhysicalIndexScan.IdxCols []*expression.Column +// PhysicalIndexScan.Columns []*model.ColumnInfo +func (p *PhysicalIndexScan) InitSchema(idxExprCols []*expression.Column, isDoubleRead bool) { + indexCols := make([]*expression.Column, len(p.IdxCols), len(p.Index.Columns)+1) + copy(indexCols, p.IdxCols) + + for i := len(p.IdxCols); i < len(p.Index.Columns); i++ { + if idxExprCols[i] != nil { + indexCols = append(indexCols, idxExprCols[i]) + } else { + // TODO: try to reuse the col generated when building the DataSource. + indexCols = append(indexCols, &expression.Column{ + ID: p.Table.Columns[p.Index.Columns[i].Offset].ID, + RetType: &p.Table.Columns[p.Index.Columns[i].Offset].FieldType, + UniqueID: p.SCtx().GetSessionVars().AllocPlanColumnID(), + }) + } + } + p.NeedCommonHandle = p.Table.IsCommonHandle + + if p.NeedCommonHandle { + for i := len(p.Index.Columns); i < len(idxExprCols); i++ { + indexCols = append(indexCols, idxExprCols[i]) + } + } + setHandle := len(indexCols) > len(p.Index.Columns) + if !setHandle { + for i, col := range p.Columns { + if (mysql.HasPriKeyFlag(col.GetFlag()) && p.Table.PKIsHandle) || col.ID == model.ExtraHandleID { + indexCols = append(indexCols, p.DataSourceSchema.Columns[i]) + setHandle = true + break + } + } + } + + var extraPhysTblCol *expression.Column + // If `dataSouceSchema` contains `model.ExtraPhysTblID`, we should add it into `indexScan.schema` + for _, col := range p.DataSourceSchema.Columns { + if col.ID == model.ExtraPhysTblID { + extraPhysTblCol = col.Clone().(*expression.Column) + break + } + } + + if isDoubleRead || p.Index.Global { + // If it's double read case, the first index must return handle. So we should add extra handle column + // if there isn't a handle column. + if !setHandle { + if !p.Table.IsCommonHandle { + indexCols = append(indexCols, &expression.Column{ + RetType: types.NewFieldType(mysql.TypeLonglong), + ID: model.ExtraHandleID, + UniqueID: p.SCtx().GetSessionVars().AllocPlanColumnID(), + OrigName: model.ExtraHandleName.O, + }) + } + } + // If it's global index, handle and PhysTblID columns has to be added, so that needed pids can be filtered. + if p.Index.Global && extraPhysTblCol == nil { + indexCols = append(indexCols, &expression.Column{ + RetType: types.NewFieldType(mysql.TypeLonglong), + ID: model.ExtraPhysTblID, + UniqueID: p.SCtx().GetSessionVars().AllocPlanColumnID(), + OrigName: model.ExtraPhysTblIDName.O, + }) + } + } + + if extraPhysTblCol != nil { + indexCols = append(indexCols, extraPhysTblCol) + } + + p.SetSchema(expression.NewSchema(indexCols...)) +} + +// AddSelectionConditionForGlobalIndex adds partition filtering conditions for global index scans. +func (p *PhysicalIndexScan) AddSelectionConditionForGlobalIndex(ds *logicalop.DataSource, physPlanPartInfo *PhysPlanPartInfo, conditions []expression.Expression) ([]expression.Expression, error) { + if !p.Index.Global { + return conditions, nil + } + args := make([]expression.Expression, 0, len(ds.PartitionNames)+1) + for _, col := range p.Schema().Columns { + if col.ID == model.ExtraPhysTblID { + args = append(args, col.Clone()) + break + } + } + + if len(args) != 1 { + return nil, errors.Errorf("Can't find column %s in schema %s", model.ExtraPhysTblIDName.O, p.Schema()) + } + + // For SQL like 'select x from t partition(p0, p1) use index(idx)', + // we will add a `Selection` like `in(t._tidb_pid, p0, p1)` into the plan. + // For truncate/drop partitions, we should only return indexes where partitions still in public state. + idxArr, err := partitionpruning.PartitionPruning(ds.SCtx(), ds.Table.GetPartitionedTable(), + physPlanPartInfo.PruningConds, + physPlanPartInfo.PartitionNames, + physPlanPartInfo.Columns, + physPlanPartInfo.ColumnNames) + if err != nil { + return nil, err + } + needNot := false + // TODO: Move all this into PartitionPruning or the PartitionProcessor! + pInfo := ds.TableInfo.GetPartitionInfo() + if len(idxArr) == 1 && idxArr[0] == FullRange { + // Filter away partitions that may exists in Global Index, + // but should not be seen. + needNot = true + for _, id := range pInfo.IDsInDDLToIgnore() { + args = append(args, expression.NewInt64Const(id)) + } + } else if len(idxArr) == 0 { + // TODO: Can we change to Table Dual somehow? + // Add an invalid pid as param for `IN` function + args = append(args, expression.NewInt64Const(-1)) + } else { + // TODO: When PartitionPruning is guaranteed to not + // return old/blocked partition ids then ignoreMap can be removed. + ignoreMap := make(map[int64]struct{}) + for _, id := range pInfo.IDsInDDLToIgnore() { + ignoreMap[id] = struct{}{} + } + for _, idx := range idxArr { + id := pInfo.Definitions[idx].ID + _, ok := ignoreMap[id] + if !ok { + args = append(args, expression.NewInt64Const(id)) + } + intest.Assert(!ok, "PartitionPruning returns partitions which should be ignored!") + } + } + if len(args) == 1 { + return conditions, nil + } + condition, err := expression.NewFunction(ds.SCtx().GetExprCtx(), ast.In, types.NewFieldType(mysql.TypeLonglong), args...) + if err != nil { + return nil, err + } + if needNot { + condition, err = expression.NewFunction(ds.SCtx().GetExprCtx(), ast.UnaryNot, types.NewFieldType(mysql.TypeLonglong), condition) + if err != nil { + return nil, err + } + } + return append(conditions, condition), nil +} + +// NeedExtraOutputCol is designed for check whether need an extra column for +// pid or physical table id when build indexReq. +func (p *PhysicalIndexScan) NeedExtraOutputCol() bool { + if p.Table.Partition == nil { + return false + } + // has global index, should return pid + if p.Index.Global { + return true + } + // has embedded limit, should return physical table id + if len(p.ByItems) != 0 && p.SCtx().GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { + return true + } + return false +} + +// IsPartitionTable returns true and partition ID if it works on a partition. +func (p *PhysicalIndexScan) IsPartitionTable() (bool, int64) { + return p.IsPartition, p.PhysicalTableID +} + +// IsPointGetByUniqueKey checks whether is a point get by unique key. +func (p *PhysicalIndexScan) IsPointGetByUniqueKey(tc types.Context) bool { + return len(p.Ranges) == 1 && + p.Index.Unique && + len(p.Ranges[0].LowVal) == len(p.Index.Columns) && + p.Ranges[0].IsPointNonNullable(tc) +} + +// ToPB implements PhysicalPlan ToPB interface. +func (p *PhysicalIndexScan) ToPB(_ *base.BuildPBContext, _ kv.StoreType) (*tipb.Executor, error) { + columns := make([]*model.ColumnInfo, 0, p.Schema().Len()) + tableColumns := p.Table.Cols() + for _, col := range p.Schema().Columns { + if col.ID == model.ExtraHandleID { + columns = append(columns, model.NewExtraHandleColInfo()) + } else if col.ID == model.ExtraPhysTblID { + columns = append(columns, model.NewExtraPhysTblIDColInfo()) + } else { + columns = append(columns, model.FindColumnInfoByID(tableColumns, col.ID)) + } + } + var pkColIDs []int64 + if p.NeedCommonHandle { + pkColIDs = tables.TryGetCommonPkColumnIds(p.Table) + } + idxExec := &tipb.IndexScan{ + TableId: p.Table.ID, + IndexId: p.Index.ID, + Columns: pkgutil.ColumnsToProto(columns, p.Table.PKIsHandle, true, false), + Desc: p.Desc, + PrimaryColumnIds: pkColIDs, + } + if p.IsPartition { + idxExec.TableId = p.PhysicalTableID + } + unique := checkCoverIndex(p.Index, p.Ranges) + idxExec.Unique = &unique + return &tipb.Executor{Tp: tipb.ExecType_TypeIndexScan, IdxScan: idxExec}, nil +} + +// checkCoverIndex checks whether we can pass unique info to TiKV. We should push it if and only if the length of +// range and index are equal. +func checkCoverIndex(idx *model.IndexInfo, ranges []*ranger.Range) bool { + // If the index is (c1, c2) but the query range only contains c1, it is not a unique get. + if !idx.Unique { + return false + } + for _, rg := range ranges { + if len(rg.LowVal) != len(idx.Columns) { + return false + } + for _, v := range rg.LowVal { + if v.IsNull() { + // a unique index may have duplicated rows with NULLs, so we cannot set the unique attribute to true when the range has NULL + // please see https://github.com/pingcap/tidb/issues/29650 for more details + return false + } + } + for _, v := range rg.HighVal { + if v.IsNull() { + return false + } + } + } + return true +} + +// GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *PhysicalIndexScan) GetPlanCostVer1(taskType property.TaskType, + option *costusage.PlanCostOption) (float64, error) { + return utilfuncp.GetPlanCostVer14PhysicalIndexScan(p, taskType, option) +} + +// GetPlanCostVer2 returns the plan-cost of this sub-plan, which is: +// plan-cost = rows * log2(row-size) * scan-factor +// log2(row-size) is from experiments. +func (p *PhysicalIndexScan) GetPlanCostVer2(taskType property.TaskType, + option *costusage.PlanCostOption, args ...bool) (costusage.CostVer2, error) { + return utilfuncp.GetPlanCostVer24PhysicalIndexScan(p, taskType, option, args...) +} + +// GetPhysicalIndexScan4LogicalIndexScan returns PhysicalIndexScan for the logical IndexScan. +func GetPhysicalIndexScan4LogicalIndexScan(s *logicalop.LogicalIndexScan, _ *expression.Schema, stats *property.StatsInfo) *PhysicalIndexScan { + ds := s.Source + is := PhysicalIndexScan{ + Table: ds.TableInfo, + TableAsName: ds.TableAsName, + DBName: ds.DBName, + Columns: s.Columns, + Index: s.Index, + IdxCols: s.IdxCols, + IdxColLens: s.IdxColLens, + AccessCondition: s.AccessConds, + Ranges: s.Ranges, + DataSourceSchema: ds.Schema(), + IsPartition: ds.PartitionDefIdx != nil, + PhysicalTableID: ds.PhysicalTableID, + TblColHists: ds.TblColHists, + PkIsHandleCol: ds.GetPKIsHandleCol(), + }.Init(ds.SCtx(), ds.QueryBlockOffset()) + is.SetStats(stats) + is.InitSchema(s.FullIdxCols, s.IsDoubleRead) + return is +} + +// GetOriginalPhysicalIndexScan creates a new PhysicalIndexScan from the given DataSource and AccessPath. +func GetOriginalPhysicalIndexScan(ds *logicalop.DataSource, prop *property.PhysicalProperty, path *util.AccessPath, isMatchProp bool, isSingleScan bool) *PhysicalIndexScan { + idx := path.Index + is := PhysicalIndexScan{ + Table: ds.TableInfo, + TableAsName: ds.TableAsName, + DBName: ds.DBName, + Columns: sliceutil.DeepClone(ds.Columns), + Index: idx, + IdxCols: path.IdxCols, + IdxColLens: path.IdxColLens, + AccessCondition: path.AccessConds, + Ranges: path.Ranges, + DataSourceSchema: ds.Schema(), + IsPartition: ds.PartitionDefIdx != nil, + PhysicalTableID: ds.PhysicalTableID, + TblColHists: ds.TblColHists, + PkIsHandleCol: ds.GetPKIsHandleCol(), + ConstColsByCond: path.ConstCols, + Prop: prop, + NotAlwaysValid: path.PartIdxCondNotAlwaysValid, + }.Init(ds.SCtx(), ds.QueryBlockOffset()) + rowCount := path.CountAfterAccess + is.InitSchema(append(path.FullIdxCols, ds.CommonHandleCols...), !isSingleScan) + + // If (1) tidb_opt_ordering_index_selectivity_threshold is enabled (not 0) + // and (2) there exists an index whose selectivity is smaller than or equal to the threshold, + // and (3) there is Selection on the IndexScan, we don't use the ExpectedCnt to + // adjust the estimated row count of the IndexScan. + ignoreExpectedCnt := ds.SCtx().GetSessionVars().OptOrderingIdxSelThresh != 0 && + ds.AccessPathMinSelectivity <= ds.SCtx().GetSessionVars().OptOrderingIdxSelThresh && + len(path.IndexFilters)+len(path.TableFilters) > 0 + + if (isMatchProp || prop.IsSortItemEmpty()) && prop.ExpectedCnt < ds.StatsInfo().RowCount && !ignoreExpectedCnt { + rowCount = cardinality.AdjustRowCountForIndexScanByLimit(ds.SCtx(), + ds.StatsInfo(), ds.TableStats, ds.StatisticTable, + path, prop.ExpectedCnt, isMatchProp && prop.SortItems[0].Desc) + } + + // ScaleByExpectCnt only allows to scale the row count smaller than the table total row count. + // But for MV index, it's possible that the IndexRangeScan row count is larger than the table total row count. + // Please see the Case 2 in CalcTotalSelectivityForMVIdxPath for an example. + if idx.MVIndex && rowCount > ds.TableStats.RowCount { + is.SetStats(ds.TableStats.Scale(ds.SCtx().GetSessionVars(), rowCount/ds.TableStats.RowCount)) + } else { + is.SetStats(ds.TableStats.ScaleByExpectCnt(ds.SCtx().GetSessionVars(), rowCount)) + } + usedStats := ds.SCtx().GetSessionVars().StmtCtx.GetUsedStatsInfo(false) + if usedStats != nil && usedStats.GetUsedInfo(is.PhysicalTableID) != nil { + is.UsedStatsInfo = usedStats.GetUsedInfo(is.PhysicalTableID) + } + if isMatchProp { + is.Desc = prop.SortItems[0].Desc + is.KeepOrder = true + } + return is +} + +// ConvertToPartialIndexScan converts a DataSource to a PhysicalIndexScan for IndexMerge. +func ConvertToPartialIndexScan(ds *logicalop.DataSource, physPlanPartInfo *PhysPlanPartInfo, prop *property.PhysicalProperty, path *util.AccessPath, matchProp property.PhysicalPropMatchResult, byItems []*util.ByItems) (base.PhysicalPlan, []expression.Expression, error) { + intest.Assert(matchProp != property.PropMatchedNeedMergeSort, + "partial paths of index merge path should not match property using merge sort") + is := GetOriginalPhysicalIndexScan(ds, prop, path, matchProp.Matched(), false) + // TODO: Consider using isIndexCoveringColumns() to avoid another TableRead + indexConds := path.IndexFilters + if matchProp.Matched() { + if is.Table.GetPartitionInfo() != nil && !is.Index.Global && is.SCtx().GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { + tmpColumns, tmpSchema, _ := AddExtraPhysTblIDColumn(is.SCtx(), is.Columns, is.Schema()) + is.Columns = tmpColumns + is.SetSchema(tmpSchema) + } + // Add sort items for index scan for merge-sort operation between partitions. + is.ByItems = byItems + } + + // Add a `Selection` for `IndexScan` with global index. + // It should pushdown to TiKV, DataSource schema doesn't contain partition id column. + indexConds, err := is.AddSelectionConditionForGlobalIndex(ds, physPlanPartInfo, indexConds) + if err != nil { + return nil, nil, err + } + + if len(indexConds) > 0 { + pushedFilters, remainingFilter := extractFiltersForIndexMerge(util.GetPushDownCtx(ds.SCtx()), indexConds) + var selectivity float64 + if path.CountAfterAccess > 0 { + selectivity = path.CountAfterIndex / path.CountAfterAccess + } + rowCount := is.StatsInfo().RowCount * selectivity + stats := &property.StatsInfo{RowCount: rowCount} + stats.StatsVersion = ds.StatisticTable.Version + if ds.StatisticTable.Pseudo { + stats.StatsVersion = statistics.PseudoVersion + } + indexPlan := PhysicalSelection{Conditions: pushedFilters}.Init(is.SCtx(), stats, ds.QueryBlockOffset()) + indexPlan.SetChildren(is) + return indexPlan, remainingFilter, nil + } + return is, nil, nil +} diff --git a/pkg/planner/core/partidx/BUILD.bazel b/pkg/planner/core/partidx/BUILD.bazel new file mode 100644 index 0000000000000..361c0cf0504bb --- /dev/null +++ b/pkg/planner/core/partidx/BUILD.bazel @@ -0,0 +1,16 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "partidx", + srcs = ["check_constraint.go"], + importpath = "github.com/pingcap/tidb/pkg/planner/core/partidx", + visibility = ["//visibility:public"], + deps = [ + "//pkg/expression", + "//pkg/parser/ast", + "//pkg/planner/planctx", + "//pkg/util/intest", + "//pkg/util/ranger", + "//pkg/util/ranger/context", + ], +) diff --git a/pkg/planner/core/partidx/check_constraint.go b/pkg/planner/core/partidx/check_constraint.go new file mode 100644 index 0000000000000..56b024909778c --- /dev/null +++ b/pkg/planner/core/partidx/check_constraint.go @@ -0,0 +1,221 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package partidx + +import ( + "github.com/pingcap/tidb/pkg/expression" + "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/planner/planctx" + "github.com/pingcap/tidb/pkg/util/intest" + "github.com/pingcap/tidb/pkg/util/ranger" + "github.com/pingcap/tidb/pkg/util/ranger/context" +) + +// CheckConstraints checks whether the filters can meet the constraints from the predefined predicates in index meta. +// If the prePredicates has comparison filters like =, >, <, >=, <=, IN, IS NULL, IS NOT NULL on single column, we can try our best to match them. +// If the prePredicates has other filters like `sin(a) > 0`, we can only try to match them exactly. +// TODO: now we only support the comparison filters on single column. +func CheckConstraints(sctx planctx.PlanContext, prePredicates []expression.Expression, filters []expression.Expression) bool { + if len(prePredicates) == 0 { + return true + } + intest.Assert(len(prePredicates) == 1) + if exactMatch(sctx.GetExprCtx().GetEvalCtx(), prePredicates, filters) { + return true + } + + if canBeImpliedFromExprs(sctx.GetRangerCtx(), prePredicates[0], filters) { + return true + } + + return false +} + +func exactMatch(evalctx expression.EvalContext, prePredicates []expression.Expression, filters []expression.Expression) bool { + matched := make([]bool, len(filters)) + for _, pre := range prePredicates { + found := false + for i, filter := range filters { + if matched[i] { + continue + } + if pre.Equal(evalctx, filter) { + matched[i] = true + found = true + break + } + } + if !found { + return false + } + } + return true +} + +func canBeImpliedFromExprs( + rangerctx *context.RangerContext, + pre expression.Expression, + filters []expression.Expression, +) bool { + sf := pre.(*expression.ScalarFunction) + + if sf.FuncName.L == ast.UnaryNot { + nf, ok := sf.GetArgs()[0].(*expression.ScalarFunction) + if !ok || nf.FuncName.L != ast.IsNull { + return false + } + col, ok := nf.GetArgs()[0].(*expression.Column) + if !ok { + return false + } + return implIsNotNull(rangerctx, col, filters) + } + + if _, ok := expression.CompareOpMap[sf.FuncName.L]; !ok { + return false + } + return implCompareExpr(rangerctx, sf, filters) +} + +func implCompareExpr(rangerctx *context.RangerContext, pre *expression.ScalarFunction, filters []expression.Expression) bool { + var col *expression.Column + if _, ok := pre.GetArgs()[0].(*expression.Column); ok { + col = pre.GetArgs()[0].(*expression.Column) + } else if _, ok := pre.GetArgs()[1].(*expression.Column); ok { + col = pre.GetArgs()[1].(*expression.Column) + } else { + return false + } + ranges, _, _, err := ranger.BuildColumnRange([]expression.Expression{pre}, rangerctx, col.RetType, -1, 0) + if len(ranges) == 0 || err != nil { + return false + } + columnConds := ranger.ExtractAccessConditionsForColumn(rangerctx, filters, col) + if len(columnConds) == 0 { + return false + } + rangesFromFilters, _, _, err := ranger.BuildColumnRange(columnConds, rangerctx, col.RetType, -1, 0) + if len(rangesFromFilters) == 0 || err != nil { + return false + } + unionedBuff := make([]*ranger.Range, len(rangesFromFilters)+len(ranges)) + copy(unionedBuff, rangesFromFilters) + copy(unionedBuff[len(rangesFromFilters):], ranges) + unionedRange, err := ranger.UnionRanges(rangerctx, unionedBuff, false) + if err != nil { + return false + } + if len(unionedRange) != len(ranges) { + return false + } + for i, ran := range unionedRange { + if !ran.Equal(ranges[i]) { + return false + } + } + return true +} + +func implIsNotNull(rangerctx *context.RangerContext, targetCol *expression.Column, filters []expression.Expression) bool { + columnConds := ranger.ExtractAccessConditionsForColumn(rangerctx, filters, targetCol) + if len(columnConds) == 0 { + return false + } + rangesFromFilters, _, _, err := ranger.BuildColumnRange(columnConds, rangerctx, targetCol.RetType, -1, 0) + if len(rangesFromFilters) == 0 || err != nil { + return false + } + for _, ran := range rangesFromFilters { + if ran.LowVal[0].IsNull() && !ran.LowExclude { + return false + } + } + return true +} + +// AlwaysMeetConstraints checks whether the filters always meet the constraints from the predefined predicates in index meta. +// This check is only applied to the pre condition that is single IS NOT NULL. +// e.g. for partial index idx(b) where a IS NOT NULL, if the filter is b = ? and a is not null/a > 10, then we can guarantee that a IS NOT NULL is always true. +// Because the `IsNullRejected` has correctness issues. So we implement a simpler version here. +func AlwaysMeetConstraints(sctx planctx.PlanContext, prePredicates, filters []expression.Expression) bool { + if len(prePredicates) != 1 { + return false + } + sf, ok := prePredicates[0].(*expression.ScalarFunction) + if !ok || sf.FuncName.L != ast.UnaryNot { + return false + } + innerSf, ok := sf.GetArgs()[0].(*expression.ScalarFunction) + if !ok || innerSf.FuncName.L != ast.IsNull { + return false + } + col, ok := innerSf.GetArgs()[0].(*expression.Column) + if !ok { + return false + } + // When the index is chosen, the given filters can not be empty. + for _, filter := range filters { + sf, ok := filter.(*expression.ScalarFunction) + if !ok { + continue + } + if checkIsNullRejected(sctx, col, sf) { + return true + } + } + return false +} + +func checkIsNullRejected(sctx planctx.PlanContext, targetCol *expression.Column, filter *expression.ScalarFunction) bool { + if filter.FuncName.L == ast.LogicOr { + leavesAllRejected := true + for _, arg := range filter.GetArgs() { + sf, ok := arg.(*expression.ScalarFunction) + if !ok || !checkIsNullRejected(sctx, targetCol, sf) { + leavesAllRejected = false + break + } + } + return leavesAllRejected + } + if filter.FuncName.L == ast.LogicAnd { + for _, arg := range filter.GetArgs() { + sf, ok := arg.(*expression.ScalarFunction) + if ok && checkIsNullRejected(sctx, targetCol, sf) { + return true + } + } + return false + } + if filter.FuncName.L == ast.IsNull { + col, ok := filter.GetArgs()[0].(*expression.Column) + if ok && col.Equal(sctx.GetExprCtx().GetEvalCtx(), targetCol) { + return false + } + } + if _, ok := expression.CompareOpMap[filter.FuncName.L]; ok { + if filter.FuncName.L == ast.NullEQ { + return false + } + col, ok := filter.GetArgs()[0].(*expression.Column) + if !ok { + col, ok = filter.GetArgs()[1].(*expression.Column) + } + if ok && col.Equal(sctx.GetExprCtx().GetEvalCtx(), targetCol) { + return true + } + } + return false +} diff --git a/pkg/planner/core/plan_cacheable_checker.go b/pkg/planner/core/plan_cacheable_checker.go index 112dab3200177..53d55a1dc6f29 100644 --- a/pkg/planner/core/plan_cacheable_checker.go +++ b/pkg/planner/core/plan_cacheable_checker.go @@ -582,6 +582,7 @@ func isPhysicalPlanCacheable(sctx base.PlanContext, p base.PhysicalPlan, paramNu return false, "the plan with IndexMerge accessing Multi-Valued Index is un-cacheable" } underIndexMerge = true +<<<<<<< HEAD subPlans = append(subPlans, x.partialPlans...) case *PhysicalIndexScan: if underIndexMerge && x.isFullScan() { @@ -589,6 +590,23 @@ func isPhysicalPlanCacheable(sctx base.PlanContext, p base.PhysicalPlan, paramNu } case *PhysicalTableScan: if underIndexMerge && x.isFullScan() { +======= + subPlans = append(subPlans, x.PartialPlansRaw...) + case *physicalop.PhysicalIndexReader: + subPlans = append(subPlans, x.IndexPlan) + case *physicalop.PhysicalIndexLookUpReader: + // Currently, there's no need to check the table plan of the IndexLookUpReader. + subPlans = append(subPlans, x.IndexPlan) + case *physicalop.PhysicalIndexScan: + if underIndexMerge && x.IsFullScan() { + return false, "IndexMerge plan with full-scan is un-cacheable" + } + if x.Index.HasCondition() && x.NotAlwaysValid { + return false, "IndexScan of partial index is un-cacheable" + } + case *physicalop.PhysicalTableScan: + if underIndexMerge && x.IsFullScan() { +>>>>>>> 959bf330874 (planner: support basic usage of partial index (#65051)) return false, "IndexMerge plan with full-scan is un-cacheable" } case *PhysicalApply: diff --git a/pkg/planner/core/planbuilder.go b/pkg/planner/core/planbuilder.go index 09d433dbb9967..73b9be82771b1 100644 --- a/pkg/planner/core/planbuilder.go +++ b/pkg/planner/core/planbuilder.go @@ -1451,15 +1451,22 @@ func getPossibleAccessPaths(ctx base.PlanContext, tableHints *hint.PlanHints, in available = append(available, tablePath) } - // If all available paths are Multi-Valued Index, it's possible that the only multi-valued index is inapplicable, + // If all available paths are Multi-Valued Index, Partial index or other index that need to check its usability in later phase, + // it's possible that all these path are inapplicable, // so that the table paths are still added here to avoid failing to find any physical plan. - allMVIIndexPath := true + allUndeterminedPath := true for _, availablePath := range available { +<<<<<<< HEAD if !isMVIndexPath(availablePath) { allMVIIndexPath = false +======= + if !availablePath.IsUndetermined() { + allUndeterminedPath = false + break +>>>>>>> 959bf330874 (planner: support basic usage of partial index (#65051)) } } - if allMVIIndexPath { + if allUndeterminedPath { available = append(available, tablePath) } diff --git a/pkg/planner/core/point_get_plan.go b/pkg/planner/core/point_get_plan.go index 3a8a6c0fd0700..0c0e7c28bc5e7 100644 --- a/pkg/planner/core/point_get_plan.go +++ b/pkg/planner/core/point_get_plan.go @@ -1172,14 +1172,14 @@ func newBatchPointGetPlan( } for _, idxInfo := range tbl.Indices { if !idxInfo.Unique || idxInfo.State != model.StatePublic || (idxInfo.Invisible && !ctx.GetSessionVars().OptimizerUseInvisibleIndexes) || idxInfo.MVIndex || - !indexIsAvailableByHints( - ctx.GetSessionVars().CurrentDB, - dbName, - tblAlias, - idxInfo, - tblHints, - indexHints, - ) { + idxInfo.HasCondition() || !indexIsAvailableByHints( + ctx.GetSessionVars().CurrentDB, + dbName, + tblAlias, + idxInfo, + tblHints, + indexHints, + ) { continue } if len(idxInfo.Columns) != len(whereColNames) || idxInfo.HasPrefixIndex() { @@ -1544,14 +1544,14 @@ func checkTblIndexForPointPlan(ctx base.PlanContext, tblName *resolve.TableNameW } for _, idxInfo := range tbl.Indices { if !idxInfo.Unique || idxInfo.State != model.StatePublic || (idxInfo.Invisible && !ctx.GetSessionVars().OptimizerUseInvisibleIndexes) || idxInfo.MVIndex || - !indexIsAvailableByHints( - ctx.GetSessionVars().CurrentDB, - dbName, - tblAlias, - idxInfo, - tblHints, - tblName.IndexHints, - ) { + idxInfo.HasCondition() || !indexIsAvailableByHints( + ctx.GetSessionVars().CurrentDB, + dbName, + tblAlias, + idxInfo, + tblHints, + tblName.IndexHints, + ) { continue } if idxInfo.Global { diff --git a/pkg/planner/core/rule/rule_prune_indexes.go b/pkg/planner/core/rule/rule_prune_indexes.go new file mode 100644 index 0000000000000..6623c074cb331 --- /dev/null +++ b/pkg/planner/core/rule/rule_prune_indexes.go @@ -0,0 +1,577 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rule + +import ( + "fmt" + "slices" + "strings" + + "github.com/pingcap/failpoint" + "github.com/pingcap/tidb/pkg/expression" + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop" + "github.com/pingcap/tidb/pkg/planner/util" + "github.com/pingcap/tidb/pkg/planner/util/fixcontrol" +) + +const ( + // defaultMaxIndexes is the default maximum number of indexes to keep when pruning. + // This prevents overly aggressive pruning when the threshold is small. + // TODO: We should add a second pruning phase around the fillIndexPath step, + // where we can refine the indexes further based on the actual column statistics. + // Therefore, if a customer did set tidn_opt_index_prune_threshold < 10, we could + // a minimum of 10 in the first pruning phase, and prune further in the second phase. + defaultMaxIndexes = 10 +) + +// indexWithScore stores an access path along with its coverage scores for ranking. +type indexWithScore struct { + path *util.AccessPath + interestingCount int // Total number of interesting columns covered + consecutiveColumnIDs []int64 // IDs of consecutive columns (for detecting different orderings) +} + +// columnRequirements holds the column maps needed for index pruning. +type columnRequirements struct { + interestingColIDs map[int64]struct{} +} + +// ShouldPreferIndexMerge returns true if index merge should be preferred, either due to hints or fix control. +func ShouldPreferIndexMerge(ds *logicalop.DataSource) bool { + return len(ds.IndexMergeHints) > 0 || fixcontrol.GetBoolWithDefault( + ds.SCtx().GetSessionVars().GetOptimizerFixControlMap(), + fixcontrol.Fix52869, + false, + ) +} + +// PruneIndexesByWhereAndOrder prunes indexes based on their coverage of interesting columns. +// It keeps the most promising indexes up to the threshold, prioritizing those that: +// 1. Cover more interesting columns +// 2. Have consecutive column matches from the index start (enabling index prefix usage) +// 3. Support single-scan (covering index without table lookups) +// 4. Have different consecutive column orderings (e.g., if interesting columns are A, B, keep both (A,B) and (B,A)) +// The threshold controls the behavior: +// threshold = -1: disable pruning (handled by caller) +// threshold = 0: only prune indexes with no interesting columns (score == 0) +// threshold > 0: keep at least threshold indexes (but at least defaultMaxIndexes) +// but if there are fewer than threshold indexes, we will still prune zero-score indexes. +func PruneIndexesByWhereAndOrder(ds *logicalop.DataSource, paths []*util.AccessPath, interestingColumns []*expression.Column, threshold int) []*util.AccessPath { + if len(paths) <= 1 { + return paths + } + + totalPathCount := len(paths) + + // If we disabled the prune, return directly. + if threshold < 0 { + return paths + } + // Now the prune must happen. + + // If threshold is 0 or greater than total paths, we only prune zero-score indexes. + onlyPruneZeroScore := threshold == 0 || (threshold > totalPathCount) + + // Build column ID maps and calculate totals + req := buildColumnRequirements(interestingColumns) + + preferredIndexes := make([]indexWithScore, 0, totalPathCount) + tablePaths := make([]*util.AccessPath, 0, 1) + mvIndexPaths := make([]*util.AccessPath, 0, 1) + indexMergeIndexPaths := make([]*util.AccessPath, 0, 1) + preferMerge := ShouldPreferIndexMerge(ds) + + // Check if IndexMerge hints specify specific index names + // We need to import the function from indexmerge_path.go, but since it's in a different package, + // we'll implement the check inline here + hasSpecifiedIndexes := false + if len(ds.IndexMergeHints) > 0 { + for _, hint := range ds.IndexMergeHints { + if hint.IndexHint != nil && len(hint.IndexHint.IndexNames) > 0 { + hasSpecifiedIndexes = true + break + } + } + } + + // Categorize each index path + for _, path := range paths { + if path.IsTablePath() { + tablePaths = append(tablePaths, path) + continue + } + + // Always keep multi-value indexes (like table paths) + if path.Index != nil && path.Index.MVIndex { + mvIndexPaths = append(mvIndexPaths, path) + continue + } + + // If we have forced paths, we shouldn't prune any paths + if path.Forced { + return paths + } + + // Skip paths with nil Index + if path.Index == nil { + continue + } + + // Calculate coverage for this index + // Use TableInfo.Columns (not ds.Columns) because IndexColumn.Offset refers to TableInfo.Columns + var tableColumns []*model.ColumnInfo + if ds.TableInfo != nil { + tableColumns = ds.TableInfo.Columns + } + idxScore := scoreIndexPath(ds, path, req, tableColumns) + + if path.FullIdxCols != nil { + path.IsSingleScan = ds.IsSingleScan(path.FullIdxCols, path.FullIdxColLens) + } + + // Check if this index is specified in IndexMerge hints + // Note: Indexes specified in USE_INDEX_MERGE(t, idx1, idx2) are NOT marked as "forced" + // (only USE_INDEX/FORCE_INDEX mark indexes as forced), so we need to collect them here + // to ensure they're not pruned. This is only needed when hints specify specific index names. + if hasSpecifiedIndexes { + indexName := path.Index.Name.L + isSpecified := false + for _, hint := range ds.IndexMergeHints { + if hint.IndexHint == nil || len(hint.IndexHint.IndexNames) == 0 { + continue + } + for _, hintName := range hint.IndexHint.IndexNames { + // Use case-insensitive comparison like isSpecifiedInIndexMergeHints does + if strings.EqualFold(indexName, hintName.String()) { + isSpecified = true + break + } + } + if isSpecified { + break + } + } + if isSpecified { + // This index is explicitly specified in IndexMerge hints, keep it + // Add it to indexMergeIndexPaths so it's guaranteed to be included even if it doesn't score well + indexMergeIndexPaths = append(indexMergeIndexPaths, path) + continue + } + } + + // If index merge is preferred (via general hints without index names, or fix control), + // keep indexes that have any coverage. Note: When specific indexes are mentioned in hints, + // those are handled above. Here we handle general IndexMerge hints (no specific index names) + // or fix control. We still apply some filtering (len(consecutiveColumnIDs) > 0 OR other coverage) + // to avoid keeping completely useless indexes, but we're more lenient than normal pruning. + if preferMerge && !hasSpecifiedIndexes { + // When IndexMerge is preferred without specific index names, keep any index with coverage + if len(idxScore.consecutiveColumnIDs) > 0 || path.IsSingleScan || idxScore.interestingCount > 0 { + preferredIndexes = append(preferredIndexes, idxScore) + continue + } + } + + // Add to preferred indexes if it has any coverage or is a covering scan + // We'll handle ordering diversity in buildFinalResult to ensure we keep + // different orderings even if they have lower scores + if path.IsSingleScan || idxScore.interestingCount > 0 { + preferredIndexes = append(preferredIndexes, idxScore) + } + } + + // Build final result by sorting and selecting top indexes + maxToKeep := max(threshold, defaultMaxIndexes) + result := buildFinalResult(tablePaths, mvIndexPaths, indexMergeIndexPaths, preferredIndexes, maxToKeep, onlyPruneZeroScore, req) + + failpoint.InjectCall("InjectCheckForIndexPrune", result) + + // Safety check: if we ended up with nothing, return the original paths + if len(result) == 0 { + return paths + } + + // Additional safety: if we only have table paths and MVIndex paths and no regular indexes, keep original + if len(result) == len(tablePaths)+len(mvIndexPaths) && len(preferredIndexes) == 0 { + return paths + } + + return result +} + +// buildColumnRequirements builds column ID maps for efficient lookup. +func buildColumnRequirements(interestingColumns []*expression.Column) columnRequirements { + req := columnRequirements{ + interestingColIDs: make(map[int64]struct{}, len(interestingColumns)), + } + + // Build interesting column IDs + for _, col := range interestingColumns { + req.interestingColIDs[col.ID] = struct{}{} + } + + return req +} + +// buildOrderingKey creates a string key representing the consecutive column ordering. +// This is used to detect and keep indexes with different orderings. +func buildOrderingKey(columnIDs []int64) string { + if len(columnIDs) == 0 { + return "" + } + // Create a simple string representation of the column ID sequence + // Using a format like "1,2,3" for columns with IDs 1, 2, 3 + var builder strings.Builder + // Pre-allocate capacity: estimate ~4 bytes per ID (for small IDs) + commas + builder.Grow(len(columnIDs) * 5) + for i, id := range columnIDs { + if i > 0 { + builder.WriteString(",") + } + builder.WriteString(fmt.Sprintf("%d", id)) + } + return builder.String() +} + +// scoreIndexPath calculates coverage metrics for a single index path. +// When FullIdxCols is nil (e.g., in static pruning mode), it uses path.Index.Columns +// and tableColumns to determine interesting columns. Note that consecutiveColumnIDs +// cannot be determined when FullIdxCols is nil, so it will remain empty. +func scoreIndexPath( + ds *logicalop.DataSource, + path *util.AccessPath, + req columnRequirements, + tableColumns []*model.ColumnInfo, +) indexWithScore { + score := indexWithScore{path: path} + + if path.Index != nil && path.Index.ConditionExprString != "" { + for _, col := range path.Index.AffectColumn { + // Some columns from the constraint is not found. Then the path can not be selected. + // We mixed the WHERE clause and other clauses like JOIN/ORDER BY to prune the indexes. + // So this check is not the strictest one. + if _, found := req.interestingColIDs[ds.TableInfo.Columns[col.Offset].ID]; !found { + return score + } + } + // Pre check for partial index passed, continue to calculate the score. + } + + if path.FullIdxCols != nil { + // Normal path: use FullIdxCols which contains expression.Column with IDs + for i, idxCol := range path.FullIdxCols { + if idxCol == nil { + continue + } + idxColID := idxCol.ID + + // Check if this index column matches an interesting column + if _, found := req.interestingColIDs[idxColID]; found { + score.interestingCount++ + // Track consecutive columns from the start of the index + if i == len(score.consecutiveColumnIDs) { + score.consecutiveColumnIDs = append(score.consecutiveColumnIDs, idxColID) + } + } + // Note: We continue checking all columns to count all interesting columns, + // even if they're not consecutive from the start. The consecutive tracking + // will naturally stop once we hit a non-interesting column, since the condition + // `i == len(score.consecutiveColumnIDs)` will no longer be true. + } + } else if path.Index != nil && tableColumns != nil { + // Fallback path: use Index.Columns (for static pruning mode when FullIdxCols is nil) + // Map IndexColumn.Offset to column ID via tableColumns + for _, idxCol := range path.Index.Columns { + if idxCol.Offset < 0 || idxCol.Offset >= len(tableColumns) { + continue + } + colInfo := tableColumns[idxCol.Offset] + if colInfo == nil { + continue + } + idxColID := colInfo.ID + + // Check if this index column matches an interesting column + if _, found := req.interestingColIDs[idxColID]; found { + score.interestingCount++ + // Note: We cannot track consecutiveColumnIDs here because we don't have + // the full column information needed to determine if columns are consecutive + // in the index. This is acceptable as the user indicated. + } + } + } + + return score +} + +// buildFinalResult sorts and selects the top indexes to keep, combining table paths, +// multi-value indexes, index merge indexes, and preferred indexes. +type scoredIndex struct { + info indexWithScore + score int + columns int + isSingleScan bool + totalConsecutive int +} + +func scoreAndSort(indexes []indexWithScore, req columnRequirements) []scoredIndex { + if len(indexes) == 0 { + return nil + } + scored := make([]scoredIndex, 0, len(indexes)) + for _, candidate := range indexes { + score := calculateScoreFromCoverage(candidate, len(req.interestingColIDs), candidate.path.IsSingleScan) + // Skip indexes with score == 0 as they don't provide any value + if score == 0 { + continue + } + cols := len(candidate.path.FullIdxCols) + scored = append(scored, scoredIndex{ + info: candidate, + score: score, + columns: cols, + isSingleScan: candidate.path.IsSingleScan, + totalConsecutive: len(candidate.consecutiveColumnIDs), + }) + } + slices.SortFunc(scored, func(a, b scoredIndex) int { + // Tie-breaker: prefer indexes with higher score + if a.score != b.score { + return b.score - a.score + } + // Tie-breaker: prefer indexes with more consecutive columns + if a.totalConsecutive != b.totalConsecutive { + return b.totalConsecutive - a.totalConsecutive + } + // Tie-breaker: prefer indexes with single-scan + if a.isSingleScan != b.isSingleScan { + if a.isSingleScan { + return -1 + } + return 1 + } + // Tie-breaker: prefer indexes with fewer columns if + // they have only 1 consecutive column. + if a.totalConsecutive == 1 && a.columns != b.columns { + return a.columns - b.columns + } + // Tie-breaker: use index ID for deterministic ordering when all other criteria are equal + // This ensures stable sorting for functionally identical indexes (e.g., k1 and k2 with same expressions) + if a.info.path.Index != nil && b.info.path.Index != nil { + // Use proper three-way comparison to avoid integer overflow + // (a.info.path.Index.ID is int64, casting the difference to int can overflow) + if a.info.path.Index.ID < b.info.path.Index.ID { + return -1 + } + if a.info.path.Index.ID > b.info.path.Index.ID { + return 1 + } + } + return 0 + }) + return scored +} + +func buildFinalResult(tablePaths, mvIndexPaths, indexMergeIndexPaths []*util.AccessPath, preferredIndexes []indexWithScore, maxToKeep int, onlyPruneZeroScore bool, req columnRequirements) []*util.AccessPath { + result := make([]*util.AccessPath, 0, len(tablePaths)+len(indexMergeIndexPaths)+len(preferredIndexes)) + + // CRITICAL: Always include table paths - this is mandatory for correctness + result = append(result, tablePaths...) + // CRITICAL: Always include multi-value index paths - we do not have sufficient + // information to determine if they should be pruned in this function. + result = append(result, mvIndexPaths...) + // CRITICAL: Always include indexes specified in IndexMerge hints - index merge needs them to build partial paths + result = append(result, indexMergeIndexPaths...) + + added := make(map[*util.AccessPath]struct{}, len(tablePaths)+len(mvIndexPaths)+len(indexMergeIndexPaths)) + for _, path := range tablePaths { + added[path] = struct{}{} + } + for _, path := range mvIndexPaths { + added[path] = struct{}{} + } + for _, path := range indexMergeIndexPaths { + added[path] = struct{}{} + } + + preferredScored := scoreAndSort(preferredIndexes, req) + + // Prune all the path with 0 score. + if onlyPruneZeroScore { + for _, entry := range preferredScored { + if _, ok := added[entry.info.path]; !ok { + result = append(result, entry.info.path) + } + } + return result + } + + // Apply two-phase selection to limit the number of indexes + phase1Limit := maxToKeep / 2 + selectionState := newIndexSelectionState(phase1Limit, maxToKeep) + + result = selectIndexes(preferredScored, added, req, selectionState, result) + + return result +} + +// indexSelectionState tracks state during two-phase index selection +type indexSelectionState struct { + phase1Limit int + remaining int + hasNonZeroScore bool + phase1Count int + seenConsecutiveColumnIDs map[int64]struct{} + seenOrderingKeys map[string]struct{} +} + +func newIndexSelectionState(phase1Limit, maxToKeep int) *indexSelectionState { + return &indexSelectionState{ + phase1Limit: phase1Limit, + remaining: maxToKeep, + seenConsecutiveColumnIDs: make(map[int64]struct{}), + seenOrderingKeys: make(map[string]struct{}), + } +} + +// selectIndexes performs two-phase selection of indexes +func selectIndexes(preferredScored []scoredIndex, added map[*util.AccessPath]struct{}, req columnRequirements, state *indexSelectionState, result []*util.AccessPath) []*util.AccessPath { + for _, entry := range preferredScored { + path := entry.info.path + if _, ok := added[path]; ok { + continue + } + if state.remaining == 0 { + break + } + if state.hasNonZeroScore && entry.score == 0 { + continue + } + + shouldAdd := shouldAddIndex(entry, path, req, state) + if shouldAdd { + result = append(result, path) + added[path] = struct{}{} + if entry.score > 0 { + state.hasNonZeroScore = true + } + state.remaining-- + } + } + return result +} + +// shouldAddIndex determines if an index should be added based on phase and diversity rules +func shouldAddIndex(entry scoredIndex, path *util.AccessPath, req columnRequirements, state *indexSelectionState) bool { + if state.phase1Count < state.phase1Limit { + // Phase 1: Keep top threshold/2 based solely on score + state.phase1Count++ + recordConsecutiveColumns(entry.info.consecutiveColumnIDs, state) + return true + } + + // Phase 2: Apply diversity rules + hasConsecutive := len(entry.info.consecutiveColumnIDs) > 0 + if hasConsecutive { + return shouldAddIndexWithConsecutive(entry.info.consecutiveColumnIDs, state) + } + + return shouldAddIndexWithoutConsecutive(entry, path, req, state) +} + +// recordConsecutiveColumns tracks consecutive column IDs and ordering keys +func recordConsecutiveColumns(consecutiveColumnIDs []int64, state *indexSelectionState) { + for _, colID := range consecutiveColumnIDs { + state.seenConsecutiveColumnIDs[colID] = struct{}{} + } + if len(consecutiveColumnIDs) > 0 { + orderingKey := buildOrderingKey(consecutiveColumnIDs) + state.seenOrderingKeys[orderingKey] = struct{}{} + } +} + +// shouldAddIndexWithConsecutive checks if an index with consecutive columns should be added in phase 2 +func shouldAddIndexWithConsecutive(consecutiveColumnIDs []int64, state *indexSelectionState) bool { + orderingKey := buildOrderingKey(consecutiveColumnIDs) + if _, seen := state.seenOrderingKeys[orderingKey]; seen { + return false + } + state.seenOrderingKeys[orderingKey] = struct{}{} + recordConsecutiveColumns(consecutiveColumnIDs, state) + return true +} + +// shouldAddIndexWithoutConsecutive checks if an index without consecutive columns should be added in phase 2 +func shouldAddIndexWithoutConsecutive(entry scoredIndex, path *util.AccessPath, req columnRequirements, state *indexSelectionState) bool { + if entry.info.interestingCount != 1 { + return true + } + + // For single-column indexes, check if the column is already covered by a consecutive column + singleColID := findSingleInterestingColumn(path, req) + if singleColID < 0 { + return true + } + + // Don't add if the column is already in a consecutive column, unless it's a single scan + _, covered := state.seenConsecutiveColumnIDs[singleColID] + return !covered || path.IsSingleScan +} + +// findSingleInterestingColumn finds the single interesting column ID in an index. +// Returns -1 if FullIdxCols is nil (e.g., in static pruning mode) since we cannot +// determine the specific column without FullIdxCols. This causes the caller to +// keep the index, which is the safe default. +func findSingleInterestingColumn(path *util.AccessPath, req columnRequirements) int64 { + if path.FullIdxCols == nil { + return -1 + } + for _, idxCol := range path.FullIdxCols { + if idxCol != nil { + if _, found := req.interestingColIDs[idxCol.ID]; found { + return idxCol.ID + } + } + } + return -1 +} + +// calculateScoreFromCoverage calculates a ranking score using already-computed coverage information. +// This avoids re-iterating through index columns. +func calculateScoreFromCoverage(info indexWithScore, totalColumns int, isSingleScan bool) int { + score := 0 + + // Score for interesting column coverage + score += info.interestingCount * 10 + + // Bonus for consecutive interesting columns from start (critical for index usage) + // Consecutive columns are much more valuable than scattered matches + // Index on (a,b,c,d) with interesting columns a, b, c can use first 3 columns + // But with interesting columns a, d, can only use first 1 column + score += len(info.consecutiveColumnIDs) * 10 + + // Bonus if the index is covering all interesting columns + if info.interestingCount == totalColumns { + score += 10 + } + + // Bonus for single-scan (covering index without table lookups) + if isSingleScan { + score += 20 + } + + return score +} diff --git a/pkg/planner/core/stats.go b/pkg/planner/core/stats.go index 560ab9ec23040..93f80d60516b0 100644 --- a/pkg/planner/core/stats.go +++ b/pkg/planner/core/stats.go @@ -135,7 +135,14 @@ func deriveStats4DataSource(lp base.LogicalPlan, colGroups [][]*expression.Colum ds.PushedDownConds[i] = expression.PushDownNot(exprCtx, expr) ds.PushedDownConds[i] = expression.EliminateNoPrecisionLossCast(exprCtx, ds.PushedDownConds[i]) } +<<<<<<< HEAD for _, path := range ds.PossibleAccessPaths { +======= + // Index pruning is now done earlier in CollectPredicateColumnsPoint to avoid loading stats for pruned indexes. + // Fill index paths for all paths + ds.CheckPartialIndexes() + for _, path := range ds.AllPossibleAccessPaths { +>>>>>>> 959bf330874 (planner: support basic usage of partial index (#65051)) if path.IsTablePath() { continue } diff --git a/pkg/planner/util/path.go b/pkg/planner/util/path.go index 5053474e0dc27..1ec86e7c6b659 100644 --- a/pkg/planner/util/path.go +++ b/pkg/planner/util/path.go @@ -120,6 +120,18 @@ type AccessPath struct { // This field is used to rebuild GroupedRanges from ranges using GroupRangesByCols(). // It's used in plan cache or Apply. GroupByColIdxs []int + + // PartIdxCondNotAlwaysValid indicates that this index path is not guaranteed to be valid + // for all parameter values when a partial index condition is involved. + // It's designed for partial indexes with a WHERE condition (for example, idx(b) WHERE a IS NOT NULL) + // and tracks whether that condition is always satisfied by the current filter. + // e.g. for partial index idx(b) WHERE a IS NOT NULL: + // - if the filter is "b = ? AND a > 0", the partial index condition is always satisfied, + // so PartIdxCondNotAlwaysValid is false (the index path is always valid); + // - if the filter is "b = ?" or "b = ? AND a IS NULL", the partial index condition may not hold, + // so PartIdxCondNotAlwaysValid is true (the index path is not always valid). + // We add this field to make the plan cache usable for partial indexes in these limited cases. + PartIdxCondNotAlwaysValid bool } // Clone returns a deep copy of the original AccessPath. @@ -157,6 +169,7 @@ func (path *AccessPath) Clone() *AccessPath { IndexLookUpPushDownBy: path.IndexLookUpPushDownBy, GroupedRanges: make([][]*ranger.Range, 0, len(path.GroupedRanges)), GroupByColIdxs: slices.Clone(path.GroupByColIdxs), + PartIdxCondNotAlwaysValid: path.PartIdxCondNotAlwaysValid, } if path.IndexMergeORSourceFilter != nil { ret.IndexMergeORSourceFilter = path.IndexMergeORSourceFilter.Clone() @@ -410,3 +423,46 @@ func (path *AccessPath) GetCol2LenFromAccessConds(ctx planctx.PlanContext) Col2L } return ExtractCol2Len(ctx.GetExprCtx().GetEvalCtx(), path.AccessConds, path.IdxCols, path.IdxColLens) } +<<<<<<< HEAD +======= + +// IsFullScanRange checks that a table scan does not have any filtering such that it can limit the range of +// the table scan. +func (path *AccessPath) IsFullScanRange(tableInfo *model.TableInfo) bool { + var unsignedIntHandle bool + if path.IsIntHandlePath && tableInfo.PKIsHandle { + if pkColInfo := tableInfo.GetPkColInfo(); pkColInfo != nil { + unsignedIntHandle = mysql.HasUnsignedFlag(pkColInfo.GetFlag()) + } + } + if ranger.HasFullRange(path.Ranges, unsignedIntHandle) { + return true + } + return false +} + +// IsUndetermined checks if the path is undetermined. +// The undetermined path is the one that may not be always valid. +// e.g. The multi value index for JSON is not always valid, because the index must be used with JSON functions. +func (path *AccessPath) IsUndetermined() bool { + if path.IsTablePath() || path.Index == nil { + return false + } + if path.Index.MVIndex || path.Index.ConditionExprString != "" { + return true + } + return false +} + +// IsIndexJoinUnapplicable checks if the path is unapplicable for index join. +// If path is mv index path: +// for mv index like mvi(a, json, b), if driving condition is a=1, and we build a prefix scan with range [1,1] +// on mvi, it will return many index rows which breaks handle-unique attribute here. +// So we cannot use mv index path for index join. +// If path is partial index path: +// We need to first determine whether we already meet the partial index condition. +// Currently we don't support that, so we conservatively return true here. +func (path *AccessPath) IsIndexJoinUnapplicable() bool { + return path.IsUndetermined() +} +>>>>>>> 959bf330874 (planner: support basic usage of partial index (#65051)) diff --git a/pkg/util/ranger/types.go b/pkg/util/ranger/types.go index 10b2beedd748c..9d48c8024ff61 100644 --- a/pkg/util/ranger/types.go +++ b/pkg/util/ranger/types.go @@ -268,6 +268,33 @@ func (ran *Range) Encode(ec errctx.Context, loc *time.Location, lowBuffer, highB return lowBuffer, highBuffer, nil } +// Equal checks if two ranges are equal. +func (ran *Range) Equal(other *Range) bool { + if ran == other { + return true + } + if ran == nil || other == nil { + return false + } + if ran.LowExclude != other.LowExclude || ran.HighExclude != other.HighExclude { + return false + } + if len(ran.LowVal) != len(other.LowVal) || len(ran.HighVal) != len(other.HighVal) { + return false + } + for i := range ran.LowVal { + if !ran.LowVal[i].Equals(other.LowVal[i]) { + return false + } + } + for i := range ran.HighVal { + if !ran.HighVal[i].Equals(other.HighVal[i]) { + return false + } + } + return true +} + // PrefixEqualLen tells you how long the prefix of the range is a point. // e.g. If this range is (1 2 3, 1 2 +inf), then the return value is 2. func (ran *Range) PrefixEqualLen(tc types.Context) (int, error) { diff --git a/tests/integrationtest/r/planner/core/casetest/index/partialindex.result b/tests/integrationtest/r/planner/core/casetest/index/partialindex.result new file mode 100644 index 0000000000000..0d8d7d469cb64 --- /dev/null +++ b/tests/integrationtest/r/planner/core/casetest/index/partialindex.result @@ -0,0 +1,110 @@ +create table t( +a int, b int, c int, +index idx1(a) where a is not null, +index idx2(b) where b > 2, +unique index idx3(c) where b > 10 +); +create table tt( +a int, b int, c int, +index idx1(a), +index idx2(b), +unique index idx3(c) +); +explain format='brief' select * from t where a = 1; +id estRows task access object operator info +IndexLookUp 10.00 root +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:idx1(a) range:[1,1], keep order:false, stats:pseudo +└─TableRowIDScan(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo +explain format='brief' select * from t use index(idx2) where b > 1; +id estRows task access object operator info +TableReader 3333.33 root data:Selection +└─Selection 3333.33 cop[tikv] gt(planner__core__casetest__index__partialindex.t.b, 1) + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +explain format='brief' select * from t use index(idx2) where b > 2; +id estRows task access object operator info +IndexLookUp 3333.33 root +├─IndexRangeScan(Build) 3333.33 cop[tikv] table:t, index:idx2(b) range:(2,+inf], keep order:false, stats:pseudo +└─TableRowIDScan(Probe) 3333.33 cop[tikv] table:t keep order:false, stats:pseudo +explain format='brief' select * from t use index(idx2) where b > 5; +id estRows task access object operator info +IndexLookUp 3333.33 root +├─IndexRangeScan(Build) 3333.33 cop[tikv] table:t, index:idx2(b) range:(5,+inf], keep order:false, stats:pseudo +└─TableRowIDScan(Probe) 3333.33 cop[tikv] table:t keep order:false, stats:pseudo +explain format='brief' select * from t use index(idx2) where b > 4 and b < 100; +id estRows task access object operator info +IndexLookUp 250.00 root +├─IndexRangeScan(Build) 250.00 cop[tikv] table:t, index:idx2(b) range:(4,100), keep order:false, stats:pseudo +└─TableRowIDScan(Probe) 250.00 cop[tikv] table:t keep order:false, stats:pseudo +explain format='brief' select * from t use index(idx2) where b < 100; +id estRows task access object operator info +TableReader 3323.33 root data:Selection +└─Selection 3323.33 cop[tikv] lt(planner__core__casetest__index__partialindex.t.b, 100) + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +explain format='brief' select * from t where c = 10 and b > 30; +id estRows task access object operator info +Selection 1.00 root gt(planner__core__casetest__index__partialindex.t.b, 30) +└─Point_Get 1.00 root table:t, index:idx3(c) +explain format='brief' select * from t where c = 10 and b > 40; +id estRows task access object operator info +Selection 1.00 root gt(planner__core__casetest__index__partialindex.t.b, 40) +└─Point_Get 1.00 root table:t, index:idx3(c) +explain format='brief' select * from t where c > 20; +id estRows task access object operator info +TableReader 3333.33 root data:Selection +└─Selection 3333.33 cop[tikv] gt(planner__core__casetest__index__partialindex.t.c, 20) + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +explain format='brief' select * from t where c = 1; +id estRows task access object operator info +TableReader 1.00 root data:Selection +└─Selection 1.00 cop[tikv] eq(planner__core__casetest__index__partialindex.t.c, 1) + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +explain format='brief' select /*+ inl_join(t2) */ * from t t1, t t2 where t1.a=t2.b; +id estRows task access object operator info +HashJoin 12487.50 root inner join, equal:[eq(planner__core__casetest__index__partialindex.t.a, planner__core__casetest__index__partialindex.t.b)] +├─TableReader(Build) 9990.00 root data:Selection +│ └─Selection 9990.00 cop[tikv] not(isnull(planner__core__casetest__index__partialindex.t.b)) +│ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo +└─TableReader(Probe) 9990.00 root data:Selection + └─Selection 9990.00 cop[tikv] not(isnull(planner__core__casetest__index__partialindex.t.a)) + └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +explain format='brief' select /*+ inl_join(t2) */ * from tt t1, tt t2 where t1.a=t2.b; +id estRows task access object operator info +IndexJoin 12487.50 root inner join, inner:IndexLookUp, outer key:planner__core__casetest__index__partialindex.tt.a, inner key:planner__core__casetest__index__partialindex.tt.b, equal cond:eq(planner__core__casetest__index__partialindex.tt.a, planner__core__casetest__index__partialindex.tt.b) +├─TableReader(Build) 9990.00 root data:Selection +│ └─Selection 9990.00 cop[tikv] not(isnull(planner__core__casetest__index__partialindex.tt.a)) +│ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +└─IndexLookUp(Probe) 12487.50 root + ├─Selection(Build) 12487.50 cop[tikv] not(isnull(planner__core__casetest__index__partialindex.tt.b)) + │ └─IndexRangeScan 12500.00 cop[tikv] table:t2, index:idx2(b) range: decided by [eq(planner__core__casetest__index__partialindex.tt.b, planner__core__casetest__index__partialindex.tt.a)], keep order:false, stats:pseudo + └─TableRowIDScan(Probe) 12487.50 cop[tikv] table:t2 keep order:false, stats:pseudo +explain format='brief' select /*+ inl_join(t2) */ * from t t1, t t2 where t1.b=t2.a; +id estRows task access object operator info +HashJoin 12487.50 root inner join, equal:[eq(planner__core__casetest__index__partialindex.t.b, planner__core__casetest__index__partialindex.t.a)] +├─TableReader(Build) 9990.00 root data:Selection +│ └─Selection 9990.00 cop[tikv] not(isnull(planner__core__casetest__index__partialindex.t.a)) +│ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo +└─TableReader(Probe) 9990.00 root data:Selection + └─Selection 9990.00 cop[tikv] not(isnull(planner__core__casetest__index__partialindex.t.b)) + └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +explain format='brief' select /*+ inl_join(t2) */ * from tt t1, tt t2 where t1.b=t2.a; +id estRows task access object operator info +IndexJoin 12487.50 root inner join, inner:IndexLookUp, outer key:planner__core__casetest__index__partialindex.tt.b, inner key:planner__core__casetest__index__partialindex.tt.a, equal cond:eq(planner__core__casetest__index__partialindex.tt.b, planner__core__casetest__index__partialindex.tt.a) +├─TableReader(Build) 9990.00 root data:Selection +│ └─Selection 9990.00 cop[tikv] not(isnull(planner__core__casetest__index__partialindex.tt.b)) +│ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +└─IndexLookUp(Probe) 12487.50 root + ├─Selection(Build) 12487.50 cop[tikv] not(isnull(planner__core__casetest__index__partialindex.tt.a)) + │ └─IndexRangeScan 12500.00 cop[tikv] table:t2, index:idx1(a) range: decided by [eq(planner__core__casetest__index__partialindex.tt.a, planner__core__casetest__index__partialindex.tt.b)], keep order:false, stats:pseudo + └─TableRowIDScan(Probe) 12487.50 cop[tikv] table:t2 keep order:false, stats:pseudo +explain format='brief' select /*+ use_index_merge(t, idx1, idx2) */ * from t where (a > 10) or (b > 10); +id estRows task access object operator info +TableReader 5555.56 root data:Selection +└─Selection 5555.56 cop[tikv] or(gt(planner__core__casetest__index__partialindex.t.a, 10), gt(planner__core__casetest__index__partialindex.t.b, 10)) + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +explain format='brief' select /*+ use_index_merge(tt, idx1, idx2) */ * from tt where (a > 10) or (b > 10); +id estRows task access object operator info +IndexMerge 5555.56 root type: union +├─IndexRangeScan(Build) 3333.33 cop[tikv] table:tt, index:idx1(a) range:(10,+inf], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 3333.33 cop[tikv] table:tt, index:idx2(b) range:(10,+inf], keep order:false, stats:pseudo +└─TableRowIDScan(Probe) 5555.56 cop[tikv] table:tt keep order:false, stats:pseudo +drop table t, tt; diff --git a/tests/integrationtest/t/planner/core/casetest/index/partialindex.test b/tests/integrationtest/t/planner/core/casetest/index/partialindex.test new file mode 100644 index 0000000000000..652e12eb90b47 --- /dev/null +++ b/tests/integrationtest/t/planner/core/casetest/index/partialindex.test @@ -0,0 +1,39 @@ +# Test cases for partial index. +create table t( + a int, b int, c int, + index idx1(a) where a is not null, + index idx2(b) where b > 2, + unique index idx3(c) where b > 10 +); +create table tt( + a int, b int, c int, + index idx1(a), + index idx2(b), + unique index idx3(c) +); + +# basic tests. +explain format='brief' select * from t where a = 1; +explain format='brief' select * from t use index(idx2) where b > 1; +explain format='brief' select * from t use index(idx2) where b > 2; +explain format='brief' select * from t use index(idx2) where b > 5; +explain format='brief' select * from t use index(idx2) where b > 4 and b < 100; +explain format='brief' select * from t use index(idx2) where b < 100; +explain format='brief' select * from t where c = 10 and b > 30; +explain format='brief' select * from t where c = 10 and b > 40; +explain format='brief' select * from t where c > 20; + +# test the TryFastPath short-circuit. +explain format='brief' select * from t where c = 1; + +# test index join(reject) +explain format='brief' select /*+ inl_join(t2) */ * from t t1, t t2 where t1.a=t2.b; +explain format='brief' select /*+ inl_join(t2) */ * from tt t1, tt t2 where t1.a=t2.b; +explain format='brief' select /*+ inl_join(t2) */ * from t t1, t t2 where t1.b=t2.a; +explain format='brief' select /*+ inl_join(t2) */ * from tt t1, tt t2 where t1.b=t2.a; + +# test index merge(reject) +explain format='brief' select /*+ use_index_merge(t, idx1, idx2) */ * from t where (a > 10) or (b > 10); +explain format='brief' select /*+ use_index_merge(tt, idx1, idx2) */ * from tt where (a > 10) or (b > 10); + +drop table t, tt;